I’m building a prototype app that showcases the functionality on iOS to capture a device’s screen, connect to another device, and then video stream this (as high res and low latency) over a local WiFi network. The chosen lib I’m implementing this with is WebRTC (open to other suggestions).
The Issue:
I load the app onto 2 devices, I broadcast the screen of one device, it appears from the logs to connect to the other device I have set to receive, but after that they appear to drop connection with “[GCKSession] Not in connected state, so giving up for participant [xxxxxxxx] on channel [x]” on 5 channel.
I’ve included the full code you can run in Xcode below along with a console from the connecting device during the last build and dropped connection.
I’m at a frustrating DEAD END.
WebRTCManager.swift
import Foundation
import WebRTC
import ReplayKit
import MultipeerConnectivity
class WebRTCManager: NSObject, ObservableObject {
private var peerConnection: RTCPeerConnection?
@Published var localVideoTrack: RTCVideoTrack?
@Published var remoteVideoTrack: RTCVideoTrack?
private var peerConnectionFactory: RTCPeerConnectionFactory?
private var videoSource: RTCVideoSource?
private var videoCapturer: RTCVideoCapturer?
private var peerID: MCPeerID
private var session: MCSession
private var advertiser: MCNearbyServiceAdvertiser?
private var browser: MCNearbyServiceBrowser?
@Published var connectedPeers: [MCPeerID] = []
@Published var localSDP: String = ""
@Published var localICECandidates: [String] = []
@Published var isBroadcasting: Bool = false
@Published var remoteTrackAdded: Bool = false
private var isConnected = false
override init() {
RTCInitializeSSL()
peerConnectionFactory = RTCPeerConnectionFactory()
peerID = MCPeerID(displayName: UIDevice.current.name)
session = MCSession(peer: peerID, securityIdentity: nil, encryptionPreference: .none)
super.init()
session.delegate = self
}
func startBroadcasting() {
isBroadcasting = true
advertiser = MCNearbyServiceAdvertiser(peer: peerID, discoveryInfo: nil, serviceType: "screen-share")
advertiser?.delegate = self
advertiser?.startAdvertisingPeer()
setupPeerConnection()
setupVideoSource()
startScreenCapture()
}
func startReceiving() {
browser = MCNearbyServiceBrowser(peer: peerID, serviceType: "screen-share")
browser?.delegate = self
browser?.startBrowsingForPeers()
setupPeerConnection()
}
private func setupPeerConnection() {
let configuration = RTCConfiguration()
configuration.iceServers = [RTCIceServer(urlStrings: ["stun:stun.l.google.com:19302"])]
configuration.sdpSemantics = .unifiedPlan
let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
peerConnection = peerConnectionFactory?.peerConnection(with: configuration, constraints: constraints, delegate: self)
// Ensure peer connection state is checked before proceeding
guard let peerConnection = peerConnection else {
print("Failed to create peer connection")
return
}
peerConnection.delegate = self
}
private func setupVideoSource() {
videoSource = peerConnectionFactory?.videoSource()
#if targetEnvironment(simulator)
videoCapturer = RTCFileVideoCapturer(delegate: videoSource!)
#else
videoCapturer = RTCCameraVideoCapturer(delegate: videoSource!)
#endif
localVideoTrack = peerConnectionFactory?.videoTrack(with: videoSource!, trackId: "video0")
if let localVideoTrack = localVideoTrack {
peerConnection?.add(localVideoTrack, streamIds: ["stream0"])
print("Local video track added to peer connection")
} else {
print("Failed to create local video track")
}
}
private func startScreenCapture() {
let recorder = RPScreenRecorder.shared()
recorder.startCapture { [weak self] (sampleBuffer, type, error) in
guard let self = self else { return }
if let error = error {
print("Error starting screen capture: (error)")
return
}
if type == .video {
self.processSampleBuffer(sampleBuffer, with: type)
}
} completionHandler: { error in
if let error = error {
print("Error in screen capture completion: (error)")
} else {
print("Screen capture started successfully")
}
}
}
private func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
switch sampleBufferType {
case .video:
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let timestamp = NSDate().timeIntervalSince1970 * 1000
let videoFrame = RTCVideoFrame(buffer: RTCCVPixelBuffer(pixelBuffer: pixelBuffer),
rotation: ._0,
timeStampNs: Int64(timestamp))
self.videoSource?.capturer(self.videoCapturer!, didCapture: videoFrame)
default:
break
}
}
func stopBroadcasting() {
isBroadcasting = false
advertiser?.stopAdvertisingPeer()
advertiser = nil
stopScreenCapture()
closePeerConnection()
}
func stopReceiving() {
browser?.stopBrowsingForPeers()
browser = nil
closePeerConnection()
}
private func stopScreenCapture() {
RPScreenRecorder.shared().stopCapture { [weak self] error in
if let error = error {
print("Error stopping screen capture: (error)")
} else {
print("Screen capture stopped successfully")
DispatchQueue.main.async {
self?.localVideoTrack = nil
}
}
}
}
private func closePeerConnection() {
guard let peerConnection = peerConnection else { return }
if isConnected {
// Properly handle disconnection if connected
peerConnection.close()
}
self.peerConnection = nil
DispatchQueue.main.async {
self.remoteVideoTrack = nil
self.localSDP = ""
self.localICECandidates.removeAll()
}
print("Peer connection closed")
}
private func createOffer() {
print("Creating offer")
let constraints = RTCMediaConstraints(mandatoryConstraints: [
"OfferToReceiveVideo": "true",
"OfferToReceiveAudio": "false"
], optionalConstraints: nil)
peerConnection?.offer(for: constraints) { [weak self] sdp, error in
guard let self = self, let sdp = sdp else {
print("Failed to create offer: (error?.localizedDescription ?? "unknown error")")
return
}
self.peerConnection?.setLocalDescription(sdp) { error in
if let error = error {
print("Error setting local description: (error)")
} else {
print("Local description (offer) set successfully")
self.sendSDP(sdp)
}
}
}
}
private func createAnswer() {
print("Creating answer")
let constraints = RTCMediaConstraints(mandatoryConstraints: [
"OfferToReceiveVideo": "true",
"OfferToReceiveAudio": "false"
], optionalConstraints: nil)
peerConnection?.answer(for: constraints) { [weak self] sdp, error in
guard let self = self, let sdp = sdp else {
print("Failed to create answer: (error?.localizedDescription ?? "unknown error")")
return
}
self.peerConnection?.setLocalDescription(sdp) { error in
if let error = error {
print("Error setting local description: (error)")
} else {
print("Local description (answer) set successfully")
self.sendSDP(sdp)
}
}
}
}
private func setRemoteDescription(_ sdp: RTCSessionDescription) {
peerConnection?.setRemoteDescription(sdp) { error in
if let error = error {
print("Error setting remote description: (error)")
} else {
print("Remote description set successfully")
}
}
}
private func addIceCandidate(_ candidate: RTCIceCandidate) {
guard let peerConnection = peerConnection, isConnected else {
print("Cannot add ICE candidate, peer connection is not connected")
return
}
peerConnection.add(candidate)
print("ICE candidate added")
}
private func sendSDP(_ sdp: RTCSessionDescription) {
let dict: [String: Any] = ["type": sdp.type.rawValue, "sdp": sdp.sdp]
if let data = try? JSONSerialization.data(withJSONObject: dict, options: []) {
do {
try session.send(data, toPeers: session.connectedPeers, with: .reliable)
print("SDP sent to peers")
} catch {
print("Failed to send SDP: (error)")
}
}
}
}
extension WebRTCManager: MCSessionDelegate, MCNearbyServiceAdvertiserDelegate, MCNearbyServiceBrowserDelegate {
func session(_ session: MCSession, peer peerID: MCPeerID, didChange state: MCSessionState) {
DispatchQueue.main.async {
self.connectedPeers = session.connectedPeers
switch state {
case .connected:
print("Peer connected: (peerID.displayName)")
self.browser?.stopBrowsingForPeers()
self.advertiser?.stopAdvertisingPeer()
self.isConnected = true // Set isConnected to true here
if self.isBroadcasting {
self.createOffer()
}
case .connecting:
print("Peer connecting: (peerID.displayName)")
case .notConnected:
print("Peer not connected: (peerID.displayName)")
self.isConnected = false
@unknown default:
print("Unknown state: (peerID.displayName)")
}
}
}
func session(_ session: MCSession, didReceive data: Data, fromPeer peerID: MCPeerID) {
let dict = try? JSONSerialization.jsonObject(with: data, options: []) as? [String: Any]
if let typeInt = dict?["type"] as? Int, let sdp = dict?["sdp"] as? String,
let type = RTCSdpType(rawValue: typeInt) {
let rtcSdp = RTCSessionDescription(type: type, sdp: sdp)
self.peerConnection?.setRemoteDescription(rtcSdp) { [weak self] error in
if let error = error {
print("Error setting remote description: (error)")
} else {
print("Remote description set successfully")
if type == .offer {
self?.createAnswer()
}
}
}
} else if let sdp = dict?["candidate"] as? String,
let sdpMid = dict?["sdpMid"] as? String,
let sdpMLineIndexString = dict?["sdpMLineIndex"] as? String,
let sdpMLineIndex = Int32(sdpMLineIndexString) {
let candidate = RTCIceCandidate(sdp: sdp, sdpMLineIndex: sdpMLineIndex, sdpMid: sdpMid)
self.peerConnection?.add(candidate)
}
}
func session(_ session: MCSession, didReceive stream: InputStream, withName streamName: String, fromPeer peerID: MCPeerID) {}
func session(_ session: MCSession, didStartReceivingResourceWithName resourceName: String, fromPeer peerID: MCPeerID, with progress: Progress) {}
func session(_ session: MCSession, didFinishReceivingResourceWithName resourceName: String, fromPeer peerID: MCPeerID, at localURL: URL?, withError error: Error?) {}
func advertiser(_ advertiser: MCNearbyServiceAdvertiser, didReceiveInvitationFromPeer peerID: MCPeerID, withContext context: Data?, invitationHandler: @escaping (Bool, MCSession?) -> Void) {
invitationHandler(true, session)
}
func browser(_ browser: MCNearbyServiceBrowser, foundPeer peerID: MCPeerID, withDiscoveryInfo info: [String : String]?) {
browser.invitePeer(peerID, to: session, withContext: nil, timeout: 30)
}
func browser(_ browser: MCNearbyServiceBrowser, lostPeer peerID: MCPeerID) {}
}
extension WebRTCManager: RTCPeerConnectionDelegate {
func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) {
print("Signaling state changed: (stateChanged.rawValue)")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {
print("Stream added with ID: (stream.streamId)")
if let videoTrack = stream.videoTracks.first {
print("Video track added: (videoTrack.trackId)")
DispatchQueue.main.async {
self.remoteVideoTrack = videoTrack
self.remoteTrackAdded = true
self.objectWillChange.send()
print("Remote video track set")
}
} else {
print("No video tracks in the stream")
}
}
func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {
print("Stream removed")
}
func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {
print("Negotiation needed")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
print("ICE connection state changed: (newState.rawValue)")
switch newState {
case .checking, .connected, .completed:
print("ICE connected")
self.isConnected = true
case .failed, .disconnected, .closed:
print("ICE connection failed or closed")
self.isConnected = false
// Handle reconnection or cleanup if necessary
case .new:
print("New ICE connection")
case .count:
print("ICE count")
@unknown default:
print("Unknown ICE state")
}
}
func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState) {
print("ICE gathering state changed: (newState.rawValue)")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {
print("ICE candidate generated: (candidate.sdp)")
DispatchQueue.main.async {
self.localICECandidates.append(candidate.sdp)
}
// Always send ICE candidates
let dict: [String: Any] = ["candidate": candidate.sdp, "sdpMid": candidate.sdpMid ?? "", "sdpMLineIndex": candidate.sdpMLineIndex]
if let data = try? JSONSerialization.data(withJSONObject: dict, options: []) {
do {
try session.send(data, toPeers: session.connectedPeers, with: .reliable)
print("ICE candidate sent to peers")
} catch {
print("Failed to send ICE candidate: (error)")
}
}
}
func peerConnection(_ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate]) {
print("Removed ICE candidates")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) {
print("Data channel opened")
}
}
extension Notification.Name {
static let remoteVideoTrackAdded = Notification.Name("remoteVideoTrackAdded")
}
RTCVideoView.swift
import SwiftUI
import WebRTC
struct RTCVideoView: UIViewRepresentable {
@ObservedObject var webRTCManager: WebRTCManager
var isLocal: Bool
func makeUIView(context: Context) -> RTCMTLVideoView {
let videoView = RTCMTLVideoView(frame: .zero)
videoView.videoContentMode = .scaleAspectFit
updateVideoTrack(videoView)
return videoView
}
func updateUIView(_ uiView: RTCMTLVideoView, context: Context) {
updateVideoTrack(uiView)
}
private func updateVideoTrack(_ uiView: RTCMTLVideoView) {
if isLocal {
if let localVideoTrack = webRTCManager.localVideoTrack {
localVideoTrack.add(uiView)
print("Local video track added to view")
} else {
print("Local video track is nil")
}
} else {
if let remoteVideoTrack = webRTCManager.remoteVideoTrack {
remoteVideoTrack.add(uiView)
print("Remote video track added to view")
} else {
print("Remote video track is nil")
}
}
}
func makeCoordinator() -> Coordinator {
Coordinator(self)
}
class Coordinator: NSObject {
var parent: RTCVideoView
init(_ parent: RTCVideoView) {
self.parent = parent
super.init()
NotificationCenter.default.addObserver(self, selector: #selector(remoteVideoTrackAdded), name: .remoteVideoTrackAdded, object: nil)
}
@objc func remoteVideoTrackAdded() {
DispatchQueue.main.async {
self.parent.webRTCManager.objectWillChange.send()
}
}
}
}
ContentView.swift
import SwiftUI
import WebRTC
import MultipeerConnectivity
import ReplayKit
struct ContentView: View {
@StateObject private var webRTCManager = WebRTCManager()
@State private var isBroadcasting = false
@State private var isReceiving = false
var body: some View {
VStack {
if isBroadcasting {
Text("Broadcasting")
.font(.headline)
// Local video preview
RTCVideoView(webRTCManager: webRTCManager, isLocal: true)
.frame(height: 200)
.background(Color.gray.opacity(0.3)) // Add a semi-transparent gray background
.cornerRadius(10)
Button("Stop Broadcasting") {
webRTCManager.stopBroadcasting()
isBroadcasting = false
}
.padding()
.background(Color.red)
.foregroundColor(.white)
.cornerRadius(10)
// ... (existing code for SDP and ICE candidates)
} else if isReceiving {
Text("Receiving")
.font(.headline)
RTCVideoView(webRTCManager: webRTCManager, isLocal: false)
.frame(height: 300)
.background(Color.gray.opacity(0.3)) // Add a semi-transparent gray background
.cornerRadius(10)
Button("Stop Receiving") {
webRTCManager.stopReceiving()
isReceiving = false
}
.padding()
.background(Color.red)
.foregroundColor(.white)
.cornerRadius(10)
} else {
Button("Start Broadcasting") {
webRTCManager.startBroadcasting()
isBroadcasting = true
}
.padding()
.background(Color.blue)
.foregroundColor(.white)
.cornerRadius(10)
Button("Start Receiving") {
webRTCManager.startReceiving()
isReceiving = true
}
.padding()
.background(Color.green)
.foregroundColor(.white)
.cornerRadius(10)
}
Text("Connected Peers: (webRTCManager.connectedPeers.count)")
.font(.headline)
.padding()
if !webRTCManager.connectedPeers.isEmpty {
Text("Connected to:")
ForEach(webRTCManager.connectedPeers, id: .self) { peer in
Text(peer.displayName)
}
}
}
.padding()
}
}
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
ContentView()
}
}
Console of connecting device:
2024-07-08 22:04:22.574897+0100 ScreenShare[2745:914162] Metal API Validation Enabled
Remote video track is nil
Remote video track is nil
2024-07-08 22:04:56.058516+0100 ScreenShare[2745:914299] [Client] Updating selectors after delegate removal failed with: Error Domain=NSCocoaErrorDomain Code=4099 "The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process." UserInfo={NSDebugDescription=The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process.}
2024-07-08 22:04:56.058657+0100 ScreenShare[2745:914299] [Client] Updating selectors after delegate addition failed with: Error Domain=NSCocoaErrorDomain Code=4099 "The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process." UserInfo={NSDebugDescription=The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this process.}
Peer connecting: iPhone
Remote video track is nil
Peer connected: iPhone
Remote video track is nil
Signaling state changed: 3
Stream added with ID: stream0
Video track added: video0
Remote description set successfully
Creating answer
Remote video track set
Remote video track added to view
Signaling state changed: 0
Local description (answer) set successfully
SDP sent to peers
ICE gathering state changed: 1
ICE candidate generated: candidate:617392483 1 udp 2122260223 192.168.1.112 63716 typ host generation 0 ufrag umCW network-id 1 network-cost 10
ICE candidate sent to peers
ICE candidate generated: candidate:3503244297 1 udp 2122194687 169.254.104.95 64683 typ host generation 0 ufrag umCW network-id 2 network-cost 10
ICE candidate sent to peers
Remote video track added to view
Remote video track added to view
ICE candidate generated: candidate:1783584147 1 tcp 1518280447 192.168.1.112 51096 typ host tcptype passive generation 0 ufrag umCW network-id 1 network-cost 10
ICE candidate sent to peers
ICE candidate generated: candidate:2655828217 1 tcp 1518214911 169.254.104.95 51097 typ host tcptype passive generation 0 ufrag umCW network-id 2 network-cost 10
ICE candidate sent to peers
Remote video track added to view
ICE candidate generated: candidate:2776936407 1 udp 1686052607 2.28.217.67 63716 typ srflx raddr 192.168.1.112 rport 63716 generation 0 ufrag umCW network-id 1 network-cost 10
ICE candidate sent to peers
Remote video track added to view
Remote video track added to view
2024-07-08 22:05:06.151870+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [0].
2024-07-08 22:05:06.155864+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [1].
2024-07-08 22:05:06.158066+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [2].
2024-07-08 22:05:06.159428+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [3].
2024-07-08 22:05:06.160762+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [4].
2024-07-08 22:05:06.161831+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [5].
2024-07-08 22:05:06.162682+0100 ScreenShare[2745:914169] [GCKSession] Not in connected state, so giving up for participant [780785AF] on channel [6].
ICE gathering state changed: 2
I have fiddled with the projects settings:
• Added every vaguely sounding relevant background mode
• Anything I’ve found could be causing the issue due to its absence from the Info.plist and adding it:
<plist version="1.0">
<dict>
<key>NSBonjourServices</key>
<array>
<string>_screen-share._tcp</string>
<string>_screen-share._udp</string>
</array>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
<key>UISceneConfigurations</key>
<dict/>
</dict>
<key>UIBackgroundModes</key>
<array>
<string>audio</string>
<string>fetch</string>
<string>nearby-interaction</string>
<string>processing</string>
<string>voip</string>
</array>
</dict>
</plist>