From 7ce6595b3d9c37bccc3b12f179adac9b6b008ec3 Mon Sep 17 00:00:00 2001 From: td Date: Fri, 17 Feb 2023 07:56:17 +0530 Subject: [PATCH] fix: ability to upgrade audio calls to video calls fix: setMicrophoneMuted is now async to match setVideoMuted --- lib/src/voip/call.dart | 134 ++++++++++++++++++++++++++++++----- lib/src/voip/group_call.dart | 14 ++-- 2 files changed, 123 insertions(+), 25 deletions(-) diff --git a/lib/src/voip/call.dart b/lib/src/voip/call.dart index 59f33351..621a9b46 100644 --- a/lib/src/voip/call.dart +++ b/lib/src/voip/call.dart @@ -410,6 +410,20 @@ class CallSession { return null; } + /// returns whether a 1:1 call sender has video tracks + Future hasVideoToSend() async { + final transceivers = await pc!.getTransceivers(); + final localUserMediaVideoTrack = localUserMediaStream?.stream + ?.getTracks() + .singleWhereOrNull((track) => track.kind == 'video'); + + // check if we have a video track locally and have transceivers setup correctly. + return localUserMediaVideoTrack != null && + transceivers.singleWhereOrNull((transceiver) => + transceiver.sender.track?.id == localUserMediaVideoTrack.id) != + null; + } + Timer? inviteTimer; Timer? ringingTimer; @@ -622,8 +636,15 @@ class CallSession { try { await pc!.setRemoteDescription(description); + RTCSessionDescription? answer; if (description.type == 'offer') { - final answer = await pc!.createAnswer({}); + try { + answer = await pc!.createAnswer({}); + } catch (e) { + await terminate(CallParty.kLocal, CallErrorCode.CreateAnswer, true); + return; + } + await sendCallNegotiate( room, callId, Timeouts.lifetimeMs, localPartyId, answer.sdp!, type: answer.type!); @@ -747,7 +768,9 @@ class CallSession { if (stream == null) { return false; } - stream.getVideoTracks().forEach((track) { + stream.getTracks().forEach((track) { + // screen sharing should only have 1 video track anyway, so this only + // fires once track.onEnded = () { setScreensharingEnabled(false); }; @@ -761,16 +784,21 @@ class CallSession { return false; } } else { - for (final sender in screensharingSenders) { - await pc!.removeTrack(sender); + try { + for (final sender in screensharingSenders) { + await pc!.removeTrack(sender); + } + for (final track in localScreenSharingStream!.stream!.getTracks()) { + await track.stop(); + } + localScreenSharingStream!.stopped = true; + await _removeStream(localScreenSharingStream!.stream!); + fireCallEvent(CallEvent.kFeedsChanged); + return false; + } catch (e) { + Logs().e('[VOIP] stopping screen sharing track failed', e); + return false; } - for (final track in localScreenSharingStream!.stream!.getTracks()) { - await track.stop(); - } - localScreenSharingStream!.stopped = true; - await _removeStream(localScreenSharingStream!.stream!); - fireCallEvent(CallEvent.kFeedsChanged); - return false; } } @@ -918,16 +946,85 @@ class CallSession { fireCallEvent(CallEvent.kState); } - void setLocalVideoMuted(bool muted) { + Future setLocalVideoMuted(bool muted) async { + if (!muted) { + final videoToSend = await hasVideoToSend(); + if (!videoToSend) { + if (remoteSDPStreamMetadata == null) return; + await insertVideoTrackToAudioOnlyStream(); + } + } localUserMediaStream?.setVideoMuted(muted); - _updateMuteStatus(); + await _updateMuteStatus(); + } + + // used for upgrading 1:1 calls + Future insertVideoTrackToAudioOnlyStream() async { + if (localUserMediaStream != null && localUserMediaStream!.stream != null) { + final stream = await _getUserMedia(CallType.kVideo); + if (stream != null) { + Logs().e('[VOIP] running replaceTracks() on stream: ${stream.id}'); + _setTracksEnabled(stream.getVideoTracks(), true); + // replace local tracks + for (final track in localUserMediaStream!.stream!.getTracks()) { + try { + await localUserMediaStream!.stream!.removeTrack(track); + await track.stop(); + } catch (e) { + Logs().w('failed to stop track'); + } + } + final streamTracks = stream.getTracks(); + for (final newTrack in streamTracks) { + await localUserMediaStream!.stream!.addTrack(newTrack); + } + + // remove any screen sharing or remote transceivers, these don't need + // to be replaced anyway. + final transceivers = await pc!.getTransceivers(); + transceivers.removeWhere((transceiver) => + transceiver.sender.track == null || + (localScreenSharingStream != null && + localScreenSharingStream!.stream != null && + localScreenSharingStream!.stream! + .getTracks() + .map((e) => e.id) + .contains(transceiver.sender.track?.id))); + + // in an ideal case the following should happen + // - audio track gets replaced + // - new video track gets added + for (final newTrack in streamTracks) { + final transceiver = transceivers.singleWhereOrNull( + (transceiver) => transceiver.sender.track!.kind == newTrack.kind); + if (transceiver != null) { + Logs().d( + '[VOIP] replacing ${transceiver.sender.track} in transceiver'); + final oldSender = transceiver.sender; + await oldSender.replaceTrack(newTrack); + await transceiver.setDirection( + await transceiver.getDirection() == + TransceiverDirection.Inactive // upgrade, send now + ? TransceiverDirection.SendOnly + : TransceiverDirection.SendRecv, + ); + } else { + // adding transceiver + Logs().d('[VOIP] adding track $newTrack to pc'); + await pc!.addTrack(newTrack, localUserMediaStream!.stream!); + } + } + // for renderer to be able to show new video track + localUserMediaStream?.renderer.srcObject = stream; + } + } } bool get isLocalVideoMuted => localUserMediaStream?.isVideoMuted() ?? false; - void setMicrophoneMuted(bool muted) { + Future setMicrophoneMuted(bool muted) async { localUserMediaStream?.setAudioMuted(muted); - _updateMuteStatus(); + await _updateMuteStatus(); } bool get isMicrophoneMuted => localUserMediaStream?.isAudioMuted() ?? false; @@ -1375,9 +1472,12 @@ class CallSession { if (event.streams.isNotEmpty) { final stream = event.streams[0]; _addRemoteStream(stream); - stream.getVideoTracks().forEach((track) { + stream.getTracks().forEach((track) { track.onEnded = () { - _removeStream(stream); + if (stream.getTracks().isEmpty) { + Logs().d('[VOIP] detected a empty stream, removing it'); + _removeStream(stream); + } }; }); } diff --git a/lib/src/voip/group_call.dart b/lib/src/voip/group_call.dart index 3985a8b9..f6b44a78 100644 --- a/lib/src/voip/group_call.dart +++ b/lib/src/voip/group_call.dart @@ -582,8 +582,9 @@ class GroupCall { final stream = await _getDisplayMedia(); stream.getTracks().forEach((track) { track.onEnded = () { + // screen sharing should only have 1 video track anyway, so this only + // fires once setScreensharingEnabled(false, ''); - track.onEnded = null; }; }); Logs().v( @@ -1159,15 +1160,12 @@ class GroupCall { void onActiveSpeakerLoop() async { String? nextActiveSpeaker; // idc about screen sharing atm. - for (final callFeed in userMediaStreams) { - if (callFeed.userId == client.userID && callFeed.pc == null) { - activeSpeakerLoopTimeout?.cancel(); - activeSpeakerLoopTimeout = - Timer(activeSpeakerInterval, onActiveSpeakerLoop); + for (final stream in userMediaStreams) { + if (stream.userId == client.userID && stream.pc == null) { continue; } - final List statsReport = await callFeed.pc!.getStats(); + final List statsReport = await stream.pc!.getStats(); statsReport .removeWhere((element) => !element.values.containsKey('audioLevel')); @@ -1178,7 +1176,7 @@ class GroupCall { element.values['kind'] == 'audio') ?.values['audioLevel']; if (otherPartyAudioLevel != null) { - audioLevelsMap[callFeed.userId] = otherPartyAudioLevel; + audioLevelsMap[stream.userId] = otherPartyAudioLevel; } // https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-source