From 00154f3c7814524de9bde370d267b73c91166be2 Mon Sep 17 00:00:00 2001 From: td Date: Mon, 30 Jan 2023 15:46:21 +0530 Subject: [PATCH] feat: active speaker in group calls --- lib/src/voip/call.dart | 46 ++++++++------- lib/src/voip/group_call.dart | 108 +++++++++++++++++++---------------- lib/src/voip/voip.dart | 4 +- 3 files changed, 85 insertions(+), 73 deletions(-) diff --git a/lib/src/voip/call.dart b/lib/src/voip/call.dart index b067a394..37e7e56b 100644 --- a/lib/src/voip/call.dart +++ b/lib/src/voip/call.dart @@ -58,6 +58,7 @@ class WrappedMediaStream { VideoRenderer renderer; final bool isWeb; final bool isGroupCall; + final RTCPeerConnection? pc; /// for debug String get title => '$displayName:$purpose:a[$audioMuted]:v[$videoMuted]'; @@ -70,6 +71,7 @@ class WrappedMediaStream { WrappedMediaStream( {this.stream, + this.pc, required this.renderer, required this.room, required this.userId, @@ -778,16 +780,18 @@ class CallSession { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - userId: client.userID!, - room: opts.room, - stream: stream, - purpose: purpose, - client: client, - audioMuted: stream.getAudioTracks().isEmpty, - videoMuted: stream.getVideoTracks().isEmpty, - isWeb: voip.delegate.isWeb, - isGroupCall: groupCallId != null); + renderer: voip.delegate.createRenderer(), + userId: client.userID!, + room: opts.room, + stream: stream, + purpose: purpose, + client: client, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isWeb: voip.delegate.isWeb, + isGroupCall: groupCallId != null, + pc: pc, + ); await newStream.initialize(); streams.add(newStream); onStreamAdd.add(newStream); @@ -839,16 +843,18 @@ class CallSession { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - userId: remoteUser!.id, - room: opts.room, - stream: stream, - purpose: purpose, - client: client, - audioMuted: audioMuted, - videoMuted: videoMuted, - isWeb: voip.delegate.isWeb, - isGroupCall: groupCallId != null); + renderer: voip.delegate.createRenderer(), + userId: remoteUser!.id, + room: opts.room, + stream: stream, + purpose: purpose, + client: client, + audioMuted: audioMuted, + videoMuted: videoMuted, + isWeb: voip.delegate.isWeb, + isGroupCall: groupCallId != null, + pc: pc, + ); await newStream.initialize(); streams.add(newStream); onStreamAdd.add(newStream); diff --git a/lib/src/voip/group_call.dart b/lib/src/voip/group_call.dart index da4da9a1..cfada213 100644 --- a/lib/src/voip/group_call.dart +++ b/lib/src/voip/group_call.dart @@ -176,10 +176,8 @@ class GroupCall { static const updateExpireTsTimerDuration = Duration(seconds: 15); static const expireTsBumpDuration = Duration(seconds: 45); + static const activeSpeakerInterval = Duration(seconds: 5); - var activeSpeakerInterval = 1000; - var retryCallInterval = 5000; - var participantTimeout = 1000 * 15; final Client client; final VoIP voip; final Room room; @@ -189,7 +187,7 @@ class GroupCall { final RTCDataChannelInit? dataChannelOptions; String state = GroupCallState.LocalCallFeedUninitialized; StreamSubscription? _callSubscription; - + final Map audioLevelsMap = {}; String? activeSpeaker; // userId WrappedMediaStream? localUserMediaStream; WrappedMediaStream? localScreenshareStream; @@ -373,18 +371,18 @@ class GroupCall { } final userId = client.userID; - final newStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - stream: stream, - userId: userId!, - room: room, - client: client, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().isEmpty, - videoMuted: stream.getVideoTracks().isEmpty, - isWeb: voip.delegate.isWeb, - isGroupCall: true); + renderer: voip.delegate.createRenderer(), + stream: stream, + userId: userId!, + room: room, + client: client, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isWeb: voip.delegate.isWeb, + isGroupCall: true, + ); localUserMediaStream = newStream; await localUserMediaStream!.initialize(); @@ -597,16 +595,17 @@ class GroupCall { 'Screensharing permissions granted. Setting screensharing enabled on all calls'); localDesktopCapturerSourceId = desktopCapturerSourceId; localScreenshareStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - stream: stream, - userId: client.userID!, - room: room, - client: client, - purpose: SDPStreamMetadataPurpose.Screenshare, - audioMuted: stream.getAudioTracks().isEmpty, - videoMuted: stream.getVideoTracks().isEmpty, - isWeb: voip.delegate.isWeb, - isGroupCall: true); + renderer: voip.delegate.createRenderer(), + stream: stream, + userId: client.userID!, + room: room, + client: client, + purpose: SDPStreamMetadataPurpose.Screenshare, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isWeb: voip.delegate.isWeb, + isGroupCall: true, + ); addScreenshareStream(localScreenshareStream!); await localScreenshareStream!.initialize(); @@ -1123,7 +1122,7 @@ class GroupCall { } userMediaStreams.removeWhere((element) => element.userId == stream.userId); - + audioLevelsMap.remove(stream.userId); onStreamRemoved.add(stream); if (stream.isLocal()) { @@ -1139,41 +1138,50 @@ class GroupCall { } } - void onActiveSpeakerLoop() { - /* TODO(duan): - var topAvg = 0.0; + void onActiveSpeakerLoop() async { String? nextActiveSpeaker; - - userMediaFeeds.forEach((callFeed) { - if (callFeed.userId == client.userID && userMediaFeeds.length > 1) { - return; + // idc about screen sharing atm. + for (final callFeed in userMediaStreams) { + if (callFeed.userId == client.userID && callFeed.pc == null) { + activeSpeakerLoopTimeout?.cancel(); + activeSpeakerLoopTimeout = + Timer(activeSpeakerInterval, onActiveSpeakerLoop); + continue; } - - var total = 0; - for (var i = 0; i < callFeed.speakingVolumeSamples.length; i++) { - final volume = callFeed.speakingVolumeSamples[i]; - total += max(volume, SPEAKING_THRESHOLD); - } + final List statsReport = await callFeed.pc!.getStats(); + statsReport + .removeWhere((element) => !element.values.containsKey('audioLevel')); - final avg = total / callFeed.speakingVolumeSamples.length; + // https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-source + // firefox does not seem to have this though. Works on chrome and android + audioLevelsMap[client.userID!] = statsReport + .lastWhere((element) => + element.type == 'media-source' && + element.values['kind'] == 'audio') + .values['audioLevel']; + // works everywhere? + audioLevelsMap[callFeed.userId] = statsReport + .lastWhere((element) => element.type == 'inbound-rtp') + .values['audioLevel']; + } - if (topAvg != 0 || avg > topAvg) { - topAvg = avg; - nextActiveSpeaker = callFeed.userId; - } + double maxAudioLevel = double.negativeInfinity; + // TODO: we probably want a threshold here? + audioLevelsMap.forEach((key, value) { + if (value > maxAudioLevel) { + nextActiveSpeaker = key; + maxAudioLevel = value; + } }); - if (nextActiveSpeaker != null && - activeSpeaker != nextActiveSpeaker && - topAvg > SPEAKING_THRESHOLD) { + if (nextActiveSpeaker != null && activeSpeaker != nextActiveSpeaker) { activeSpeaker = nextActiveSpeaker; onGroupCallEvent.add(GroupCallEvent.ActiveSpeakerChanged); } - + activeSpeakerLoopTimeout?.cancel(); activeSpeakerLoopTimeout = - Timer(Duration(seconds: activeSpeakerInterval), onActiveSpeakerLoop); - */ + Timer(activeSpeakerInterval, onActiveSpeakerLoop); } WrappedMediaStream? getScreenshareStreamByUserId(String userId) { diff --git a/lib/src/voip/voip.dart b/lib/src/voip/voip.dart index 72620a12..8b44f58d 100644 --- a/lib/src/voip/voip.dart +++ b/lib/src/voip/voip.dart @@ -856,8 +856,7 @@ class VoIP { final Map participants = {}; final callMemberEvents = room.states.tryGetMap( EventTypes.GroupCallMemberPrefix); - Logs().e( - 'callmemeberEvents length ${callMemberEvents?.length}'); + if (callMemberEvents != null) { callMemberEvents.forEach((userId, memberEvent) async { final callMemberEvent = groupCallEvent.room.getState( @@ -872,7 +871,6 @@ class VoIP { }); } - Logs().e(participants.toString()); if (!participants.values.any((expire_ts) => expire_ts > DateTime.now().millisecondsSinceEpoch)) { Logs().i(