diff --git a/lib/matrix.dart b/lib/matrix.dart index 98f6b3c9..6ddd30fb 100644 --- a/lib/matrix.dart +++ b/lib/matrix.dart @@ -29,8 +29,11 @@ export 'src/database/hive_collections_database.dart'; export 'src/event.dart'; export 'src/presence.dart'; export 'src/event_status.dart'; -export 'src/voip.dart'; -export 'src/voip_content.dart'; +export 'src/voip/call.dart'; +export 'src/voip/group_call.dart'; +export 'src/voip/voip.dart'; +export 'src/voip/voip_content.dart'; +export 'src/voip/utils.dart'; export 'src/room.dart'; export 'src/timeline.dart'; export 'src/user.dart'; @@ -50,7 +53,6 @@ export 'src/utils/sync_update_extension.dart'; export 'src/utils/to_device_event.dart'; export 'src/utils/uia_request.dart'; export 'src/utils/uri_extension.dart'; -export 'src/voip_content.dart'; export 'msc_extensions/extension_recent_emoji/recent_emoji.dart'; export 'msc_extensions/msc_1236_widgets/msc_1236_widgets.dart'; diff --git a/lib/src/client.dart b/lib/src/client.dart index 3a253bbd..4aafc6cb 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -27,6 +27,7 @@ import 'package:matrix/src/utils/sync_update_item_count.dart'; import 'package:mime/mime.dart'; import 'package:olm/olm.dart' as olm; import 'package:collection/collection.dart' show IterableExtension; +import 'package:random_string/random_string.dart'; import '../encryption.dart'; import '../matrix.dart'; @@ -221,6 +222,11 @@ class Client extends MatrixApi { String? get deviceName => _deviceName; String? _deviceName; + // for group calls + // A unique identifier used for resolving duplicate group call sessions from a given device. When the session_id field changes from an incoming m.call.member event, any existing calls from this device in this call should be terminated. The id is generated once per client load. + String? get groupCallSessionId => _groupCallSessionId; + String? _groupCallSessionId; + /// Returns the current login state. LoginState get loginState => __loginState; LoginState __loginState; @@ -613,6 +619,7 @@ class Client extends MatrixApi { List? initialState, Visibility? visibility, bool waitForSync = true, + bool groupCall = false, }) async { enableEncryption ??= encryptionEnabled && preset != CreateRoomPreset.publicChat; @@ -628,12 +635,18 @@ class Client extends MatrixApi { } } final roomId = await createRoom( - invite: invite, - preset: preset, - name: groupName, - initialState: initialState, - visibility: visibility, - ); + invite: invite, + preset: preset, + name: groupName, + initialState: initialState, + visibility: visibility, + powerLevelContentOverride: groupCall + ? { + 'events': { + 'org.matrix.msc3401.call.member': 0, + }, + } + : null); if (waitForSync) { if (getRoomById(roomId) == null) { @@ -1026,6 +1039,11 @@ class Client extends MatrixApi { final StreamController onUiaRequest = StreamController.broadcast(); + final StreamController onGroupCallRequest = + StreamController.broadcast(); + + final StreamController onGroupMember = StreamController.broadcast(); + /// How long should the app wait until it retrys the synchronisation after /// an error? int syncErrorTimeoutSec = 3; @@ -1258,6 +1276,8 @@ class Client extends MatrixApi { ); } + _groupCallSessionId = randomAlpha(12); + String? olmAccount; String? accessToken; String? _userID; @@ -1844,6 +1864,10 @@ class Client extends MatrixApi { EventTypes.CallSDPStreamMetadataChangedPrefix) { onSDPStreamMetadataChangedReceived .add(Event.fromJson(rawUnencryptedEvent, room)); + // TODO(duan): Only used (org.matrix.msc3401.call) during the current test, + // need to add GroupCallPrefix in matrix_api_lite + } else if (rawUnencryptedEvent['type'] == EventTypes.GroupCallPrefix) { + onGroupCallRequest.add(Event.fromJson(rawUnencryptedEvent, room)); } } } diff --git a/lib/src/voip/README.md b/lib/src/voip/README.md new file mode 100644 index 00000000..505e56c7 --- /dev/null +++ b/lib/src/voip/README.md @@ -0,0 +1,175 @@ +# VOIP for Matrix SDK + +1:1 and group calls + +## Overview + +`VoIP` is a module that provides a simple API for making 1:1 and group calls. + +`CallSession` objects are created by calling `inviteToCall` and `onCallInvite`. + +`GroupCall` objects are created by calling `createGroupCall`. + +## 1:1 calls + +### 1. Basic call flow + +This flow explains the code flow for a 1v1 call. +This code flow is still used in group call, the only difference is that group call uses `toDevice` message to send `m.call.*` events + +![1v1 call](images/famedly-1v1-call.drawio.png) + +### 2.Implement the event handlers + +The code here is to adapt to the difference between `flutter app` and `dart web app` and prevent importing `flutter` dependencies in `dart` app. + +We need to import `dart_webrtc` or `flutter_webrtc`, and map the platform-specific API `(mediaDevices, createPeerConnection, createRenderer)` +implementations to the corresponding packages. + +In addition, we can respond to the call to start and end in this delegate, start or turn off the incoming call ringing + +``` dart +// for dart app +import 'package:dart_webrtc/dart_webrtc.dart' as webrtc_impl; +// for flutter app +// import 'package:flutter_webrtc/flutter_webrtc.dart' as webrtc_impl; + +class MyVoipApp implements WebRTCDelegate { + @override + MediaDevices get mediaDevices => webrtc_impl.navigator.mediaDevices; + @override + Future createPeerConnection( + Map configuration, + [Map constraints = const {}]) => + webrtc_impl.createPeerConnection(configuration, constraints); + @override + VideoRenderer createRenderer() => RTCVideoRenderer(); + + @override + void playRingtone(){ + // play ringtone + } + void stopRingtone() { + // stop ringtone + } + + void handleNewCall(CallSession session) { + // handle new call incoming or outgoing + switch(session.direction) { + case CallDirection.kIncoming: + // show incoming call window + break; + case CallDirection.kOutgoing: + // show outgoing call window + break; + } + } + + void handleCallEnded(CallSession session) { + // handle call ended by local or remote + } +} +``` + +### 3.Start a outgoing call + +When the delegate is set we can initiate a new outgoing call. +We need to use the matrix roomId to initiate the call, the initial call can be +`CallType.kVoice` or `CallType.kVideo`. + +After the call is sent, you can use `onCallStateChanged` to listen the call state events. These events are used to change the display of the call UI state, for example, change the control buttons, display `Hangup (cancel)` button before connecting, and display `mute mic, mute cam, hold/unhold, hangup` buttons after connected. + +```dart +final voip = VoIP(client, MyVoipApp()); + +/// Create a new call +final newCall = await voip.inviteToCall(roomId, CallType.kVideo); + +newCall.onCallStateChanged.stream.listen((state) { + /// handle call state change event， + /// You can change UI state here, such as Ringing, + /// Connecting, Connected, Disconnected, etc. +}); + +/// Then you can pop up the incoming call window at MyVoipApp.handleNewCall. +class MyVoipApp implements WebRTCDelegate { +... + void handleNewCall(CallSession session) { + switch(session.direction) { + case CallDirection.kOutgoing: + // show outgoing call window + break; + } + } +... + +/// end the call by local +newCall.hangup(); +``` + +### 4.Answer a incoming call + +When a new incoming call comes in, handleNewCall will be called, and the answering interface can pop up at this time, and use `onCallStateChanged` to listen to the call state. + +The incoming call window need display `answer` and `reject` buttons, by calling `newCall.answer();` or `newCall.reject();` to decide whether to connect the call. + +```dart +... + void handleNewCall(CallSession newCall) { + switch(newCall.direction) { + case CallDirection.kIncoming: + /// show incoming call window + newCall.onCallStateChanged.stream.listen((state) { + /// handle call state change event + }); + break; + } + } +... + +/// Answer the call +newCall.answer(); +// or reject the call +newCall.reject(); +``` + +### 5.Render media stream + +The basic process of rendering a video stream is as follow code. + +```dart +class RemoteVideoView extends Widget { +VideoElement get videoElement => renderer.element; + +RTCVideoRenderer get renderer => remoteStream.renderer as RTCVideoRenderer; + +final WrappedMediaStream remoteStream; + +RemoteVideoView(this.remoteStream){ + renderer.srcObject = remoteStream.mediaStream; +} +... + @override + Element build() { + return divElement( + children: [ + ... + videoElement, + ... + ]); +} +... + +} +``` + +Usually there are four media streams in a 1v1 call, which are + +* `localUserMediaStream` +* `localScreenSharingStream` +* `remoteUserMediaStream` +* `remoteScreenSharingStream` + +They can be get by the methods of `CallSession`. the `newCall.onCallStreamsChanged` event is fired when these streams are added or removed. +When the media stream changes, we can change the UI display according to the priority. +`remoteScreenSharingStream` always needs to be displayed first, followed by `remoteUserMediaStream` \ No newline at end of file diff --git a/lib/src/voip.dart b/lib/src/voip/call.dart similarity index 74% rename from lib/src/voip.dart rename to lib/src/voip/call.dart index 97340cbd..ab9ad840 100644 --- a/lib/src/voip.dart +++ b/lib/src/voip/call.dart @@ -1,31 +1,32 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2021 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + import 'dart:async'; import 'dart:core'; import 'package:webrtc_interface/webrtc_interface.dart'; -import 'package:sdp_transform/sdp_transform.dart' as sdp_transform; -import '../matrix.dart'; +import '../../matrix.dart'; /// https://github.com/matrix-org/matrix-doc/pull/2746 /// version 1 const String voipProtoVersion = '1'; -/// Delegate WebRTC basic functionality. -abstract class WebRTCDelegate { - MediaDevices get mediaDevices; - Future createPeerConnection( - Map configuration, - [Map constraints = const {}]); - VideoRenderer createRenderer(); - void playRingtone(); - void stopRingtone(); - void handleNewCall(CallSession session); - void handleCallEnded(CallSession session); - - bool get isBackgroud; - bool get isWeb; -} - /// The default life time for call events, in millisecond. const lifetimeMs = 10 * 1000; @@ -45,11 +46,15 @@ class WrappedMediaStream { final Client client; VideoRenderer renderer; final bool isWeb; + final bool isGroupCall; /// for debug String get title => '$displayName:$purpose:a[$audioMuted]:v[$videoMuted]'; bool stopped = false; - void Function(bool audioMuted, bool videoMuted)? onMuteStateChanged; + + final StreamController onMuteStateChanged = + StreamController.broadcast(); + void Function(MediaStream stream)? onNewStream; WrappedMediaStream( @@ -61,7 +66,8 @@ class WrappedMediaStream { required this.client, required this.audioMuted, required this.videoMuted, - required this.isWeb}); + required this.isWeb, + required this.isGroupCall}); /// Initialize the video renderer Future initialize() async { @@ -75,7 +81,7 @@ class WrappedMediaStream { Future dispose() async { renderer.srcObject = null; - if (isLocal() && stream != null) { + if (isLocal() && !isGroupCall && stream != null) { if (isWeb) { stream!.getTracks().forEach((element) { element.stop(); @@ -86,6 +92,8 @@ class WrappedMediaStream { } } + Uri? get avatarUrl => getUser().avatarUrl; + String get avatarName => getUser().calcDisplayname(mxidLocalPartFallback: false); @@ -117,16 +125,12 @@ class WrappedMediaStream { void setAudioMuted(bool muted) { audioMuted = muted; - if (onMuteStateChanged != null) { - onMuteStateChanged?.call(audioMuted, videoMuted); - } + onMuteStateChanged.add(this); } void setVideoMuted(bool muted) { videoMuted = muted; - if (onMuteStateChanged != null) { - onMuteStateChanged?.call(audioMuted, videoMuted); - } + onMuteStateChanged.add(this); } } @@ -264,6 +268,7 @@ enum CallParty { kLocal, kRemote } /// Initialization parameters of the call session. class CallOptions { late String callId; + String? groupCallId; late CallType type; late CallDirection dir; late String localPartyId; @@ -279,6 +284,7 @@ class CallSession { CallType get type => opts.type; Room get room => opts.room; VoIP get voip => opts.voip; + String? get groupCallId => opts.groupCallId; String get callId => opts.callId; String get localPartyId => opts.localPartyId; String? get displayName => room.displayname; @@ -300,12 +306,40 @@ class CallSession { bool makingOffer = false; bool ignoreOffer = false; String facingMode = 'user'; + bool get answeredByUs => _answeredByUs; Client get client => opts.room.client; String? remotePartyId; - late User remoteUser; + String? opponentDeviceId; + String? opponentSessionId; + String? invitee; + User? remoteUser; late CallParty hangupParty; - late String hangupReason; + String? hangupReason; late CallError lastError; + CallSession? successor; + bool waitForLocalAVStream = false; + int toDeviceSeq = 0; + + final StreamController onCallStreamsChanged = + StreamController.broadcast(); + + final StreamController onCallReplaced = + StreamController.broadcast(); + + final StreamController onCallHangup = + StreamController.broadcast(); + + final StreamController onCallStateChanged = + StreamController.broadcast(); + + final StreamController onCallEventChanged = + StreamController.broadcast(); + + final StreamController onStreamAdd = + StreamController.broadcast(); + + final StreamController onStreamRemoved = + StreamController.broadcast(); SDPStreamMetadata? remoteSDPStreamMetadata; List usermediaSenders = []; @@ -352,12 +386,6 @@ class CallSession { return null; } - final _callStateController = - StreamController.broadcast(sync: true); - Stream get onCallStateChanged => _callStateController.stream; - final _callEventController = - StreamController.broadcast(sync: true); - Stream get onCallEventChanged => _callEventController.stream; Timer? inviteTimer; Timer? ringingTimer; @@ -366,7 +394,7 @@ class CallSession { setCallState(CallState.kCreateOffer); final stream = await _getUserMedia(type); if (stream != null) { - _addLocalStream(stream, SDPStreamMetadataPurpose.Usermedia); + addLocalStream(stream, SDPStreamMetadataPurpose.Usermedia); } } @@ -376,7 +404,7 @@ class CallSession { final stream = await _getUserMedia(type); if (stream != null) { - _addLocalStream(stream, SDPStreamMetadataPurpose.Usermedia); + addLocalStream(stream, SDPStreamMetadataPurpose.Usermedia); } if (metadata != null) { @@ -399,6 +427,96 @@ class CallSession { }); } + void answerWithStreams(List callFeeds) { + if (inviteOrAnswerSent) return; + Logs().d('nswering call $callId'); + gotCallFeedsForAnswer(callFeeds); + } + + void replacedBy(CallSession newCall) { + if (state == CallState.kWaitLocalMedia) { + Logs().v('Telling new call to wait for local media'); + newCall.waitForLocalAVStream = true; + } else if (state == CallState.kCreateOffer || + state == CallState.kInviteSent) { + Logs().v('Handing local stream to new call'); + newCall.gotCallFeedsForAnswer(getLocalStreams); + } + successor = newCall; + onCallReplaced.add(newCall); + hangup(CallErrorCode.Replaced, true); + } + + Future sendAnswer(RTCSessionDescription answer) async { + final callCapabilities = CallCapabilities() + ..dtmf = false + ..transferee = false; + + final metadata = SDPStreamMetadata({ + localUserMediaStream!.stream!.id: SDPStreamPurpose( + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: localUserMediaStream!.stream!.getAudioTracks().isEmpty, + video_muted: localUserMediaStream!.stream!.getVideoTracks().isEmpty) + }); + + final res = await sendAnswerCall(room, callId, answer.sdp!, localPartyId, + type: answer.type!, capabilities: callCapabilities, metadata: metadata); + Logs().v('[VOIP] answer res => $res'); + } + + Future gotCallFeedsForAnswer(List callFeeds) async { + if (state == CallState.kEnded) return; + + waitForLocalAVStream = false; + + callFeeds.forEach((element) { + // Safari can't send a MediaStream to multiple sources, so clone it + addLocalStream(element.stream!.clone(), element.purpose); + }); + + answer(); + } + + Future placeCallWithStreams(List callFeeds, + [bool requestScreenshareFeed = false]) async { + opts.dir = CallDirection.kOutgoing; + + voip.calls[callId] = this; + + // create the peer connection now so it can be gathering candidates while we get user + // media (assuming a candidate pool size is configured) + await _preparePeerConnection(); + gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); + } + + void gotCallFeedsForInvite(List callFeeds, + [bool requestScreenshareFeed = false]) { + if (successor != null) { + successor!.gotCallFeedsForAnswer(callFeeds); + return; + } + if (state == CallState.kEnded) { + cleanUp(); + return; + } + + callFeeds.forEach((element) { + addLocalStream(element.stream!.clone(), element.purpose); + }); + + if (requestScreenshareFeed) { + pc!.addTransceiver( + kind: RTCRtpMediaType.RTCRtpMediaTypeVideo, + init: + RTCRtpTransceiverInit(direction: TransceiverDirection.RecvOnly)); + } + + setCallState(CallState.kCreateOffer); + + Logs().d('gotUserMediaForInvite'); + // Now we wait for the negotiationneeded event + } + void initWithHangup() { setCallState(CallState.kEnded); } @@ -414,6 +532,10 @@ class CallSession { await pc!.setRemoteDescription(answer); remoteCandidates.forEach((candidate) => pc!.addCandidate(candidate)); } + + /// Send select_answer event. + await sendSelectCallAnswer( + opts.room, callId, lifetimeMs, localPartyId, remotePartyId!); } void onNegotiateReceived( @@ -442,8 +564,8 @@ class CallSession { await pc!.setRemoteDescription(description); if (description.type == 'offer') { final answer = await pc!.createAnswer({}); - await voip.sendCallNegotiate( - opts.room, callId, lifetimeMs, localPartyId, answer.sdp!, + await sendCallNegotiate( + room, callId, lifetimeMs, localPartyId, answer.sdp!, type: answer.type!); await pc!.setLocalDescription(answer); } @@ -547,7 +669,7 @@ class CallSession { setScreensharingEnabled(false); }; }); - _addLocalStream(stream, SDPStreamMetadataPurpose.Screenshare); + addLocalStream(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { fireCallEvent(CallEvent.kError); @@ -569,7 +691,7 @@ class CallSession { } } - void _addLocalStream(MediaStream stream, String purpose, + void addLocalStream(MediaStream stream, String purpose, {bool addToPeerConnection = true}) async { final existingStream = getLocalStreams.where((element) => element.purpose == purpose); @@ -577,19 +699,19 @@ class CallSession { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - userId: client.userID!, - room: opts.room, - stream: stream, - purpose: purpose, - client: client, - audioMuted: stream.getAudioTracks().isEmpty, - videoMuted: stream.getVideoTracks().isEmpty, - isWeb: voip.delegate.isWeb, - ); + renderer: voip.delegate.createRenderer(), + userId: client.userID!, + room: opts.room, + stream: stream, + purpose: purpose, + client: client, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isWeb: voip.delegate.isWeb, + isGroupCall: groupCallId != null); await newStream.initialize(); streams.add(newStream); - fireCallEvent(CallEvent.kFeedsChanged); + onStreamAdd.add(newStream); } if (addToPeerConnection) { @@ -604,7 +726,6 @@ class CallSession { usermediaSenders.add(await pc!.addTrack(track, stream)); }); } - fireCallEvent(CallEvent.kFeedsChanged); } if (purpose == SDPStreamMetadataPurpose.Usermedia) { @@ -614,10 +735,12 @@ class CallSession { audioTrack.enableSpeakerphone(speakerOn); } } + + fireCallEvent(CallEvent.kFeedsChanged); } void _addRemoteStream(MediaStream stream) async { - //const userId = this.getOpponentMember().userId; + //final userId = remoteUser.id; final metadata = remoteSDPStreamMetadata!.sdpStreamMetadatas[stream.id]; if (metadata == null) { Logs().i( @@ -637,26 +760,74 @@ class CallSession { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - userId: remoteUser.id, - room: opts.room, - stream: stream, - purpose: purpose, - client: client, - audioMuted: audioMuted, - videoMuted: videoMuted, - isWeb: voip.delegate.isWeb, - ); + renderer: voip.delegate.createRenderer(), + userId: remoteUser!.id, + room: opts.room, + stream: stream, + purpose: purpose, + client: client, + audioMuted: audioMuted, + videoMuted: videoMuted, + isWeb: voip.delegate.isWeb, + isGroupCall: groupCallId != null); await newStream.initialize(); streams.add(newStream); + onStreamAdd.add(newStream); } fireCallEvent(CallEvent.kFeedsChanged); Logs().i('Pushed remote stream (id="${stream.id}", purpose=$purpose)'); } + void deleteAllStreams() { + streams.forEach((stream) async { + if (stream.isLocal() || groupCallId == null) { + await stream.dispose(); + } + }); + streams.clear(); + fireCallEvent(CallEvent.kFeedsChanged); + } + + void deleteFeedByStream(MediaStream stream) { + final index = + streams.indexWhere((element) => element.stream!.id == stream.id); + if (index == -1) { + Logs().w('Didn\'t find the feed with stream id ${stream.id} to delete'); + return; + } + final wstream = streams.elementAt(index); + onStreamRemoved.add(wstream); + deleteStream(wstream); + } + + void deleteStream(WrappedMediaStream stream) { + stream.dispose(); + streams.removeAt(streams.indexOf(stream)); + fireCallEvent(CallEvent.kFeedsChanged); + } + + void removeLocalStream(WrappedMediaStream callFeed) { + final senderArray = callFeed.purpose == SDPStreamMetadataPurpose.Usermedia + ? usermediaSenders + : screensharingSenders; + + senderArray.forEach((element) async { + await pc!.removeTrack(element); + }); + + if (callFeed.purpose == SDPStreamMetadataPurpose.Screenshare) { + stopMediaStream(callFeed.stream); + } + + // Empty the array + senderArray.removeRange(0, senderArray.length); + onStreamRemoved.add(callFeed); + deleteStream(callFeed); + } + void setCallState(CallState newState) { state = newState; - _callStateController.add(newState); + onCallStateChanged.add(newState); fireCallEvent(CallEvent.kState); } @@ -732,8 +903,7 @@ class CallSession { video_muted: localUserMediaStream!.stream!.getVideoTracks().isEmpty) }); - final res = await voip.sendAnswerCall( - opts.room, callId, answer.sdp!, localPartyId, + final res = await sendAnswerCall(room, callId, answer.sdp!, localPartyId, type: answer.type!, capabilities: callCapabilities, metadata: metadata); @@ -756,7 +926,7 @@ class CallSession { } Logs().d('[VOIP] Rejecting call: $callId'); terminate(CallParty.kLocal, CallErrorCode.UserHangup, true); - voip.sendCallReject(opts.room, callId, lifetimeMs, localPartyId); + sendCallReject(room, callId, lifetimeMs, localPartyId); } void hangup([String? reason, bool suppressEvent = true]) async { @@ -767,8 +937,8 @@ class CallSession { CallParty.kLocal, reason ?? CallErrorCode.UserHangup, !suppressEvent); try { - final res = await voip.sendHangupCall( - opts.room, callId, localPartyId, 'userHangup'); + final res = + await sendHangupCall(room, callId, localPartyId, 'userHangup'); Logs().v('[VOIP] hangup res => $res'); } catch (e) { Logs().v('[VOIP] hangup error => ${e.toString()}'); @@ -786,7 +956,7 @@ class CallSession { Logs().e('Unable to find a track to send DTMF on'); } - void terminate(CallParty party, String hangupReason, bool shouldEmit) async { + void terminate(CallParty party, String reason, bool shouldEmit) async { if (state == CallState.kEnded) { return; } @@ -798,12 +968,15 @@ class CallSession { ringingTimer = null; hangupParty = party; - hangupReason = hangupReason; + hangupReason = reason; setCallState(CallState.kEnded); voip.currentCID = null; voip.calls.remove(callId); cleanUp(); + + onCallHangup.add(this); + voip.delegate.handleCallEnded(this); if (shouldEmit) { fireCallEvent(CallEvent.kHangup); @@ -848,8 +1021,8 @@ class CallSession { ..transferee = false; final metadata = _getLocalSDPStreamMetadata(); if (state == CallState.kCreateOffer) { - await voip.sendInviteToCall( - opts.room, callId, lifetimeMs, localPartyId, null, offer.sdp!, + await sendInviteToCall( + room, callId, lifetimeMs, localPartyId, null, offer.sdp!, capabilities: callCapabilities, metadata: metadata); inviteOrAnswerSent = true; setCallState(CallState.kInviteSent); @@ -862,8 +1035,8 @@ class CallSession { inviteTimer = null; }); } else { - await voip.sendCallNegotiate( - opts.room, callId, lifetimeMs, localPartyId, offer.sdp!, + await sendCallNegotiate( + room, callId, lifetimeMs, localPartyId, offer.sdp!, type: offer.type!, capabilities: callCapabilities, metadata: metadata); @@ -926,7 +1099,7 @@ class CallSession { } } - void onAnsweredElsewhere(String msg) { + void onAnsweredElsewhere() { Logs().d('Call ID $callId answered elsewhere'); terminate(CallParty.kRemote, CallErrorCode.AnsweredElsewhere, true); } @@ -955,8 +1128,8 @@ class CallSession { _setTracksEnabled(localUserMediaStream?.stream!.getVideoTracks() ?? [], !vidShouldBeMuted); - await opts.voip.sendSDPStreamMetadataChanged( - opts.room, callId, localPartyId, _getLocalSDPStreamMetadata()); + await sendSDPStreamMetadataChanged( + room, callId, localPartyId, _getLocalSDPStreamMetadata()); } void _setTracksEnabled(List tracks, bool enabled) { @@ -1034,11 +1207,20 @@ class CallSession { if (event.streams.isNotEmpty) { final stream = event.streams[0]; _addRemoteStream(stream); + stream.getVideoTracks().forEach((track) { + track.onEnded = () { + _removeStream(stream); + }; + }); } }; return pc; } + void createDataChannel(String label, RTCDataChannelInit dataChannelDict) { + pc?.createDataChannel(label, dataChannelDict); + } + void tryRemoveStopedStreams() { final removedStreams = {}; streams.forEach((stream) { @@ -1063,6 +1245,7 @@ class CallSession { } final wpstream = it.first; streams.removeWhere((element) => element.stream!.id == stream.id); + onStreamRemoved.add(wpstream); fireCallEvent(CallEvent.kFeedsChanged); await wpstream.dispose(); } @@ -1085,8 +1268,8 @@ class CallSession { localCandidates.forEach((element) { candidates.add(element.toMap()); }); - final res = await voip.sendCallCandidates( - opts.room, callId, localPartyId, candidates); + final res = + await sendCallCandidates(opts.room, callId, localPartyId, candidates); Logs().v('[VOIP] sendCallCandidates res => $res'); } catch (e) { Logs().v('[VOIP] sendCallCandidates e => ${e.toString()}'); @@ -1094,10 +1277,11 @@ class CallSession { } void fireCallEvent(CallEvent event) { - _callEventController.add(event); + onCallEventChanged.add(event); Logs().i('CallEvent: ${event.toString()}'); switch (event) { case CallEvent.kFeedsChanged: + onCallStreamsChanged.add(this); break; case CallEvent.kState: Logs().i('CallState: ${state.toString()}'); @@ -1155,365 +1339,6 @@ class CallSession { terminate(CallParty.kRemote, CallErrorCode.AnsweredElsewhere, true); } } -} - -class VoIP { - TurnServerCredentials? _turnServerCredentials; - Map calls = {}; - String? currentCID; - String? get localPartyId => client.deviceID; - final Client client; - final WebRTCDelegate delegate; - - VoIP(this.client, this.delegate) : super() { - client.onCallInvite.stream.listen(onCallInvite); - client.onCallAnswer.stream.listen(onCallAnswer); - client.onCallCandidates.stream.listen(onCallCandidates); - client.onCallHangup.stream.listen(onCallHangup); - client.onCallReject.stream.listen(onCallReject); - client.onCallNegotiate.stream.listen(onCallNegotiate); - client.onCallReplaces.stream.listen(onCallReplaces); - client.onCallSelectAnswer.stream.listen(onCallSelectAnswer); - client.onSDPStreamMetadataChangedReceived.stream - .listen(onSDPStreamMetadataChangedReceived); - client.onAssertedIdentityReceived.stream.listen(onAssertedIdentityReceived); - } - - Future onCallInvite(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - - Logs().v( - '[VOIP] onCallInvite ${event.senderId} => ${client.userID}, \ncontent => ${event.content.toString()}'); - - final String callId = event.content['call_id']; - final String partyId = event.content['party_id']; - final int lifetime = event.content['lifetime']; - - if (currentCID != null) { - // Only one session at a time. - Logs().v('[VOIP] onCallInvite: There is already a session.'); - await sendHangupCall(event.room, callId, localPartyId!, 'userBusy'); - return; - } - if (calls[callId] != null) { - // Session already exist. - Logs().v('[VOIP] onCallInvite: Session [$callId] already exist.'); - return; - } - - if (event.content['capabilities'] != null) { - final capabilities = - CallCapabilities.fromJson(event.content['capabilities']); - Logs().v( - '[VOIP] CallCapabilities: dtmf => ${capabilities.dtmf}, transferee => ${capabilities.transferee}'); - } - - var callType = CallType.kVoice; - SDPStreamMetadata? sdpStreamMetadata; - if (event.content[sdpStreamMetadataKey] != null) { - sdpStreamMetadata = - SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey]); - sdpStreamMetadata.sdpStreamMetadatas - .forEach((streamId, SDPStreamPurpose purpose) { - Logs().v( - '[VOIP] [$streamId] => purpose: ${purpose.purpose}, audioMuted: ${purpose.audio_muted}, videoMuted: ${purpose.video_muted}'); - - if (!purpose.video_muted) { - callType = CallType.kVideo; - } - }); - } else { - callType = getCallType(event.content['offer']['sdp']); - } - - final opts = CallOptions() - ..voip = this - ..callId = callId - ..dir = CallDirection.kIncoming - ..type = callType - ..room = event.room - ..localPartyId = localPartyId! - ..iceServers = await getIceSevers(); - - final newCall = createNewCall(opts); - newCall.remotePartyId = partyId; - newCall.remoteUser = (await event.fetchSenderUser()) ?? - User(event.senderId, room: event.room); - final offer = RTCSessionDescription( - event.content['offer']['sdp'], - event.content['offer']['type'], - ); - await newCall - .initWithInvite(callType, offer, sdpStreamMetadata, lifetime) - .then((_) { - // Popup CallingPage for incoming call. - if (!delegate.isBackgroud) { - delegate.handleNewCall(newCall); - } - }); - currentCID = callId; - - if (delegate.isBackgroud) { - /// Forced to enable signaling synchronization until the end of the call. - client.backgroundSync = true; - - ///TODO: notify the callkeep that the call is incoming. - } - // Play ringtone - delegate.playRingtone(); - } - - void onCallAnswer(Event event) async { - Logs().v('[VOIP] onCallAnswer => ${event.content.toString()}'); - final String callId = event.content['call_id']; - final String partyId = event.content['party_id']; - - final call = calls[callId]; - if (call != null) { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - if (!call._answeredByUs) { - delegate.stopRingtone(); - } - if (call.state == CallState.kRinging) { - call.onAnsweredElsewhere('Call ID ' + callId + ' answered elsewhere'); - } - return; - } - - call.remotePartyId = partyId; - call.remoteUser = await event.fetchSenderUser() ?? - User(event.senderId, room: event.room); - - final answer = RTCSessionDescription( - event.content['answer']['sdp'], event.content['answer']['type']); - - SDPStreamMetadata? metadata; - if (event.content[sdpStreamMetadataKey] != null) { - metadata = - SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey]); - } - call.onAnswerReceived(answer, metadata); - - /// Send select_answer event. - await sendSelectCallAnswer( - event.room, callId, lifetimeMs, localPartyId!, call.remotePartyId!); - } else { - Logs().v('[VOIP] onCallAnswer: Session [$callId] not found!'); - } - } - - void onCallCandidates(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - Logs().v('[VOIP] onCallCandidates => ${event.content.toString()}'); - final String callId = event.content['call_id']; - final call = calls[callId]; - if (call != null) { - call.onCandidatesReceived(event.content['candidates']); - } else { - Logs().v('[VOIP] onCallCandidates: Session [$callId] not found!'); - } - } - - void onCallHangup(Event event) async { - // stop play ringtone, if this is an incoming call - if (!delegate.isBackgroud) { - delegate.stopRingtone(); - } - Logs().v('[VOIP] onCallHangup => ${event.content.toString()}'); - final String callId = event.content['call_id']; - final call = calls[callId]; - if (call != null) { - // hangup in any case, either if the other party hung up or we did on another device - call.terminate(CallParty.kRemote, - event.content['reason'] ?? CallErrorCode.UserHangup, true); - } else { - Logs().v('[VOIP] onCallHangup: Session [$callId] not found!'); - } - currentCID = null; - } - - void onCallReject(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - final String callId = event.content['call_id']; - Logs().d('Reject received for call ID ' + callId); - - final call = calls[callId]; - if (call != null) { - call.onRejectReceived(event.content['reason']); - } else { - Logs().v('[VOIP] onCallHangup: Session [$callId] not found!'); - } - } - - void onCallReplaces(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - final String callId = event.content['call_id']; - Logs().d('onCallReplaces received for call ID ' + callId); - final call = calls[callId]; - if (call != null) { - //TODO: handle replaces - } - } - - void onCallSelectAnswer(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - final String callId = event.content['call_id']; - Logs().d('SelectAnswer received for call ID ' + callId); - final call = calls[callId]; - final String selectedPartyId = event.content['selected_party_id']; - - if (call != null) { - call.onSelectAnswerReceived(selectedPartyId); - } - } - - void onSDPStreamMetadataChangedReceived(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - final String callId = event.content['call_id']; - Logs().d('SDP Stream metadata received for call ID ' + callId); - final call = calls[callId]; - if (call != null) { - if (event.content[sdpStreamMetadataKey] == null) { - Logs().d('SDP Stream metadata is null'); - return; - } - call.onSDPStreamMetadataReceived( - SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey])); - } - } - - void onAssertedIdentityReceived(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - final String callId = event.content['call_id']; - Logs().d('Asserted identity received for call ID ' + callId); - final call = calls[callId]; - if (call != null) { - if (event.content['asserted_identity'] == null) { - Logs().d('asserted_identity is null '); - return; - } - call.onAssertedIdentityReceived( - AssertedIdentity.fromJson(event.content['asserted_identity'])); - } - } - - void onCallNegotiate(Event event) async { - if (event.senderId == client.userID) { - // Ignore messages to yourself. - return; - } - final String callId = event.content['call_id']; - Logs().d('Negotiate received for call ID ' + callId); - final call = calls[callId]; - if (call != null) { - final description = event.content['description']; - try { - SDPStreamMetadata? metadata; - if (event.content[sdpStreamMetadataKey] != null) { - metadata = - SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey]); - } - call.onNegotiateReceived(metadata, - RTCSessionDescription(description['sdp'], description['type'])); - } catch (err) { - Logs().e('Failed to complete negotiation ${err.toString()}'); - } - } - } - - CallType getCallType(String sdp) { - try { - final session = sdp_transform.parse(sdp); - if (session['media'].indexWhere((e) => e['type'] == 'video') != -1) { - return CallType.kVideo; - } - } catch (err) { - Logs().e('Failed to getCallType ${err.toString()}'); - } - - return CallType.kVoice; - } - - Future requestTurnServerCredentials() async { - return true; - } - - Future>> getIceSevers() async { - if (_turnServerCredentials == null) { - try { - _turnServerCredentials = await client.getTurnServer(); - } catch (e) { - Logs().v('[VOIP] getTurnServerCredentials error => ${e.toString()}'); - } - } - - if (_turnServerCredentials == null) { - return []; - } - - return [ - { - 'username': _turnServerCredentials!.username, - 'credential': _turnServerCredentials!.password, - 'urls': _turnServerCredentials!.uris[0] - } - ]; - } - - Future inviteToCall(String roomId, CallType type) async { - final room = client.getRoomById(roomId); - if (room == null) { - Logs().v('[VOIP] Invalid room id [$roomId].'); - return Null as CallSession; - } - final callId = 'cid${DateTime.now().millisecondsSinceEpoch}'; - final opts = CallOptions() - ..callId = callId - ..type = type - ..dir = CallDirection.kOutgoing - ..room = room - ..voip = this - ..localPartyId = localPartyId! - ..iceServers = await getIceSevers(); - - final newCall = createNewCall(opts); - currentCID = callId; - await newCall.initOutboundCall(type).then((_) { - if (!delegate.isBackgroud) { - delegate.handleNewCall(newCall); - } - }); - currentCID = callId; - return newCall; - } - - CallSession createNewCall(CallOptions opts) { - final call = CallSession(opts); - calls[opts.callId] = call; - return call; - } /// This is sent by the caller when they wish to establish a call. /// [callId] is a unique identifier for the call. @@ -1537,6 +1362,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, 'lifetime': lifetime, 'offer': {'sdp': sdp, 'type': type}, @@ -1569,6 +1395,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, 'lifetime': lifetime, 'selected_party_id': selected_party_id, @@ -1594,6 +1421,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, 'lifetime': lifetime, }; @@ -1622,6 +1450,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, 'lifetime': lifetime, 'description': {'sdp': sdp, 'type': type}, @@ -1668,6 +1497,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, 'candidates': candidates, }; @@ -1696,6 +1526,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, 'answer': {'sdp': sdp, 'type': type}, if (capabilities != null) 'capabilities': capabilities.toJson(), @@ -1721,6 +1552,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, if (hangupCause != null) 'reason': hangupCause, }; @@ -1753,6 +1585,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, sdpStreamMetadataKey: metadata.toJson(), }; @@ -1777,6 +1610,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, ...callReplaces.toJson(), }; @@ -1801,6 +1635,7 @@ class VoIP { final content = { 'call_id': callId, 'party_id': party_id, + if (groupCallId != null) 'conf_id': groupCallId, 'version': version, 'asserted_identity': assertedIdentity.toJson(), }; @@ -1819,20 +1654,51 @@ class VoIP { String? txid, }) async { txid ??= client.generateUniqueTransactionId(); - final mustEncrypt = room.encrypted && client.encryptionEnabled; + if (opponentDeviceId != null) { + final toDeviceSeq = this.toDeviceSeq++; - final sendMessageContent = mustEncrypt - ? await client.encryption! - .encryptGroupMessagePayload(room.id, content, type: type) - : content; - return await client.sendMessage( - room.id, - sendMessageContent.containsKey('ciphertext') - ? EventTypes.Encrypted - : type, - txid, - sendMessageContent, - ); + if (mustEncrypt) { + await client.sendToDeviceEncrypted( + [ + client.userDeviceKeys[invitee ?? remoteUser!.id]! + .deviceKeys[opponentDeviceId]! + ], + type, + { + ...content, + 'device_id': client.deviceID!, + 'seq': toDeviceSeq, + 'dest_session_id': opponentSessionId, + 'sender_session_id': client.groupCallSessionId, + }); + } else { + final data = >>{}; + data[invitee ?? remoteUser!.id] = { + opponentDeviceId!: { + ...content, + 'device_id': client.deviceID!, + 'seq': toDeviceSeq, + 'dest_session_id': opponentSessionId, + 'sender_session_id': client.groupCallSessionId, + } + }; + await client.sendToDevice(type, txid, data); + } + return ''; + } else { + final sendMessageContent = mustEncrypt + ? await client.encryption! + .encryptGroupMessagePayload(room.id, content, type: type) + : content; + return await client.sendMessage( + room.id, + sendMessageContent.containsKey('ciphertext') + ? EventTypes.Encrypted + : type, + txid, + sendMessageContent, + ); + } } } diff --git a/lib/src/voip/group_call.dart b/lib/src/voip/group_call.dart new file mode 100644 index 00000000..e8bfd5f6 --- /dev/null +++ b/lib/src/voip/group_call.dart @@ -0,0 +1,1242 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2021 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General License for more details. + * + * You should have received a copy of the GNU Affero General License + * along with this program. If not, see . + */ + +import 'dart:async'; +import 'dart:core'; + +import 'package:matrix/matrix.dart'; +import 'package:webrtc_interface/webrtc_interface.dart'; + +/// TODO(@duan): Need to add voice activity detection mechanism +/// const int SPEAKING_THRESHOLD = -60; // dB + +class GroupCallIntent { + static String Ring = 'm.ring'; + static String Prompt = 'm.prompt'; + static String Room = 'm.room'; +} + +class GroupCallType { + static String Video = 'm.video'; + static String Voice = 'm.voice'; +} + +class GroupCallTerminationReason { + static String CallEnded = 'call_ended'; +} + +class GroupCallEvent { + static String GroupCallStateChanged = 'group_call_state_changed'; + static String ActiveSpeakerChanged = 'active_speaker_changed'; + static String CallsChanged = 'calls_changed'; + static String UserMediaStreamsChanged = 'user_media_feeds_changed'; + static String ScreenshareStreamsChanged = 'screenshare_feeds_changed'; + static String LocalScreenshareStateChanged = + 'local_screenshare_state_changed'; + static String LocalMuteStateChanged = 'local_mute_state_changed'; + static String ParticipantsChanged = 'participants_changed'; + static String Error = 'error'; +} + +class GroupCallErrorCode { + static String NoUserMedia = 'no_user_media'; + static String UnknownDevice = 'unknown_device'; +} + +class GroupCallError extends Error { + final String code; + final String msg; + final dynamic err; + GroupCallError(this.code, this.msg, this.err); + + @override + String toString() { + return 'Group Call Error: [$code] $msg, err: ${err.toString()}'; + } +} + +abstract class ISendEventResponse { + String? event_id; +} + +class IGroupCallRoomMemberFeed { + String? purpose; + // TODO: Sources for adaptive bitrate + IGroupCallRoomMemberFeed.fromJson(Map json) { + purpose = json['purpose']; + } + Map toJson() { + final data = {}; + data['purpose'] = purpose; + return data; + } +} + +class IGroupCallRoomMemberDevice { + String? device_id; + String? session_id; + List feeds = []; + IGroupCallRoomMemberDevice.fromJson(Map json) { + device_id = json['device_id']; + session_id = json['session_id']; + if (json['feeds'] != null) { + feeds = (json['feeds'] as List) + .map((feed) => IGroupCallRoomMemberFeed.fromJson(feed)) + .toList(); + } + } + + Map toJson() { + final data = {}; + data['device_id'] = device_id; + data['session_id'] = session_id; + data['feeds'] = feeds.map((feed) => feed.toJson()).toList(); + return data; + } +} + +class IGroupCallRoomMemberCallState { + String? call_id; + List? foci; + List devices = []; + IGroupCallRoomMemberCallState.formJson(Map json) { + call_id = json['m.call_id']; + if (json['m.foci'] != null) { + foci = (json['m.foci'] as List).cast(); + } + if (json['m.devices'] != null) { + devices = (json['m.devices'] as List) + .map((device) => IGroupCallRoomMemberDevice.fromJson(device)) + .toList(); + } + } + Map toJson() { + final data = {}; + data['m.call_id'] = call_id; + if (foci != null) { + data['m.foci'] = foci; + } + if (devices.isNotEmpty) { + data['m.devices'] = devices.map((e) => e.toJson()).toList(); + } + return data; + } +} + +class IGroupCallRoomMemberState { + List calls = []; + IGroupCallRoomMemberState.fromJson(Map json) { + if (json['m.calls'] != null) { + (json['m.calls'] as List).forEach( + (call) => calls.add(IGroupCallRoomMemberCallState.formJson(call))); + } + } +} + +class GroupCallState { + static String LocalCallFeedUninitialized = 'local_call_feed_uninitialized'; + static String InitializingLocalCallFeed = 'initializing_local_call_feed'; + static String LocalCallFeedInitialized = 'local_call_feed_initialized'; + static String Entering = 'entering'; + static String Entered = 'entered'; + static String Ended = 'ended'; +} + +abstract class ICallHandlers { + Function(List feeds)? onCallFeedsChanged; + Function(CallState state, CallState oldState)? onCallStateChanged; + Function(CallSession call)? onCallHangup; + Function(CallSession newCall)? onCallReplaced; +} + +class GroupCall { + // Config + var activeSpeakerInterval = 1000; + var retryCallInterval = 5000; + var participantTimeout = 1000 * 15; + final Client client; + final VoIP voip; + final Room room; + final String intent; + final String type; + final bool dataChannelsEnabled; + final RTCDataChannelInit? dataChannelOptions; + String state = GroupCallState.LocalCallFeedUninitialized; + StreamSubscription? _callSubscription; + + String? activeSpeaker; // userId + WrappedMediaStream? localUserMediaStream; + WrappedMediaStream? localScreenshareStream; + String? localDesktopCapturerSourceId; + List calls = []; + List participants = []; + List userMediaStreams = []; + List screenshareStreams = []; + late String groupCallId; + + GroupCallError? lastError; + + Map callHandlers = {}; + + Timer? activeSpeakerLoopTimeout; + + Timer? retryCallLoopTimeout; + Map retryCallCounts = {}; + + final StreamController onGroupCallFeedsChanged = + StreamController.broadcast(); + + final StreamController onGroupCallState = + StreamController.broadcast(); + + final StreamController onGroupCallEvent = + StreamController.broadcast(); + + final StreamController onStreamAdd = + StreamController.broadcast(); + + final StreamController onStreamRemoved = + StreamController.broadcast(); + + GroupCall({ + String? groupCallId, + required this.client, + required this.voip, + required this.room, + required this.type, + required this.intent, + required this.dataChannelsEnabled, + required this.dataChannelOptions, + }) { + this.groupCallId = groupCallId ?? genCallID(); + } + + GroupCall create() { + voip.groupCalls[groupCallId] = this; + voip.groupCalls[room.id] = this; + + client.setRoomStateWithKey( + room.id, + EventTypes.GroupCallPrefix, + groupCallId, + { + 'm.intent': intent, + 'm.type': type, + // TODO: Specify datachannels + 'dataChannelsEnabled': dataChannelsEnabled, + 'dataChannelOptions': dataChannelOptions?.toMap() ?? {}, + 'groupCallId': groupCallId, + }, + ); + + return this; + } + + String get avatarName => + getUser().calcDisplayname(mxidLocalPartFallback: false); + + String? get displayName => getUser().displayName; + + User getUser() { + return room.unsafeGetUserFromMemoryOrFallback(client.userID!); + } + + Future> getStateEventsList(String type) async { + final roomStates = await client.getRoomState(room.id); + roomStates.sort((a, b) => a.originServerTs.compareTo(b.originServerTs)); + final events = []; + roomStates.forEach((evt) { + if (evt.type == type) { + events.add(evt); + } + }); + return events; + } + + Future getStateEvent(String type, [String? userId]) async { + final roomStates = await client.getRoomState(room.id); + roomStates.sort((a, b) => a.originServerTs.compareTo(b.originServerTs)); + MatrixEvent? event; + roomStates.forEach((value) { + if (value.type == type && (userId == null || value.senderId == userId)) { + event = value; + } + }); + return event; + } + + void setState(String newState) { + state = newState; + onGroupCallEvent.add(GroupCallEvent.GroupCallStateChanged); + } + + List getLocalStreams() { + final feeds = []; + + if (localUserMediaStream != null) { + feeds.add(localUserMediaStream!); + } + + if (localScreenshareStream != null) { + feeds.add(localScreenshareStream!); + } + + return feeds; + } + + bool hasLocalParticipant() { + final userId = client.userID; + return participants.indexWhere((member) => member.id == userId) != -1; + } + + Future _getUserMedia(CallType type) async { + final mediaConstraints = { + 'audio': true, + 'video': type == CallType.kVideo + ? { + 'mandatory': { + 'minWidth': '640', + 'minHeight': '480', + 'minFrameRate': '30', + }, + 'facingMode': 'user', + 'optional': [], + } + : false, + }; + try { + return await voip.delegate.mediaDevices.getUserMedia(mediaConstraints); + } catch (e) { + setState(GroupCallState.LocalCallFeedUninitialized); + } + return Null as MediaStream; + } + + Future _getDisplayMedia() async { + final mediaConstraints = { + 'audio': false, + 'video': true, + }; + try { + return await voip.delegate.mediaDevices.getDisplayMedia(mediaConstraints); + } catch (e) { + setState(GroupCallState.LocalCallFeedUninitialized); + } + return Null as MediaStream; + } + + /// Initializes the local user media stream. + /// The media stream must be prepared before the group call enters. + Future initLocalStream() async { + if (state != GroupCallState.LocalCallFeedUninitialized) { + throw Exception('Cannot initialize local call feed in the $state state.'); + } + + setState(GroupCallState.InitializingLocalCallFeed); + + MediaStream stream; + + try { + stream = await _getUserMedia( + type == GroupCallType.Video ? CallType.kVideo : CallType.kVoice); + } catch (error) { + setState(GroupCallState.LocalCallFeedUninitialized); + rethrow; + } + + final userId = client.userID; + + final newStream = WrappedMediaStream( + renderer: voip.delegate.createRenderer(), + stream: stream, + userId: userId!, + room: room, + client: client, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isWeb: voip.delegate.isWeb, + isGroupCall: true); + + localUserMediaStream = newStream; + await localUserMediaStream!.initialize(); + addUserMediaStream(newStream); + + setState(GroupCallState.LocalCallFeedInitialized); + + return newStream; + } + + void updateLocalUsermediaStream(WrappedMediaStream stream) { + if (localUserMediaStream != null) { + final oldStream = localUserMediaStream!.stream; + localUserMediaStream!.setNewStream(stream.stream!); + stopMediaStream(oldStream); + } + } + + /// enter the group call. + void enter() async { + if (!(state == GroupCallState.LocalCallFeedUninitialized || + state == GroupCallState.LocalCallFeedInitialized)) { + throw Exception('Cannot enter call in the $state state'); + } + + if (state == GroupCallState.LocalCallFeedUninitialized) { + await initLocalStream(); + } + + _addParticipant( + (await room.requestUser(client.userID!, ignoreErrors: true))!); + + await sendMemberStateEvent(); + + activeSpeaker = null; + + setState(GroupCallState.Entered); + + Logs().v('Entered group call $groupCallId'); + + _callSubscription = voip.onIncomingCall.stream.listen(onIncomingCall); + + for (final call in calls) { + onIncomingCall(call); + } + + // Set up participants for the members currently in the room. + // Other members will be picked up by the RoomState.members event. + + final memberStateEvents = + await getStateEventsList(EventTypes.GroupCallMemberPrefix); + + memberStateEvents.forEach((stateEvent) { + onMemberStateChanged(stateEvent); + }); + + retryCallLoopTimeout = Timer.periodic( + Duration(milliseconds: retryCallInterval), onRetryCallLoop); + onActiveSpeakerLoop(); + + voip.currentGroupCID = groupCallId; + + voip.delegate.handleNewGroupCall(this); + } + + void dispose() { + if (localUserMediaStream != null) { + removeUserMediaStream(localUserMediaStream!); + localUserMediaStream = null; + } + + if (localScreenshareStream != null) { + stopMediaStream(localScreenshareStream!.stream); + removeScreenshareStream(localScreenshareStream!); + localScreenshareStream = null; + localDesktopCapturerSourceId = null; + } + + if (state != GroupCallState.Entered) { + return; + } + + _removeParticipant(client.userID!); + + removeMemberStateEvent(); + + final callsCopy = calls.toList(); + callsCopy.forEach((element) { + removeCall(element, CallErrorCode.UserHangup); + }); + + activeSpeaker = null; + activeSpeakerLoopTimeout?.cancel(); + retryCallCounts.clear(); + retryCallLoopTimeout?.cancel(); + _callSubscription?.cancel(); + } + + void leave() { + dispose(); + setState(GroupCallState.LocalCallFeedUninitialized); + voip.currentGroupCID = null; + voip.delegate.handleGroupCallEnded(this); + } + + /// terminate group call. + void terminate({bool emitStateEvent = true}) async { + dispose(); + + participants = []; + //TODO(duan): remove this + /* client.removeListener( + 'RoomState.members', + onMemberStateChanged, + ); + */ + voip.groupCalls.remove(room.id); + + if (emitStateEvent) { + final existingStateEvent = await getStateEvent( + EventTypes.GroupCallPrefix, + groupCallId, + ); + + await client.setRoomStateWithKey( + room.id, EventTypes.GroupCallPrefix, groupCallId, { + ...existingStateEvent!.content, + 'm.terminated': GroupCallTerminationReason.CallEnded, + }); + } + voip.delegate.handleGroupCallEnded(this); + setState(GroupCallState.Ended); + } + + bool get isLocalVideoMuted { + if (localUserMediaStream != null) { + return localUserMediaStream!.isVideoMuted(); + } + + return true; + } + + bool get isMicrophoneMuted { + if (localUserMediaStream != null) { + return localUserMediaStream!.isAudioMuted(); + } + + return true; + } + + Future setMicrophoneMuted(bool muted) async { + if (!await hasAudioDevice()) { + return false; + } + + if (localUserMediaStream != null) { + localUserMediaStream!.setAudioMuted(muted); + setTracksEnabled(localUserMediaStream!.stream!.getAudioTracks(), !muted); + } + + calls.forEach((call) { + call.setMicrophoneMuted(muted); + }); + + onGroupCallEvent.add(GroupCallEvent.LocalMuteStateChanged); + return true; + } + + Future setLocalVideoMuted(bool muted) async { + if (!await hasVideoDevice()) { + return false; + } + + if (localUserMediaStream != null) { + localUserMediaStream!.setVideoMuted(muted); + setTracksEnabled(localUserMediaStream!.stream!.getVideoTracks(), !muted); + } + + calls.forEach((call) { + call.setLocalVideoMuted(muted); + }); + + onGroupCallEvent.add(GroupCallEvent.LocalMuteStateChanged); + return true; + } + + bool get screensharingEnabled => isScreensharing(); + + Future setScreensharingEnabled( + bool enabled, + String desktopCapturerSourceId, + ) async { + if (enabled == isScreensharing()) { + return enabled; + } + + if (enabled) { + try { + Logs().v('Asking for screensharing permissions...'); + final stream = await _getDisplayMedia(); + stream.getTracks().forEach((track) { + track.onEnded = () { + setScreensharingEnabled(false, ''); + track.onEnded = null; + }; + }); + Logs().v( + 'Screensharing permissions granted. Setting screensharing enabled on all calls'); + localDesktopCapturerSourceId = desktopCapturerSourceId; + localScreenshareStream = WrappedMediaStream( + renderer: voip.delegate.createRenderer(), + stream: stream, + userId: client.userID!, + room: room, + client: client, + purpose: SDPStreamMetadataPurpose.Screenshare, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isWeb: voip.delegate.isWeb, + isGroupCall: true); + + addScreenshareStream(localScreenshareStream!); + await localScreenshareStream!.initialize(); + + onGroupCallEvent.add(GroupCallEvent.LocalScreenshareStateChanged); + + calls.forEach((call) { + call.addLocalStream( + localScreenshareStream!.stream!, localScreenshareStream!.purpose); + }); + + await sendMemberStateEvent(); + + return true; + } catch (error) { + Logs().e('Enabling screensharing error', error); + lastError = GroupCallError(GroupCallErrorCode.NoUserMedia, + 'Failed to get screen-sharing stream: ', error); + onGroupCallEvent.add(GroupCallEvent.Error); + return false; + } + } else { + calls.forEach((call) { + call.removeLocalStream(call.localScreenSharingStream!); + }); + stopMediaStream(localScreenshareStream!.stream); + removeScreenshareStream(localScreenshareStream!); + localScreenshareStream = null; + localDesktopCapturerSourceId = null; + await sendMemberStateEvent(); + onGroupCallEvent.add(GroupCallEvent.LocalMuteStateChanged); + return false; + } + } + + bool isScreensharing() { + return localScreenshareStream != null; + } + + void onIncomingCall(CallSession newCall) { + // The incoming calls may be for another room, which we will ignore. + if (newCall.room.id != room.id) { + return; + } + + if (newCall.state != CallState.kRinging) { + Logs().w('Incoming call no longer in ringing state. Ignoring.'); + return; + } + + if (newCall.groupCallId == null || newCall.groupCallId != groupCallId) { + Logs().v( + 'Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call'); + newCall.reject(); + return; + } + + final opponentMemberId = newCall.remoteUser!.id; + final existingCall = getCallByUserId(opponentMemberId); + + if (existingCall != null && existingCall.callId == newCall.callId) { + return; + } + + Logs().v('GroupCall: incoming call from: $opponentMemberId'); + + // Check if the user calling has an existing call and use this call instead. + if (existingCall != null) { + replaceCall(existingCall, newCall); + } else { + addCall(newCall); + } + + newCall.answerWithStreams(getLocalStreams()); + } + + Future sendMemberStateEvent() { + final deviceId = client.deviceID; + return updateMemberCallState(IGroupCallRoomMemberCallState.formJson({ + 'm.call_id': groupCallId, + 'm.devices': [ + { + 'device_id': deviceId, + 'session_id': client.groupCallSessionId, + 'feeds': getLocalStreams() + .map((feed) => ({ + 'purpose': feed.purpose, + })) + .toList(), + // TODO: Add data channels + }, + ], + // TODO 'm.foci' + })); + } + + Future removeMemberStateEvent() { + return updateMemberCallState(); + } + + Future updateMemberCallState( + [IGroupCallRoomMemberCallState? memberCallState]) async { + final localUserId = client.userID; + + final currentStateEvent = + await getStateEvent(EventTypes.GroupCallMemberPrefix, localUserId); + final eventContent = currentStateEvent?.content ?? {}; + var calls = []; + + if (currentStateEvent != null) { + final memberStateEvent = IGroupCallRoomMemberState.fromJson(eventContent); + calls = memberStateEvent.calls; + final existingCallIndex = + calls.indexWhere((element) => groupCallId == element.call_id); + + if (existingCallIndex != -1) { + if (memberCallState != null) { + calls.replaceRange(existingCallIndex, 1, [memberCallState]); + } else { + calls.removeAt(existingCallIndex); + } + } else if (memberCallState != null) { + calls.add(memberCallState); + } + } else if (memberCallState != null) { + calls.add(memberCallState); + } + + final content = { + 'm.calls': calls.map((e) => e.toJson()).toList(), + }; + + await client.setRoomStateWithKey( + room.id, EventTypes.GroupCallMemberPrefix, localUserId!, content); + } + + void onMemberStateChanged(MatrixEvent event) async { + // The member events may be received for another room, which we will ignore. + if (event.roomId != room.id) { + return; + } + + final user = await room.requestUser(event.stateKey!); + + if (user == null) { + return; + } + + final callsState = IGroupCallRoomMemberState.fromJson(event.content); + + if (callsState is List) { + Logs() + .w('Ignoring member state from ${user.id} member not in any calls.'); + _removeParticipant(user.id); + return; + } + + // Currently we only support a single call per room. So grab the first call. + final callState = + callsState.calls.isNotEmpty ? callsState.calls.elementAt(0) : null; + + if (callState == null) { + Logs().w( + 'Room member ${user.id} does not have a valid m.call_id set. Ignoring.'); + _removeParticipant(user.id); + return; + } + + final callId = callState.call_id; + if (callId != null && callId != groupCallId) { + Logs().w( + 'Call id $callId does not match group call id $groupCallId, ignoring.'); + _removeParticipant(user.id); + return; + } + + _addParticipant(user); + + // Don't process your own member. + final localUserId = client.userID; + + if (user.id == localUserId) { + return; + } + + if (state != GroupCallState.Entered) { + return; + } + + // Only initiate a call with a user who has a userId that is lexicographically + // less than your own. Otherwise, that user will call you. + if (localUserId!.compareTo(user.id) > 0) { + Logs().i('Waiting for ${user.id} to send call invite.'); + return; + } + + final existingCall = getCallByUserId(user.id); + + if (existingCall != null) { + return; + } + + final opponentDevice = await getDeviceForMember(user.id); + + if (opponentDevice == null) { + Logs().w('No opponent device found for ${user.id}, ignoring.'); + lastError = GroupCallError( + '400', + GroupCallErrorCode.UnknownDevice, + 'Outgoing Call: No opponent device found for ${user.id}, ignoring.', + ); + onGroupCallEvent.add(GroupCallEvent.Error); + return; + } + + final opts = CallOptions() + ..callId = genCallID() + ..room = room + ..voip = voip + ..dir = CallDirection.kOutgoing + ..localPartyId = client.deviceID! + ..groupCallId = groupCallId + ..type = CallType.kVideo + ..iceServers = []; + + final newCall = voip.createNewCall(opts); + newCall.opponentDeviceId = opponentDevice.device_id; + newCall.opponentSessionId = opponentDevice.session_id; + newCall.remoteUser = await room.requestUser(user.id, ignoreErrors: true); + newCall.invitee = user.id; + + final requestScreenshareFeed = opponentDevice.feeds.indexWhere( + (IGroupCallRoomMemberFeed feed) => + feed.purpose == SDPStreamMetadataPurpose.Screenshare) != + -1; + + await newCall.placeCallWithStreams( + getLocalStreams(), requestScreenshareFeed); + + if (dataChannelsEnabled) { + newCall.createDataChannel('datachannel', dataChannelOptions!); + } + + addCall(newCall); + } + + Future getDeviceForMember(String userId) async { + final memberStateEvent = + await getStateEvent(EventTypes.GroupCallMemberPrefix, userId); + if (memberStateEvent == null) { + return null; + } + + final memberState = + IGroupCallRoomMemberState.fromJson(memberStateEvent.content); + + final memberGroupCallState = + memberState.calls.where(((call) => call.call_id == groupCallId)); + + if (memberGroupCallState.isEmpty) { + return null; + } + + final memberDevices = memberGroupCallState.first.devices; + + if (memberDevices.isEmpty) { + return null; + } + + /// NOTE: For now we only support one device so we use the device id in + /// the first source. + return memberDevices[0]; + } + + /// Monitor member status and respond to mesh calls by regularly updating + /// the state event in the room + void onRetryCallLoop(Timer _) async { + final memberStateEvents = + await getStateEventsList(EventTypes.GroupCallMemberPrefix); + + memberStateEvents.forEach((event) { + final memberId = event.senderId; + final existingCall = + calls.indexWhere((call) => call.remoteUser!.id == memberId) != -1; + final retryCallCount = retryCallCounts[memberId] ?? 0; + if (!existingCall && retryCallCount < 3) { + retryCallCounts[memberId] = retryCallCount + 1; + onMemberStateChanged(event); + } + }); + } + + CallSession? getCallByUserId(String userId) { + final value = calls.where((item) => item.remoteUser!.id == userId); + if (value.isNotEmpty) { + return value.first; + } + return null; + } + + void addCall(CallSession call) { + calls.add(call); + initCall(call); + onGroupCallEvent.add(GroupCallEvent.CallsChanged); + } + + void replaceCall(CallSession existingCall, CallSession replacementCall) { + final existingCallIndex = + calls.indexWhere((element) => element == existingCall); + + if (existingCallIndex == -1) { + throw Exception('Couldn\'t find call to replace'); + } + + calls.removeAt(existingCallIndex); + calls.add(replacementCall); + + disposeCall(existingCall, CallErrorCode.Replaced); + initCall(replacementCall); + + onGroupCallEvent.add(GroupCallEvent.CallsChanged); + } + + /// Removes a peer call from group calls. + void removeCall(CallSession call, String hangupReason) { + disposeCall(call, hangupReason); + + calls.removeWhere((element) => call.callId == element.callId); + + onGroupCallEvent.add(GroupCallEvent.CallsChanged); + } + + /// init a peer call from group calls. + void initCall(CallSession call) { + final opponentMemberId = call.opponentDeviceId; + + if (opponentMemberId == null) { + throw Exception('Cannot init call without user id'); + } + + call.onCallStateChanged.stream + .listen(((event) => onCallStateChanged(call, event))); + + call.onCallReplaced.stream.listen((CallSession newCall) { + replaceCall(call, newCall); + }); + + call.onCallStreamsChanged.stream.listen((call) { + call.tryRemoveStopedStreams(); + onStreamsChanged(call); + }); + + call.onCallHangup.stream.listen((event) { + onCallHangup(call); + }); + + call.onStreamAdd.stream.listen((stream) { + if (!stream.isLocal()) { + onStreamAdd.add(stream); + } + }); + + call.onStreamRemoved.stream.listen((stream) { + if (!stream.isLocal()) { + onStreamRemoved.add(stream); + } + }); + } + + void disposeCall(CallSession call, String hangupReason) { + final opponentMemberId = call.opponentDeviceId; + + if (opponentMemberId == null) { + throw Exception('Cannot dispose call without user id'); + } + + callHandlers.remove(opponentMemberId); + + if (call.hangupReason == CallErrorCode.Replaced) { + return; + } + + if (call.state != CallState.kEnded) { + call.hangup(hangupReason, false); + } + + final usermediaStream = getUserMediaStreamByUserId(opponentMemberId); + + if (usermediaStream != null) { + removeUserMediaStream(usermediaStream); + } + + final screenshareStream = getScreenshareStreamByUserId(opponentMemberId); + + if (screenshareStream != null) { + removeScreenshareStream(screenshareStream); + } + } + + String? getCallUserId(CallSession call) { + return call.remoteUser?.id ?? call.invitee; + } + + void onStreamsChanged(CallSession call) { + final opponentMemberId = getCallUserId(call); + + if (opponentMemberId == null) { + throw Exception('Cannot change call streams without user id'); + } + + final currentUserMediaStream = getUserMediaStreamByUserId(opponentMemberId); + final remoteUsermediaStream = call.remoteUserMediaStream; + final remoteStreamChanged = remoteUsermediaStream != currentUserMediaStream; + + if (remoteStreamChanged) { + if (currentUserMediaStream == null && remoteUsermediaStream != null) { + addUserMediaStream(remoteUsermediaStream); + } else if (currentUserMediaStream != null && + remoteUsermediaStream != null) { + replaceUserMediaStream(currentUserMediaStream, remoteUsermediaStream); + } else if (currentUserMediaStream != null && + remoteUsermediaStream == null) { + removeUserMediaStream(currentUserMediaStream); + } + } + + final currentScreenshareStream = + getScreenshareStreamByUserId(opponentMemberId); + final remoteScreensharingStream = call.remoteScreenSharingStream; + final remoteScreenshareStreamChanged = + remoteScreensharingStream != currentScreenshareStream; + + if (remoteScreenshareStreamChanged) { + if (currentScreenshareStream == null && + remoteScreensharingStream != null) { + addScreenshareStream(remoteScreensharingStream); + } else if (currentScreenshareStream != null && + remoteScreensharingStream != null) { + replaceScreenshareStream( + currentScreenshareStream, remoteScreensharingStream); + } else if (currentScreenshareStream != null && + remoteScreensharingStream == null) { + removeScreenshareStream(currentScreenshareStream); + } + } + + onGroupCallFeedsChanged.add(this); + } + + void onCallStateChanged(CallSession call, CallState state) { + final audioMuted = localUserMediaStream?.isAudioMuted() ?? true; + if (call.localUserMediaStream != null && + call.isMicrophoneMuted != audioMuted) { + call.setMicrophoneMuted(audioMuted); + } + + final videoMuted = localUserMediaStream?.isAudioMuted() ?? true; + + if (call.localUserMediaStream != null && + call.isLocalVideoMuted != videoMuted) { + call.setLocalVideoMuted(videoMuted); + } + + if (state == CallState.kConnected) { + retryCallCounts.remove(call.remoteUser!.id); + } + } + + void onCallHangup(CallSession call) { + if (call.hangupReason == CallErrorCode.Replaced) { + return; + } + onStreamsChanged(call); + removeCall(call, call.hangupReason!); + } + + WrappedMediaStream? getUserMediaStreamByUserId(String userId) { + final stream = userMediaStreams.where((stream) => stream.userId == userId); + if (stream.isNotEmpty) { + return stream.first; + } + return null; + } + + void addUserMediaStream(WrappedMediaStream stream) { + userMediaStreams.add(stream); + //callFeed.measureVolumeActivity(true); + onStreamAdd.add(stream); + onGroupCallEvent.add(GroupCallEvent.UserMediaStreamsChanged); + } + + void replaceUserMediaStream( + WrappedMediaStream existingStream, WrappedMediaStream replacementStream) { + final streamIndex = userMediaStreams + .indexWhere((stream) => stream.userId == existingStream.userId); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find user media stream to replace'); + } + + userMediaStreams.replaceRange(streamIndex, 1, [replacementStream]); + + existingStream.dispose(); + //replacementStream.measureVolumeActivity(true); + onGroupCallEvent.add(GroupCallEvent.UserMediaStreamsChanged); + } + + void removeUserMediaStream(WrappedMediaStream stream) { + final streamIndex = + userMediaStreams.indexWhere((stream) => stream.userId == stream.userId); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find user media stream to remove'); + } + + userMediaStreams.removeWhere((element) => element.userId == stream.userId); + + onStreamRemoved.add(stream); + + if (voip.delegate.isWeb) { + stream.stream!.getTracks().forEach((element) { + element.stop(); + }); + } + + stream.dispose(); + + onGroupCallEvent.add(GroupCallEvent.UserMediaStreamsChanged); + + if (activeSpeaker == stream.userId && userMediaStreams.isNotEmpty) { + activeSpeaker = userMediaStreams[0].userId; + onGroupCallEvent.add(GroupCallEvent.ActiveSpeakerChanged); + } + } + + void onActiveSpeakerLoop() { + /* TODO(duan): + var topAvg = 0.0; + String? nextActiveSpeaker; + + userMediaFeeds.forEach((callFeed) { + if (callFeed.userId == client.userID && userMediaFeeds.length > 1) { + return; + } + + var total = 0; + + for (var i = 0; i < callFeed.speakingVolumeSamples.length; i++) { + final volume = callFeed.speakingVolumeSamples[i]; + total += max(volume, SPEAKING_THRESHOLD); + } + + final avg = total / callFeed.speakingVolumeSamples.length; + + if (topAvg != 0 || avg > topAvg) { + topAvg = avg; + nextActiveSpeaker = callFeed.userId; + } + }); + + if (nextActiveSpeaker != null && + activeSpeaker != nextActiveSpeaker && + topAvg > SPEAKING_THRESHOLD) { + activeSpeaker = nextActiveSpeaker; + onGroupCallEvent.add(GroupCallEvent.ActiveSpeakerChanged); + } + + activeSpeakerLoopTimeout = + Timer(Duration(seconds: activeSpeakerInterval), onActiveSpeakerLoop); + */ + } + + WrappedMediaStream? getScreenshareStreamByUserId(String userId) { + final stream = + screenshareStreams.where((stream) => stream.userId == userId); + if (stream.isNotEmpty) { + return stream.first; + } + return null; + } + + void addScreenshareStream(WrappedMediaStream stream) { + screenshareStreams.add(stream); + onStreamAdd.add(stream); + onGroupCallEvent.add(GroupCallEvent.ScreenshareStreamsChanged); + } + + void replaceScreenshareStream( + WrappedMediaStream existingStream, WrappedMediaStream replacementStream) { + final streamIndex = screenshareStreams + .indexWhere((stream) => stream.userId == existingStream.userId); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find screenshare stream to replace'); + } + + screenshareStreams.replaceRange(streamIndex, 1, [replacementStream]); + + existingStream.dispose(); + onGroupCallEvent.add(GroupCallEvent.ScreenshareStreamsChanged); + } + + void removeScreenshareStream(WrappedMediaStream stream) { + final streamIndex = screenshareStreams + .indexWhere((stream) => stream.userId == stream.userId); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find screenshare stream to remove'); + } + + screenshareStreams + .removeWhere((element) => element.userId == stream.userId); + + onStreamRemoved.add(stream); + + if (voip.delegate.isWeb) { + stream.stream!.getTracks().forEach((element) { + element.stop(); + }); + } + + stream.dispose(); + + onGroupCallEvent.add(GroupCallEvent.ScreenshareStreamsChanged); + } + + void _addParticipant(User user) { + if (participants.indexWhere((m) => m.id == user.id) != -1) { + return; + } + + participants.add(user); + + onGroupCallEvent.add(GroupCallEvent.ParticipantsChanged); + } + + void _removeParticipant(String userid) { + final index = participants.indexWhere((m) => m.id == userid); + + if (index == -1) { + return; + } + + participants.removeAt(index); + + onGroupCallEvent.add(GroupCallEvent.ParticipantsChanged); + } +} diff --git a/lib/src/voip/images/famedly-1v1-call.drawio.png b/lib/src/voip/images/famedly-1v1-call.drawio.png new file mode 100644 index 00000000..a523c2ab Binary files /dev/null and b/lib/src/voip/images/famedly-1v1-call.drawio.png differ diff --git a/lib/src/voip/utils.dart b/lib/src/voip/utils.dart new file mode 100644 index 00000000..4eb6f223 --- /dev/null +++ b/lib/src/voip/utils.dart @@ -0,0 +1,32 @@ +import 'package:random_string/random_string.dart'; +import 'package:webrtc_interface/webrtc_interface.dart'; + +void stopMediaStream(MediaStream? stream) async { + stream?.getTracks().forEach((element) async { + await element.stop(); + }); +} + +void setTracksEnabled(List tracks, bool enabled) { + tracks.forEach((element) { + element.enabled = enabled; + }); +} + +Future hasAudioDevice() async { + //TODO(duan): implement this, check if there is any audio device + return true; +} + +Future hasVideoDevice() async { + //TODO(duan): implement this, check if there is any video device + return true; +} + +String roomAliasFromRoomName(String roomName) { + return roomName.trim().replaceAll('-', '').toLowerCase(); +} + +String genCallID() { + return '${DateTime.now().millisecondsSinceEpoch}' + randomAlphaNumeric(16); +} diff --git a/lib/src/voip/voip.dart b/lib/src/voip/voip.dart new file mode 100644 index 00000000..90eb6f47 --- /dev/null +++ b/lib/src/voip/voip.dart @@ -0,0 +1,690 @@ +import 'dart:async'; +import 'dart:core'; + +import 'package:webrtc_interface/webrtc_interface.dart'; +import 'package:sdp_transform/sdp_transform.dart' as sdp_transform; + +import '../../matrix.dart'; + +/// Delegate WebRTC basic functionality. +abstract class WebRTCDelegate { + MediaDevices get mediaDevices; + Future createPeerConnection( + Map configuration, + [Map constraints = const {}]); + VideoRenderer createRenderer(); + void playRingtone(); + void stopRingtone(); + void handleNewCall(CallSession session); + void handleCallEnded(CallSession session); + + void handleNewGroupCall(GroupCall groupCall); + void handleGroupCallEnded(GroupCall groupCall); + + bool get isBackgroud; + bool get isWeb; +} + +class VoIP { + TurnServerCredentials? _turnServerCredentials; + Map calls = {}; + Map groupCalls = {}; + final StreamController onIncomingCall = + StreamController.broadcast(); + String? currentCID; + String? currentGroupCID; + String? get localPartyId => client.deviceID; + final Client client; + final WebRTCDelegate delegate; + + void _handleEvent( + Event event, + Function(String roomId, String senderId, Map content) + func) => + func(event.roomId!, event.senderId, event.content); + + VoIP(this.client, this.delegate) : super() { + client.onCallInvite.stream + .listen((event) => _handleEvent(event, onCallInvite)); + client.onCallAnswer.stream + .listen((event) => _handleEvent(event, onCallAnswer)); + client.onCallCandidates.stream + .listen((event) => _handleEvent(event, onCallCandidates)); + client.onCallHangup.stream + .listen((event) => _handleEvent(event, onCallHangup)); + client.onCallReject.stream + .listen((event) => _handleEvent(event, onCallReject)); + client.onCallNegotiate.stream + .listen((event) => _handleEvent(event, onCallNegotiate)); + client.onCallReplaces.stream + .listen((event) => _handleEvent(event, onCallReplaces)); + client.onCallSelectAnswer.stream + .listen((event) => _handleEvent(event, onCallSelectAnswer)); + client.onSDPStreamMetadataChangedReceived.stream.listen( + (event) => _handleEvent(event, onSDPStreamMetadataChangedReceived)); + client.onAssertedIdentityReceived.stream + .listen((event) => _handleEvent(event, onAssertedIdentityReceived)); + + client.onGroupCallRequest.stream.listen((event) { + Logs().v('[VOIP] onGroupCallRequest: type ${event.toJson()}.'); + onRoomStateChanged(event); + }); + + client.onToDeviceEvent.stream.listen((event) { + Logs().v('[VOIP] onToDeviceEvent: type ${event.toJson()}.'); + + if (event.type == 'org.matrix.call_duplicate_session') { + Logs().v('[VOIP] onToDeviceEvent: duplicate session.'); + return; + } + + final confId = event.content['conf_id']; + final groupCall = groupCalls[confId]; + if (groupCall == null) { + Logs().e('[VOIP] onToDeviceEvent: groupCall is null.'); + return; + } + final roomId = groupCall.room.id; + final senderId = event.senderId; + final content = event.content; + switch (event.type) { + case EventTypes.CallInvite: + onCallInvite(roomId, senderId, content); + break; + case EventTypes.CallAnswer: + onCallAnswer(roomId, senderId, content); + break; + case EventTypes.CallCandidates: + onCallCandidates(roomId, senderId, content); + break; + case EventTypes.CallHangup: + onCallHangup(roomId, senderId, content); + break; + case EventTypes.CallReject: + onCallReject(roomId, senderId, content); + break; + case EventTypes.CallNegotiate: + onCallNegotiate(roomId, senderId, content); + break; + case EventTypes.CallReplaces: + onCallReplaces(roomId, senderId, content); + break; + case EventTypes.CallSelectAnswer: + onCallSelectAnswer(roomId, senderId, content); + break; + case EventTypes.CallSDPStreamMetadataChanged: + case EventTypes.CallSDPStreamMetadataChangedPrefix: + onSDPStreamMetadataChangedReceived(roomId, senderId, content); + break; + case EventTypes.CallAssertedIdentity: + onAssertedIdentityReceived(roomId, senderId, content); + break; + } + }); + } + + Future onCallInvite( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + + Logs().v( + '[VOIP] onCallInvite $senderId => ${client.userID}, \ncontent => ${content.toString()}'); + + final String callId = content['call_id']; + final String partyId = content['party_id']; + final int lifetime = content['lifetime']; + final String? confId = content['conf_id']; + final String? deviceId = content['device_id']; + final call = calls[callId]; + + if (call != null && call.state == CallState.kEnded) { + // Session already exist. + Logs().v('[VOIP] onCallInvite: Session [$callId] already exist.'); + return; + } + + if (content['invitee'] != null && content['invitee'] != client.userID) { + return; // This invite was meant for another user in the room + } + + if (content['capabilities'] != null) { + final capabilities = CallCapabilities.fromJson(content['capabilities']); + Logs().v( + '[VOIP] CallCapabilities: dtmf => ${capabilities.dtmf}, transferee => ${capabilities.transferee}'); + } + + var callType = CallType.kVoice; + SDPStreamMetadata? sdpStreamMetadata; + if (content[sdpStreamMetadataKey] != null) { + sdpStreamMetadata = + SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey]); + sdpStreamMetadata.sdpStreamMetadatas + .forEach((streamId, SDPStreamPurpose purpose) { + Logs().v( + '[VOIP] [$streamId] => purpose: ${purpose.purpose}, audioMuted: ${purpose.audio_muted}, videoMuted: ${purpose.video_muted}'); + + if (!purpose.video_muted) { + callType = CallType.kVideo; + } + }); + } else { + callType = getCallType(content['offer']['sdp']); + } + + final room = client.getRoomById(roomId); + + final opts = CallOptions() + ..voip = this + ..callId = callId + ..groupCallId = confId + ..dir = CallDirection.kIncoming + ..type = callType + ..room = room! + ..localPartyId = localPartyId! + ..iceServers = await getIceSevers(); + + final newCall = createNewCall(opts); + newCall.remotePartyId = partyId; + newCall.remoteUser = await room.requestUser(senderId); + newCall.opponentDeviceId = deviceId; + newCall.opponentSessionId = content['sender_session_id']; + + final offer = RTCSessionDescription( + content['offer']['sdp'], + content['offer']['type'], + ); + await newCall + .initWithInvite(callType, offer, sdpStreamMetadata, lifetime) + .then((_) { + // Popup CallingPage for incoming call. + if (!delegate.isBackgroud && confId == null) { + delegate.handleNewCall(newCall); + } + onIncomingCall.add(newCall); + }); + currentCID = callId; + + if (delegate.isBackgroud) { + /// Forced to enable signaling synchronization until the end of the call. + client.backgroundSync = true; + + ///TODO: notify the callkeep that the call is incoming. + } + // Play ringtone + delegate.playRingtone(); + } + + void onCallAnswer( + String roomId, String senderId, Map content) async { + Logs().v('[VOIP] onCallAnswer => ${content.toString()}'); + final String callId = content['call_id']; + final String partyId = content['party_id']; + + final call = calls[callId]; + if (call != null) { + if (senderId == client.userID) { + // Ignore messages to yourself. + if (!call.answeredByUs) { + delegate.stopRingtone(); + } + if (call.state == CallState.kRinging) { + call.onAnsweredElsewhere(); + } + return; + } + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call answer for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + call.remotePartyId = partyId; + call.remoteUser = await call.room.requestUser(senderId); + + final answer = RTCSessionDescription( + content['answer']['sdp'], content['answer']['type']); + + SDPStreamMetadata? metadata; + if (content[sdpStreamMetadataKey] != null) { + metadata = SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey]); + } + call.onAnswerReceived(answer, metadata); + } else { + Logs().v('[VOIP] onCallAnswer: Session [$callId] not found!'); + } + } + + void onCallCandidates( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + Logs().v('[VOIP] onCallCandidates => ${content.toString()}'); + final String callId = content['call_id']; + final call = calls[callId]; + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call candidates for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + call.onCandidatesReceived(content['candidates']); + } else { + Logs().v('[VOIP] onCallCandidates: Session [$callId] not found!'); + } + } + + void onCallHangup(String roomId, String _ /*senderId unused*/, + Map content) async { + // stop play ringtone, if this is an incoming call + if (!delegate.isBackgroud) { + delegate.stopRingtone(); + } + Logs().v('[VOIP] onCallHangup => ${content.toString()}'); + final String callId = content['call_id']; + final call = calls[callId]; + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call hangup for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + // hangup in any case, either if the other party hung up or we did on another device + call.terminate(CallParty.kRemote, + content['reason'] ?? CallErrorCode.UserHangup, true); + } else { + Logs().v('[VOIP] onCallHangup: Session [$callId] not found!'); + } + currentCID = null; + } + + void onCallReject( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + final String callId = content['call_id']; + Logs().d('Reject received for call ID ' + callId); + + final call = calls[callId]; + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call reject for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + call.onRejectReceived(content['reason']); + } else { + Logs().v('[VOIP] onCallHangup: Session [$callId] not found!'); + } + } + + void onCallReplaces( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + final String callId = content['call_id']; + Logs().d('onCallReplaces received for call ID ' + callId); + final call = calls[callId]; + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call replace for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + //TODO: handle replaces + } + } + + void onCallSelectAnswer( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + final String callId = content['call_id']; + Logs().d('SelectAnswer received for call ID ' + callId); + final call = calls[callId]; + final String selectedPartyId = content['selected_party_id']; + + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call select answer for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + call.onSelectAnswerReceived(selectedPartyId); + } + } + + void onSDPStreamMetadataChangedReceived( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + final String callId = content['call_id']; + Logs().d('SDP Stream metadata received for call ID ' + callId); + final call = calls[callId]; + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call sdp metadata change for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + + if (content[sdpStreamMetadataKey] == null) { + Logs().d('SDP Stream metadata is null'); + return; + } + call.onSDPStreamMetadataReceived( + SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey])); + } + } + + void onAssertedIdentityReceived( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + final String callId = content['call_id']; + Logs().d('Asserted identity received for call ID ' + callId); + final call = calls[callId]; + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call asserted identity for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + + if (content['asserted_identity'] == null) { + Logs().d('asserted_identity is null '); + return; + } + call.onAssertedIdentityReceived( + AssertedIdentity.fromJson(content['asserted_identity'])); + } + } + + void onCallNegotiate( + String roomId, String senderId, Map content) async { + if (senderId == client.userID) { + // Ignore messages to yourself. + return; + } + final String callId = content['call_id']; + Logs().d('Negotiate received for call ID ' + callId); + final call = calls[callId]; + if (call != null) { + if (call.room.id != roomId) { + Logs().w( + 'Ignoring call negotiation for room $roomId claiming to be for call in room ${call.room.id}'); + return; + } + + final description = content['description']; + try { + SDPStreamMetadata? metadata; + if (content[sdpStreamMetadataKey] != null) { + metadata = SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey]); + } + call.onNegotiateReceived(metadata, + RTCSessionDescription(description['sdp'], description['type'])); + } catch (err) { + Logs().e('Failed to complete negotiation ${err.toString()}'); + } + } + } + + CallType getCallType(String sdp) { + try { + final session = sdp_transform.parse(sdp); + if (session['media'].indexWhere((e) => e['type'] == 'video') != -1) { + return CallType.kVideo; + } + } catch (err) { + Logs().e('Failed to getCallType ${err.toString()}'); + } + + return CallType.kVoice; + } + + Future requestTurnServerCredentials() async { + return true; + } + + Future>> getIceSevers() async { + if (_turnServerCredentials == null) { + try { + _turnServerCredentials = await client.getTurnServer(); + } catch (e) { + Logs().v('[VOIP] getTurnServerCredentials error => ${e.toString()}'); + } + } + + if (_turnServerCredentials == null) { + return []; + } + + return [ + { + 'username': _turnServerCredentials!.username, + 'credential': _turnServerCredentials!.password, + 'url': _turnServerCredentials!.uris[0] + } + ]; + } + + /// Make a P2P call to room + /// + /// [roomId] The room id to call + /// + /// [type] The type of call to be made. + Future inviteToCall(String roomId, CallType type) async { + final room = client.getRoomById(roomId); + if (room == null) { + Logs().v('[VOIP] Invalid room id [$roomId].'); + return Null as CallSession; + } + final callId = 'cid${DateTime.now().millisecondsSinceEpoch}'; + final opts = CallOptions() + ..callId = callId + ..type = type + ..dir = CallDirection.kOutgoing + ..room = room + ..voip = this + ..localPartyId = localPartyId! + ..iceServers = await getIceSevers(); + + final newCall = createNewCall(opts); + currentCID = callId; + await newCall.initOutboundCall(type).then((_) { + if (!delegate.isBackgroud) { + delegate.handleNewCall(newCall); + } + }); + currentCID = callId; + return newCall; + } + + CallSession createNewCall(CallOptions opts) { + final call = CallSession(opts); + calls[opts.callId] = call; + return call; + } + + /// Create a new group call in an existing room. + /// + /// [roomId] The room id to call + /// + /// [type] The type of call to be made. + /// + /// [intent] The intent of the call. + /// + /// [dataChannelsEnabled] Whether data channels are enabled. + /// + /// [dataChannelOptions] The data channel options. + Future newGroupCall(String roomId, String type, String intent, + [bool? dataChannelsEnabled, + RTCDataChannelInit? dataChannelOptions]) async { + final room = client.getRoomById(roomId); + if (room == null) { + Logs().v('[VOIP] Invalid room id [$roomId].'); + return null; + } + final groupId = genCallID(); + final groupCall = GroupCall( + groupCallId: groupId, + client: client, + voip: this, + room: room, + type: type, + intent: intent, + dataChannelsEnabled: dataChannelsEnabled ?? false, + dataChannelOptions: dataChannelOptions ?? RTCDataChannelInit(), + ).create(); + groupCalls[groupId] = groupCall; + return groupCall; + } + + GroupCall? getGroupCallForRoom(String roomId) { + return groupCalls[roomId]; + } + + GroupCall? getGroupCallById(String groupCallId) { + return groupCalls[groupCallId]; + } + + void startGroupCalls() async { + final rooms = client.rooms; + rooms.forEach((element) { + createGroupCallForRoom(element); + }); + } + + void stopGroupCalls() { + groupCalls.forEach((_, groupCall) { + groupCall.terminate(); + }); + groupCalls.clear(); + } + + /// Create a new group call in an existing room. + Future createGroupCallForRoom(Room room) async { + final events = await client.getRoomState(room.id); + events.sort((a, b) => a.originServerTs.compareTo(b.originServerTs)); + events.forEach((element) async { + if (element.type == EventTypes.GroupCallPrefix) { + if (element.content['m.terminated'] != null) { + return; + } + await createGroupCallFromRoomStateEvent(element); + } + }); + return; + } + + /// Create a new group call from a room state event. + Future createGroupCallFromRoomStateEvent( + MatrixEvent event) async { + final roomId = event.roomId; + final content = event.content; + + final room = client.getRoomById(roomId!); + + if (room == null) { + Logs().w('Couldn\'t find room $roomId for GroupCall'); + return null; + } + + final groupCallId = event.stateKey; + + final callType = content['m.type']; + + if (callType != GroupCallType.Video && callType != GroupCallType.Voice) { + Logs().w('Received invalid group call type $callType for room $roomId.'); + return null; + } + + final callIntent = content['m.intent']; + + if (callIntent != GroupCallIntent.Prompt && + callIntent != GroupCallIntent.Room && + callIntent != GroupCallIntent.Ring) { + Logs() + .w('Received invalid group call intent $callType for room $roomId.'); + return null; + } + + final dataChannelOptionsMap = content['m.data_channel_options']; + + var dataChannelsEnabled = false; + final dataChannelOptions = RTCDataChannelInit(); + + if (dataChannelOptionsMap != null) { + dataChannelsEnabled = + dataChannelOptionsMap['dataChannelsEnabled'] as bool; + dataChannelOptions.ordered = dataChannelOptionsMap['ordered'] as bool; + dataChannelOptions.maxRetransmits = + dataChannelOptionsMap['maxRetransmits'] as int; + dataChannelOptions.maxRetransmits = + dataChannelOptionsMap['maxRetransmits'] as int; + dataChannelOptions.protocol = dataChannelOptionsMap['protocol'] as String; + } + + final groupCall = GroupCall( + client: client, + voip: this, + room: room, + groupCallId: groupCallId, + type: callType, + intent: callIntent, + dataChannelsEnabled: dataChannelsEnabled, + dataChannelOptions: dataChannelOptions); + + groupCalls[groupCallId!] = groupCall; + groupCalls[room.id] = groupCall; + delegate.handleNewGroupCall(groupCall); + return groupCall; + } + + void onRoomStateChanged(MatrixEvent event) { + final eventType = event.type; + final roomId = event.roomId; + if (eventType == EventTypes.GroupCallPrefix) { + final groupCallId = event.content['groupCallId']; + final content = event.content; + final currentGroupCall = groupCalls[groupCallId]; + if (currentGroupCall == null && content['m.terminated'] == null) { + createGroupCallFromRoomStateEvent(event); + } else if (currentGroupCall != null && + currentGroupCall.groupCallId == groupCallId) { + if (content['m.terminated'] != null) { + currentGroupCall.terminate(emitStateEvent: false); + } else if (content['m.type'] != currentGroupCall.type) { + // TODO: Handle the callType changing when the room state changes + Logs().w( + 'The group call type changed for room: $roomId. Changing the group call type is currently unsupported.'); + } + } else if (currentGroupCall != null && + currentGroupCall.groupCallId != groupCallId) { + // TODO: Handle new group calls and multiple group calls + Logs().w( + 'Multiple group calls detected for room: $roomId. Multiple group calls are currently unsupported.'); + } + } else if (eventType == EventTypes.GroupCallMemberPrefix) { + final groupCall = groupCalls[roomId]; + if (groupCall == null) { + return; + } + groupCall.onMemberStateChanged(event); + } + } +} diff --git a/lib/src/voip_content.dart b/lib/src/voip/voip_content.dart similarity index 100% rename from lib/src/voip_content.dart rename to lib/src/voip/voip_content.dart diff --git a/test/fake_voip_delegate.dart b/test/fake_voip_delegate.dart deleted file mode 100644 index 4e0ade42..00000000 --- a/test/fake_voip_delegate.dart +++ /dev/null @@ -1,52 +0,0 @@ -import 'package:matrix/src/voip.dart'; -import 'package:webrtc_interface/src/rtc_video_renderer.dart'; -import 'package:webrtc_interface/src/rtc_peerconnection.dart'; -import 'package:webrtc_interface/src/mediadevices.dart'; - -class FakeVoIPDelegate extends WebRTCDelegate { - @override - Future createPeerConnection( - Map configuration, - [Map constraints = const {}]) { - // TODO: implement createPeerConnection - throw UnimplementedError(); - } - - @override - VideoRenderer createRenderer() { - // TODO: implement createRenderer - throw UnimplementedError(); - } - - @override - void handleCallEnded(CallSession session) { - // TODO: implement handleCallEnded - } - - @override - void handleNewCall(CallSession session) { - // TODO: implement handleNewCall - } - - @override - // TODO: implement isBackgroud - bool get isBackgroud => throw UnimplementedError(); - - @override - // TODO: implement isWeb - bool get isWeb => throw UnimplementedError(); - - @override - // TODO: implement mediaDevices - MediaDevices get mediaDevices => throw UnimplementedError(); - - @override - void playRingtone() { - // TODO: implement playRingtone - } - - @override - void stopRingtone() { - // TODO: implement stopRingtone - } -} diff --git a/test/room_test.dart b/test/room_test.dart index e475566c..63d1f47e 100644 --- a/test/room_test.dart +++ b/test/room_test.dart @@ -25,19 +25,16 @@ import 'package:test/test.dart'; import 'fake_client.dart'; import 'fake_matrix_api.dart'; -import 'fake_voip_delegate.dart'; void main() { late Client matrix; late Room room; - late VoIP voip; /// All Tests related to the Event group('Room', () { Logs().level = Level.error; test('Login', () async { matrix = await getClient(); - voip = VoIP(matrix, FakeVoIPDelegate()); }); test('Create from json', () async { @@ -761,18 +758,19 @@ void main() { }); test('Test call methods', () async { - await voip.sendInviteToCall(room, '1234', 1234, '4567', '7890', 'sdp', + final call = CallSession(CallOptions()..room = room); + await call.sendInviteToCall(room, '1234', 1234, '4567', '7890', 'sdp', txid: '1234'); - await voip.sendAnswerCall(room, '1234', 'sdp', '4567', txid: '1234'); - await voip.sendCallCandidates(room, '1234', '4567', [], txid: '1234'); - await voip.sendSelectCallAnswer(room, '1234', 1234, '4567', '6789', + await call.sendAnswerCall(room, '1234', 'sdp', '4567', txid: '1234'); + await call.sendCallCandidates(room, '1234', '4567', [], txid: '1234'); + await call.sendSelectCallAnswer(room, '1234', 1234, '4567', '6789', txid: '1234'); - await voip.sendCallReject(room, '1234', 1234, '4567', txid: '1234'); - await voip.sendCallNegotiate(room, '1234', 1234, '4567', 'sdp', + await call.sendCallReject(room, '1234', 1234, '4567', txid: '1234'); + await call.sendCallNegotiate(room, '1234', 1234, '4567', 'sdp', txid: '1234'); - await voip.sendHangupCall(room, '1234', '4567', 'user_hangup', + await call.sendHangupCall(room, '1234', '4567', 'user_hangup', txid: '1234'); - await voip.sendAssertedIdentity( + await call.sendAssertedIdentity( room, '1234', '4567', @@ -780,9 +778,9 @@ void main() { ..displayName = 'name' ..id = 'some_id', txid: '1234'); - await voip.sendCallReplaces(room, '1234', '4567', CallReplaces(), + await call.sendCallReplaces(room, '1234', '4567', CallReplaces(), txid: '1234'); - await voip.sendSDPStreamMetadataChanged( + await call.sendSDPStreamMetadataChanged( room, '1234', '4567', SDPStreamMetadata({}), txid: '1234'); });