diff --git a/.github/workflows/versions.env b/.github/workflows/versions.env index 771a477f..2ae227a1 100644 --- a/.github/workflows/versions.env +++ b/.github/workflows/versions.env @@ -1,2 +1,2 @@ -flutter_version=3.19.0 -dart_version=3.3.0 +flutter_version=3.16.9 +dart_version=3.2.6 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c39eb18..95cf85c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -303,7 +303,7 @@ or before logout), excessive linebreaks in markdown messages and a few edge case - fix: Check the max server file size after shrinking not before (Krille) - fix: casting of a List to List in getEventList and getEventIdList (td) - fix: Skip rules with unknown conditions (Nicolas Werner) -- fix: allow passing a WrappedMediaStream to GroupCall.enter() to use as the local user media stream (td) +- fix: allow passing a WrappedMediaStream to GroupCallSession.enter() to use as the local user media stream (td) ## [0.19.0] - 21st April 2023 diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index 59a64ef4..d44116d3 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -561,6 +561,8 @@ class OlmManager { DateTime.now() .subtract(Duration(hours: 1)) .isBefore(_restoredOlmSessionsTime[mapKey]!)) { + Logs().w( + '[OlmManager] Skipping restore session, one was restored in the past hour'); return; } _restoredOlmSessionsTime[mapKey] = DateTime.now(); @@ -736,7 +738,7 @@ class OlmManager { Future handleToDeviceEvent(ToDeviceEvent event) async { if (event.type == EventTypes.Dummy) { - // We receive dan encrypted m.dummy. This means that the other end was not able to + // We received an encrypted m.dummy. This means that the other end was not able to // decrypt our last message. So, we re-send it. final encryptedContent = event.encryptedContent; if (encryptedContent == null || encryption.olmDatabase == null) { diff --git a/lib/matrix.dart b/lib/matrix.dart index 8268a0d6..e09b315f 100644 --- a/lib/matrix.dart +++ b/lib/matrix.dart @@ -30,13 +30,22 @@ export 'src/database/sqflite_encryption_helper.dart'; export 'src/event.dart'; export 'src/presence.dart'; export 'src/event_status.dart'; -export 'src/voip/call.dart'; -export 'src/voip/group_call.dart'; +export 'src/voip/call_session.dart'; +export 'src/voip/group_call_session.dart'; export 'src/voip/voip.dart'; -export 'src/voip/voip_content.dart'; -export 'src/voip/conn_tester.dart'; -export 'src/voip/utils.dart'; -export 'src/voip/voip_room_extension.dart'; +export 'src/voip/backend/livekit_backend.dart'; +export 'src/voip/backend/call_backend_model.dart'; +export 'src/voip/backend/mesh_backend.dart'; +export 'src/voip/models/call_events.dart'; +export 'src/voip/models/webrtc_delegate.dart'; +export 'src/voip/models/call_participant.dart'; +export 'src/voip/models/key_provider.dart'; +export 'src/voip/utils/conn_tester.dart'; +export 'src/voip/utils/voip_constants.dart'; +export 'src/voip/utils/rtc_candidate_extension.dart'; +export 'src/voip/utils/famedly_call_extension.dart'; +export 'src/voip/utils/types.dart'; +export 'src/voip/utils/wrapped_media_stream.dart'; export 'src/room.dart'; export 'src/timeline.dart'; export 'src/user.dart'; diff --git a/lib/matrix_api_lite/model/event_types.dart b/lib/matrix_api_lite/model/event_types.dart index daa008b6..0e39bb5d 100644 --- a/lib/matrix_api_lite/model/event_types.dart +++ b/lib/matrix_api_lite/model/event_types.dart @@ -59,8 +59,6 @@ abstract class EventTypes { static const String CallAssertedIdentity = 'm.call.asserted_identity'; static const String CallAssertedIdentityPrefix = 'org.matrix.call.asserted_identity'; - static const String GroupCallPrefix = 'org.matrix.msc3401.call'; - static const String GroupCallMemberPrefix = 'org.matrix.msc3401.call.member'; static const String Unknown = 'm.unknown'; // To device event types @@ -94,6 +92,26 @@ abstract class EventTypes { static String secretStorageKey(String keyId) => 'm.secret_storage.key.$keyId'; // Spaces - static const String spaceParent = 'm.space.parent'; - static const String spaceChild = 'm.space.child'; + static const String SpaceParent = 'm.space.parent'; + static const String SpaceChild = 'm.space.child'; + + // MatrixRTC + static const String GroupCallMember = 'com.famedly.call.member'; + static const String GroupCallMemberEncryptionKeys = + '$GroupCallMember.encryption_keys'; + static const String GroupCallMemberEncryptionKeysRequest = + '$GroupCallMember.encryption_keys_request'; + static const String GroupCallMemberCandidates = '$GroupCallMember.candidates'; + static const String GroupCallMemberInvite = '$GroupCallMember.invite'; + static const String GroupCallMemberAnswer = '$GroupCallMember.answer'; + static const String GroupCallMemberHangup = '$GroupCallMember.hangup'; + static const String GroupCallMemberSelectAnswer = + '$GroupCallMember.select_answer'; + static const String GroupCallMemberReject = '$GroupCallMember.reject'; + static const String GroupCallMemberNegotiate = '$GroupCallMember.negotiate'; + static const String GroupCallMemberSDPStreamMetadataChanged = + '$GroupCallMember.sdp_stream_metadata_changed'; + static const String GroupCallMemberReplaces = '$GroupCallMember.replaces'; + static const String GroupCallMemberAssertedIdentity = + '$GroupCallMember.asserted_identity'; } diff --git a/lib/src/client.dart b/lib/src/client.dart index fb7e8e1c..0c979ff5 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -219,8 +219,8 @@ class Client extends MatrixApi { EventTypes.Encryption, EventTypes.RoomCanonicalAlias, EventTypes.RoomTombstone, - EventTypes.spaceChild, - EventTypes.spaceParent, + EventTypes.SpaceChild, + EventTypes.SpaceParent, EventTypes.RoomCreate, ]); roomPreviewLastEvents.addAll([ @@ -231,8 +231,7 @@ class Client extends MatrixApi { EventTypes.CallAnswer, EventTypes.CallReject, EventTypes.CallHangup, - EventTypes.GroupCallPrefix, - EventTypes.GroupCallMemberPrefix, + EventTypes.GroupCallMember, ]); // register all the default commands @@ -800,8 +799,7 @@ class Client extends MatrixApi { if (groupCall) { powerLevelContentOverride ??= {}; powerLevelContentOverride['events'] = { - EventTypes.GroupCallMemberPrefix: 0, - EventTypes.GroupCallPrefix: 0, + EventTypes.GroupCallMember: 0, }; } final roomId = await createRoom( @@ -1264,6 +1262,10 @@ class Client extends MatrixApi { final CachedStreamController onToDeviceEvent = CachedStreamController(); + /// Tells you about to-device and room call specific events in sync + final CachedStreamController> onCallEvents = + CachedStreamController(); + /// Called when the login state e.g. user gets logged out. final CachedStreamController onLoginStateChanged = CachedStreamController(); @@ -1295,41 +1297,6 @@ class Client extends MatrixApi { final CachedStreamController onAccountData = CachedStreamController(); - /// Will be called on call invites. - final CachedStreamController onCallInvite = CachedStreamController(); - - /// Will be called on call hangups. - final CachedStreamController onCallHangup = CachedStreamController(); - - /// Will be called on call candidates. - final CachedStreamController onCallCandidates = - CachedStreamController(); - - /// Will be called on call answers. - final CachedStreamController onCallAnswer = CachedStreamController(); - - /// Will be called on call replaces. - final CachedStreamController onCallReplaces = CachedStreamController(); - - /// Will be called on select answers. - final CachedStreamController onCallSelectAnswer = - CachedStreamController(); - - /// Will be called on rejects. - final CachedStreamController onCallReject = CachedStreamController(); - - /// Will be called on negotiates. - final CachedStreamController onCallNegotiate = - CachedStreamController(); - - /// Will be called on Asserted Identity received. - final CachedStreamController onAssertedIdentityReceived = - CachedStreamController(); - - /// Will be called on SDPStream Metadata changed. - final CachedStreamController onSDPStreamMetadataChangedReceived = - CachedStreamController(); - /// Will be called when another device is requesting session keys for a room. final CachedStreamController onRoomKeyRequest = CachedStreamController(); @@ -1343,9 +1310,6 @@ class Client extends MatrixApi { final CachedStreamController onUiaRequest = CachedStreamController(); - final CachedStreamController onGroupCallRequest = - CachedStreamController(); - final CachedStreamController onGroupMember = CachedStreamController(); final CachedStreamController onRoomState = CachedStreamController(); @@ -2069,6 +2033,7 @@ class Client extends MatrixApi { Future _handleToDeviceEvents(List events) async { final Map> roomsWithNewKeyToSessionId = {}; + final List callToDeviceEvents = []; for (final event in events) { var toDeviceEvent = ToDeviceEvent.fromJson(event.toJson()); Logs().v('Got to_device event of type ${toDeviceEvent.type}'); @@ -2089,9 +2054,16 @@ class Client extends MatrixApi { } await encryption?.handleToDeviceEvent(toDeviceEvent); } + if (toDeviceEvent.type.startsWith(CallConstants.callEventsRegxp)) { + callToDeviceEvents.add(toDeviceEvent); + } onToDeviceEvent.add(toDeviceEvent); } + if (callToDeviceEvents.isNotEmpty) { + onCallEvents.add(callToDeviceEvents); + } + // emit updates for all events in the queue for (final entry in roomsWithNewKeyToSessionId.entries) { final roomId = entry.key; @@ -2257,7 +2229,7 @@ class Client extends MatrixApi { {bool store = true}) async { // Calling events can be omitted if they are outdated from the same sync. So // we collect them first before we handle them. - final callEvents = {}; + final callEvents = []; for (final event in events) { // The client must ignore any new m.room.encryption event to prevent @@ -2308,93 +2280,17 @@ class Client extends MatrixApi { if (prevBatch != null && (type == EventUpdateType.timeline || type == EventUpdateType.decryptedTimelineQueue)) { - if ((update.content.tryGet('type')?.startsWith('m.call.') ?? - false) || - (update.content - .tryGet('type') - ?.startsWith('org.matrix.call.') ?? - false)) { + if ((update.content + .tryGet('type') + ?.startsWith(CallConstants.callEventsRegxp) ?? + false)) { final callEvent = Event.fromJson(update.content, room); - final callId = callEvent.content.tryGet('call_id'); callEvents.add(callEvent); - - // Call Invites should be omitted for a call that is already answered, - // has ended, is rejectd or replaced. - const callEndedEventTypes = { - EventTypes.CallAnswer, - EventTypes.CallHangup, - EventTypes.CallReject, - EventTypes.CallReplaces, - }; - const ommitWhenCallEndedTypes = { - EventTypes.CallInvite, - EventTypes.CallCandidates, - EventTypes.CallNegotiate, - EventTypes.CallSDPStreamMetadataChanged, - EventTypes.CallSDPStreamMetadataChangedPrefix, - }; - - if (callEndedEventTypes.contains(callEvent.type)) { - callEvents.removeWhere((event) { - if (ommitWhenCallEndedTypes.contains(event.type) && - event.content.tryGet('call_id') == callId) { - Logs().v( - 'Ommit "${event.type}" event for an already terminated call'); - return true; - } - return false; - }); - } - - final age = callEvent.unsigned?.tryGet('age') ?? - (DateTime.now().millisecondsSinceEpoch - - callEvent.originServerTs.millisecondsSinceEpoch); - - callEvents.removeWhere((element) { - if (callEvent.type == EventTypes.CallInvite && - age > - (callEvent.content.tryGet('lifetime') ?? - CallTimeouts.callInviteLifetime.inMilliseconds)) { - Logs().v( - 'Ommiting invite event ${callEvent.eventId} as age was older than lifetime'); - return true; - } - return false; - }); } } } - - callEvents.forEach(_callStreamByCallEvent); - } - - void _callStreamByCallEvent(Event event) { - if (event.type == EventTypes.CallInvite) { - onCallInvite.add(event); - } else if (event.type == EventTypes.CallHangup) { - onCallHangup.add(event); - } else if (event.type == EventTypes.CallAnswer) { - onCallAnswer.add(event); - } else if (event.type == EventTypes.CallCandidates) { - onCallCandidates.add(event); - } else if (event.type == EventTypes.CallSelectAnswer) { - onCallSelectAnswer.add(event); - } else if (event.type == EventTypes.CallReject) { - onCallReject.add(event); - } else if (event.type == EventTypes.CallNegotiate) { - onCallNegotiate.add(event); - } else if (event.type == EventTypes.CallReplaces) { - onCallReplaces.add(event); - } else if (event.type == EventTypes.CallAssertedIdentity || - event.type == EventTypes.CallAssertedIdentityPrefix) { - onAssertedIdentityReceived.add(event); - } else if (event.type == EventTypes.CallSDPStreamMetadataChanged || - event.type == EventTypes.CallSDPStreamMetadataChangedPrefix) { - onSDPStreamMetadataChangedReceived.add(event); - // TODO(duan): Only used (org.matrix.msc3401.call) during the current test, - // need to add GroupCallPrefix in matrix_api_lite - } else if (event.type == EventTypes.GroupCallPrefix) { - onGroupCallRequest.add(event); + if (callEvents.isNotEmpty) { + onCallEvents.add(callEvents); } } diff --git a/lib/src/room.dart b/lib/src/room.dart index 5a00401b..13016c80 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1835,7 +1835,7 @@ class Room { return powerForChangingStateEvent(action) <= ownPowerLevel; } - /// returns the powerlevel required for chaning the `action` defaults to + /// returns the powerlevel required for changing the `action` defaults to /// state_default if `action` isn't specified in events override. /// If there is no state_default in the m.room.power_levels event, the /// state_default is 50. If the room contains no m.room.power_levels event, @@ -1850,25 +1850,18 @@ class Room { 50; } - bool get canCreateGroupCall => - canChangeStateEvent(EventTypes.GroupCallPrefix) && groupCallsEnabled; - - bool get canJoinGroupCall => - canChangeStateEvent(EventTypes.GroupCallMemberPrefix) && - groupCallsEnabled; - - /// if returned value is not null `org.matrix.msc3401.call.member` is present + /// if returned value is not null `EventTypes.GroupCallMember` is present /// and group calls can be used - bool get groupCallsEnabled { + bool get groupCallsEnabledForEveryone { final powerLevelMap = getState(EventTypes.RoomPowerLevels)?.content; if (powerLevelMap == null) return false; - return powerForChangingStateEvent(EventTypes.GroupCallMemberPrefix) <= - getDefaultPowerLevel(powerLevelMap) && - powerForChangingStateEvent(EventTypes.GroupCallPrefix) <= - getDefaultPowerLevel(powerLevelMap); + return powerForChangingStateEvent(EventTypes.GroupCallMember) <= + getDefaultPowerLevel(powerLevelMap); } - /// sets the `org.matrix.msc3401.call.member` power level to users default for + bool get canJoinGroupCall => canChangeStateEvent(EventTypes.GroupCallMember); + + /// sets the `EventTypes.GroupCallMember` power level to users default for /// group calls, needs permissions to change power levels Future enableGroupCalls() async { if (!canChangePowerLevel) return; @@ -1878,9 +1871,7 @@ class Room { final eventsMap = newPowerLevelMap.tryGetMap('events') ?? {}; eventsMap.addAll({ - EventTypes.GroupCallPrefix: getDefaultPowerLevel(currentPowerLevelsMap), - EventTypes.GroupCallMemberPrefix: - getDefaultPowerLevel(currentPowerLevelsMap) + EventTypes.GroupCallMember: getDefaultPowerLevel(currentPowerLevelsMap) }); newPowerLevelMap.addAll({'events': eventsMap}); await client.setRoomStateWithKey( @@ -2229,14 +2220,14 @@ class Room { /// `m.space`. bool get isSpace => getState(EventTypes.RoomCreate)?.content.tryGet('type') == - RoomCreationTypes.mSpace; // TODO: Magic string! + RoomCreationTypes.mSpace; /// The parents of this room. Currently this SDK doesn't yet set the canonical /// flag and is not checking if this room is in fact a child of this space. /// You should therefore not rely on this and always check the children of /// the space. List get spaceParents => - states[EventTypes.spaceParent] + states[EventTypes.SpaceParent] ?.values .map((state) => SpaceParent.fromState(state)) .where((child) => child.via.isNotEmpty) @@ -2249,7 +2240,7 @@ class Room { /// sorted at the end of the list. List get spaceChildren => !isSpace ? throw Exception('Room is not a space!') - : (states[EventTypes.spaceChild] + : (states[EventTypes.SpaceChild] ?.values .map((state) => SpaceChild.fromState(state)) .where((child) => child.via.isNotEmpty) @@ -2268,12 +2259,12 @@ class Room { }) async { if (!isSpace) throw Exception('Room is not a space!'); via ??= [client.userID!.domain!]; - await client.setRoomStateWithKey(id, EventTypes.spaceChild, roomId, { + await client.setRoomStateWithKey(id, EventTypes.SpaceChild, roomId, { 'via': via, if (order != null) 'order': order, if (suggested != null) 'suggested': suggested, }); - await client.setRoomStateWithKey(roomId, EventTypes.spaceParent, id, { + await client.setRoomStateWithKey(roomId, EventTypes.SpaceParent, id, { 'via': via, }); return; diff --git a/lib/src/utils/matrix_default_localizations.dart b/lib/src/utils/matrix_default_localizations.dart index abf40287..3d7328d5 100644 --- a/lib/src/utils/matrix_default_localizations.dart +++ b/lib/src/utils/matrix_default_localizations.dart @@ -235,7 +235,6 @@ class MatrixDefaultLocalizations extends MatrixLocalizations { } @override - // TODO: implement youAcceptedTheInvitation String get youAcceptedTheInvitation => 'You accepted the invitation'; @override diff --git a/lib/src/utils/space_child.dart b/lib/src/utils/space_child.dart index 08c39fb1..d6cdc056 100644 --- a/lib/src/utils/space_child.dart +++ b/lib/src/utils/space_child.dart @@ -26,7 +26,7 @@ class SpaceChild { final bool? suggested; SpaceChild.fromState(Event state) - : assert(state.type == EventTypes.spaceChild), + : assert(state.type == EventTypes.SpaceChild), roomId = state.stateKey, via = state.content.tryGetList('via') ?? [], order = state.content.tryGet('order') ?? '', @@ -39,7 +39,7 @@ class SpaceParent { final bool? canonical; SpaceParent.fromState(Event state) - : assert(state.type == EventTypes.spaceParent), + : assert(state.type == EventTypes.SpaceParent), roomId = state.stateKey, via = state.content.tryGetList('via') ?? [], canonical = state.content.tryGet('canonical'); diff --git a/lib/src/voip/README.md b/lib/src/voip/README.md index 7b864d8b..ede250a4 100644 --- a/lib/src/voip/README.md +++ b/lib/src/voip/README.md @@ -1,6 +1,14 @@ -# VOIP for Matrix SDK +# Famedly Calls -1:1 and group calls +Supports +- 1:1 webrtc calls +- Group calls with: + - mesh webrtc calls + - just handling state of calls and signallnig for e2ee keys in sfu mode (check `isLivekitCall`) + +Places where we diverted from spec afaik: +- To enable p2p calls between devices of the same user, pass a `invitee_device_id` to the `m.call.invite` method +- **to-device call events such as in msc3401 MUST `room_id` to map the event to a room** ## Overview @@ -8,13 +16,82 @@ `CallSession` objects are created by calling `inviteToCall` and `onCallInvite`. -`GroupCall` objects are created by calling `createGroupCall`. +`GroupCallSession` objects are created by calling `fetchOrCreateGroupCall`. + +## Group Calls + +All communication for group calls happens over to-device events except the `com.famedly.call.member` event. + + + +Sends the `com.famedly.call.member` event to signal an active membership. The format has to be the following: + +### Events - + +```json5 +"content": { + "memberships": [ + { + "application": "m.call", + "backend": { + "type": "mesh" + }, + "call_id": "!qoQQTYnzXOHSdEgqQp:im.staging.famedly.de", + "device_id": "YVGPEWNLDD", + "expires_ts": 1705152401042, + "scope": "m.room" + } + ] +} +``` + +- **application**: could be anything f.ex `m.call`, `m.game` or `m.board` +- **backend**: see below +- **call_id**: the call id, currently setting it to the roomId makes the call for the whole room, this is to avoid parallel calls starting up. For user scoped calls in a room you could set this to `AuserId:BuserId`. The sdk does not restrict setting roomId for user scoped calls atm. +- **device_id**: The sdk supports calling between devices of same users, so this needs to be set to the sender device id. +- **expires_ts**: ms since epoch when this membership event should be considered expired. Check `lib/src/voip/utils/constants.dart` for current values of how long the inital period is and how often this gets autoupdated. +- **scope**: room scoped calls are `m.room`, user scoped can be `m.user` + + + +#### The backend can be either `mesh` or `livekit` + +##### Livekit - +```json5 +"backend": { + "livekit_alias": "!qoQQTYnzXOHSdEgqQp:im.staging.famedly.de", + "livekit_service_url": "https://famedly-livekit-server.teedee.dev/jwt", + "type": "livekit" +}, +``` + +##### Mesh - +```json5 +"backend": { + "type": "mesh" +}, +``` + +#### E2EE Events - + +When in SFU/Livekit mode, the sdk can handle sending and requesting encryption keys. Currently it uses the following events: + +- sending: `com.famedly.call.encryption_keys` +- requesting: `com.famedly.call.encryption_keys.request` + +As usual remember to send the `party_id`/`sender_session_id` to map your keys to the right userId and deviceId + +You need to implement `EncryptionKeyProvider` and set the override the methods to interact with your actual keyProvider. The main one as of now is `onSetEncryptionKey`. + +You can request missing keys whenever needed using `groupCall.requestEncrytionKey(remoteParticipants)`. + ## 1:1 calls ### 1. Basic call flow This flow explains the code flow for a 1v1 call. + This code flow is still used in group call, the only difference is that group call uses `toDevice` message to send `m.call.*` events ![1v1 call](images/famedly-1v1-call.drawio.png) @@ -79,11 +156,14 @@ We need to use the matrix roomId to initiate the call, the initial call can be After the call is sent, you can use `onCallStateChanged` to listen the call state events. These events are used to change the display of the call UI state, for example, change the control buttons, display `Hangup (cancel)` button before connecting, and display `mute mic, mute cam, hold/unhold, hangup` buttons after connected. +You cannot call a whole room, please specify the userId you intend to call in `inviteToCall` + + ```dart final voip = VoIP(client, MyVoipApp()); /// Create a new call -final newCall = await voip.inviteToCall(roomId, CallType.kVideo); +final newCall = await voip.inviteToCall(roomId, CallType.kVideo, userId); newCall.onCallStateChanged.stream.listen((state) { /// handle call state change event， diff --git a/lib/src/voip/backend/call_backend_model.dart b/lib/src/voip/backend/call_backend_model.dart new file mode 100644 index 00000000..8bdbc8bb --- /dev/null +++ b/lib/src/voip/backend/call_backend_model.dart @@ -0,0 +1,112 @@ +import 'dart:async'; + +import 'package:matrix/matrix.dart'; +import 'package:matrix/src/voip/models/call_membership.dart'; + +abstract class CallBackend { + String type; + + CallBackend({ + required this.type, + }); + + factory CallBackend.fromJson(Map json) { + final String type = json['type'] as String; + if (type == 'mesh') { + return MeshBackend( + type: type, + ); + } else if (type == 'livekit') { + return LiveKitBackend( + livekitAlias: json['livekit_alias'] as String, + livekitServiceUrl: json['livekit_service_url'] as String, + type: type, + ); + } else { + throw ArgumentError('Invalid type: $type'); + } + } + + Map toJson(); + + bool get e2eeEnabled; + + CallParticipant? get activeSpeaker; + + WrappedMediaStream? get localUserMediaStream; + + WrappedMediaStream? get localScreenshareStream; + + List get userMediaStreams; + + List get screenShareStreams; + + bool get isLocalVideoMuted; + + bool get isMicrophoneMuted; + + Future initLocalStream( + GroupCallSession groupCall, { + WrappedMediaStream? stream, + }); + + Future updateMediaDeviceForCalls(); + + Future setupP2PCallsWithExistingMembers(GroupCallSession groupCall); + + Future setupP2PCallWithNewMember( + GroupCallSession groupCall, + CallParticipant rp, + CallMembership mem, + ); + + Future dispose(GroupCallSession groupCall); + + Future onNewParticipant( + GroupCallSession groupCall, + List anyJoined, + ); + + Future onLeftParticipant( + GroupCallSession groupCall, + List anyLeft, + ); + + Future requestEncrytionKey( + GroupCallSession groupCall, + List remoteParticipants, + ); + + Future onCallEncryption( + GroupCallSession groupCall, + String userId, + String deviceId, + Map content, + ); + + Future onCallEncryptionKeyRequest( + GroupCallSession groupCall, + String userId, + String deviceId, + Map content, + ); + + Future setDeviceMuted( + GroupCallSession groupCall, + bool muted, + MediaInputKind kind, + ); + + Future setScreensharingEnabled( + GroupCallSession groupCall, + bool enabled, + String desktopCapturerSourceId, + ); + + List>? getCurrentFeeds(); + + @override + bool operator ==(Object other); + @override + int get hashCode; +} diff --git a/lib/src/voip/backend/livekit_backend.dart b/lib/src/voip/backend/livekit_backend.dart new file mode 100644 index 00000000..c7ec5fd2 --- /dev/null +++ b/lib/src/voip/backend/livekit_backend.dart @@ -0,0 +1,506 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:matrix/matrix.dart'; +import 'package:matrix/src/utils/crypto/crypto.dart'; +import 'package:matrix/src/voip/models/call_membership.dart'; + +class LiveKitBackend extends CallBackend { + final String livekitServiceUrl; + final String livekitAlias; + + @override + final bool e2eeEnabled; + + LiveKitBackend({ + required this.livekitServiceUrl, + required this.livekitAlias, + super.type = 'livekit', + this.e2eeEnabled = true, + }); + + Timer? _memberLeaveEncKeyRotateDebounceTimer; + + /// participant:keyIndex:keyBin + final Map> _encryptionKeysMap = {}; + + final List _setNewKeyTimeouts = []; + + int _indexCounter = 0; + + /// used to send the key again incase someone `onCallEncryptionKeyRequest` but don't just send + /// the last one because you also cycle back in your window which means you + /// could potentially end up sharing a past key + int get latestLocalKeyIndex => _latestLocalKeyIndex; + int _latestLocalKeyIndex = 0; + + /// the key currently being used by the local cryptor, can possibly not be the latest + /// key, check `latestLocalKeyIndex` for latest key + int get currentLocalKeyIndex => _currentLocalKeyIndex; + int _currentLocalKeyIndex = 0; + + Map? _getKeysForParticipant(CallParticipant participant) { + return _encryptionKeysMap[participant]; + } + + /// always chooses the next possible index, we cycle after 16 because + /// no real adv with infinite list + int _getNewEncryptionKeyIndex() { + final newIndex = _indexCounter % 16; + _indexCounter++; + return newIndex; + } + + /// makes a new e2ee key for local user and sets it with a delay if specified + /// used on first join and when someone leaves + /// + /// also does the sending for you + Future _makeNewSenderKey( + GroupCallSession groupCall, bool delayBeforeUsingKeyOurself) async { + final key = secureRandomBytes(32); + final keyIndex = _getNewEncryptionKeyIndex(); + Logs().i('[VOIP E2EE] Generated new key $key at index $keyIndex'); + + await _setEncryptionKey( + groupCall, + groupCall.localParticipant!, + keyIndex, + key, + delayBeforeUsingKeyOurself: delayBeforeUsingKeyOurself, + send: true, + ); + } + + /// also does the sending for you + Future _ratchetLocalParticipantKey( + GroupCallSession groupCall, + List sendTo, + ) async { + final keyProvider = groupCall.voip.delegate.keyProvider; + + if (keyProvider == null) { + throw Exception('[VOIP] _ratchetKey called but KeyProvider was null'); + } + + final myKeys = _encryptionKeysMap[groupCall.localParticipant]; + + if (myKeys == null || myKeys.isEmpty) { + await _makeNewSenderKey(groupCall, false); + return; + } + + Uint8List? ratchetedKey; + + while (ratchetedKey == null || ratchetedKey.isEmpty) { + Logs().i('[VOIP E2EE] Ignoring empty ratcheted key'); + ratchetedKey = await keyProvider.onRatchetKey( + groupCall.localParticipant!, + latestLocalKeyIndex, + ); + } + + Logs().i( + '[VOIP E2EE] Ratched latest key to $ratchetedKey at idx $latestLocalKeyIndex'); + + await _setEncryptionKey( + groupCall, + groupCall.localParticipant!, + latestLocalKeyIndex, + ratchetedKey, + delayBeforeUsingKeyOurself: false, + send: true, + sendTo: sendTo, + ); + } + + /// sets incoming keys and also sends the key if it was for the local user + /// if sendTo is null, its sent to all _participants, see `_sendEncryptionKeysEvent` + Future _setEncryptionKey( + GroupCallSession groupCall, + CallParticipant participant, + int encryptionKeyIndex, + Uint8List encryptionKeyBin, { + bool delayBeforeUsingKeyOurself = false, + bool send = false, + List? sendTo, + }) async { + final encryptionKeys = + _encryptionKeysMap[participant] ?? {}; + + encryptionKeys[encryptionKeyIndex] = encryptionKeyBin; + _encryptionKeysMap[participant] = encryptionKeys; + if (participant.isLocal) { + _latestLocalKeyIndex = encryptionKeyIndex; + } + + if (send) { + await _sendEncryptionKeysEvent( + groupCall, + encryptionKeyIndex, + sendTo: sendTo, + ); + } + + if (delayBeforeUsingKeyOurself) { + // now wait for the key to propogate and then set it, hopefully users can + // stil decrypt everything + final useKeyTimeout = Future.delayed(CallTimeouts.useKeyDelay, () async { + Logs().i( + '[VOIP E2EE] setting key changed event for ${participant.id} idx $encryptionKeyIndex key $encryptionKeyBin'); + await groupCall.voip.delegate.keyProvider?.onSetEncryptionKey( + participant, encryptionKeyBin, encryptionKeyIndex); + if (participant.isLocal) { + _currentLocalKeyIndex = encryptionKeyIndex; + } + }); + _setNewKeyTimeouts.add(useKeyTimeout); + } else { + Logs().i( + '[VOIP E2EE] setting key changed event for ${participant.id} idx $encryptionKeyIndex key $encryptionKeyBin'); + await groupCall.voip.delegate.keyProvider?.onSetEncryptionKey( + participant, encryptionKeyBin, encryptionKeyIndex); + if (participant.isLocal) { + _currentLocalKeyIndex = encryptionKeyIndex; + } + } + } + + /// sends the enc key to the devices using todevice, passing a list of + /// sendTo only sends events to them + /// setting keyIndex to null will send the latestKey + Future _sendEncryptionKeysEvent( + GroupCallSession groupCall, + int keyIndex, { + List? sendTo, + }) async { + Logs().i('Sending encryption keys event'); + + final myKeys = _getKeysForParticipant(groupCall.localParticipant!); + final myLatestKey = myKeys?[keyIndex]; + + final sendKeysTo = + sendTo ?? groupCall.participants.where((p) => !p.isLocal); + + if (myKeys == null || myLatestKey == null) { + Logs().w( + '[VOIP E2EE] _sendEncryptionKeysEvent Tried to send encryption keys event but no keys found!'); + await _makeNewSenderKey(groupCall, false); + await _sendEncryptionKeysEvent( + groupCall, + keyIndex, + sendTo: sendTo, + ); + return; + } + + try { + final keyContent = EncryptionKeysEventContent( + [EncryptionKeyEntry(keyIndex, base64Encode(myLatestKey))], + groupCall.groupCallId, + ); + final Map data = { + ...keyContent.toJson(), + // used to find group call in groupCalls when ToDeviceEvent happens, + // plays nicely with backwards compatibility for mesh calls + 'conf_id': groupCall.groupCallId, + 'device_id': groupCall.client.deviceID!, + 'room_id': groupCall.room.id, + }; + await _sendToDeviceEvent( + groupCall, + sendTo ?? sendKeysTo.toList(), + data, + EventTypes.GroupCallMemberEncryptionKeys, + ); + } catch (e, s) { + Logs().e('[VOIP] Failed to send e2ee keys, retrying', e, s); + await _sendEncryptionKeysEvent( + groupCall, + keyIndex, + sendTo: sendTo, + ); + } + } + + Future _sendToDeviceEvent( + GroupCallSession groupCall, + List remoteParticipants, + Map data, + String eventType, + ) async { + Logs().v( + '[VOIP] _sendToDeviceEvent: sending ${data.toString()} to ${remoteParticipants.map((e) => e.id)} '); + final txid = + VoIP.customTxid ?? groupCall.client.generateUniqueTransactionId(); + final mustEncrypt = + groupCall.room.encrypted && groupCall.client.encryptionEnabled; + + // could just combine the two but do not want to rewrite the enc thingy + // wrappers here again. + final List mustEncryptkeysToSendTo = []; + final Map>> unencryptedDataToSend = + {}; + + for (final participant in remoteParticipants) { + if (participant.deviceId == null) continue; + if (mustEncrypt) { + await groupCall.client.userDeviceKeysLoading; + final deviceKey = groupCall.client.userDeviceKeys[participant.userId] + ?.deviceKeys[participant.deviceId]; + if (deviceKey != null) { + mustEncryptkeysToSendTo.add(deviceKey); + } + } else { + unencryptedDataToSend.addAll({ + participant.userId: {participant.deviceId!: data} + }); + } + } + + // prepped data, now we send + if (mustEncrypt) { + await groupCall.client.sendToDeviceEncrypted( + mustEncryptkeysToSendTo, + eventType, + data, + ); + } else { + await groupCall.client.sendToDevice( + eventType, + txid, + unencryptedDataToSend, + ); + } + } + + @override + Map toJson() { + return { + 'type': type, + 'livekit_service_url': livekitServiceUrl, + 'livekit_alias': livekitAlias, + }; + } + + @override + Future requestEncrytionKey( + GroupCallSession groupCall, + List remoteParticipants, + ) async { + final Map data = { + 'conf_id': groupCall.groupCallId, + 'device_id': groupCall.client.deviceID!, + 'room_id': groupCall.room.id, + }; + + await _sendToDeviceEvent( + groupCall, + remoteParticipants, + data, + EventTypes.GroupCallMemberEncryptionKeysRequest, + ); + } + + @override + Future onCallEncryption( + GroupCallSession groupCall, + String userId, + String deviceId, + Map content, + ) async { + if (!e2eeEnabled) { + Logs().w('[VOIP] got sframe key but we do not support e2ee'); + return; + } + final keyContent = EncryptionKeysEventContent.fromJson(content); + + final callId = keyContent.callId; + + if (keyContent.keys.isEmpty) { + Logs().w( + '[VOIP E2EE] Received m.call.encryption_keys where keys is empty: callId=$callId'); + return; + } else { + Logs().i( + '[VOIP E2EE]: onCallEncryption, got keys from $userId:$deviceId ${keyContent.toJson()}'); + } + + for (final key in keyContent.keys) { + final encryptionKey = key.key; + final encryptionKeyIndex = key.index; + await _setEncryptionKey( + groupCall, + CallParticipant(groupCall.voip, userId: userId, deviceId: deviceId), + encryptionKeyIndex, + // base64Decode here because we receive base64Encoded version + base64Decode(encryptionKey), + delayBeforeUsingKeyOurself: false, + send: false, + ); + } + } + + @override + Future onCallEncryptionKeyRequest( + GroupCallSession groupCall, + String userId, + String deviceId, + Map content, + ) async { + if (!e2eeEnabled) { + Logs().w('[VOIP] got sframe key request but we do not support e2ee'); + return; + } + final mems = groupCall.room.getCallMembershipsForUser(userId); + if (mems + .where( + (mem) => + mem.callId == groupCall.groupCallId && + mem.userId == userId && + mem.deviceId == deviceId && + !mem.isExpired && + // sanity checks + mem.backend.type == groupCall.backend.type && + mem.roomId == groupCall.room.id && + mem.application == groupCall.application, + ) + .isNotEmpty) { + Logs().d( + '[VOIP] onCallEncryptionKeyRequest: request checks out, sending key on index: $latestLocalKeyIndex to $userId:$deviceId'); + await _sendEncryptionKeysEvent( + groupCall, + _latestLocalKeyIndex, + sendTo: [ + CallParticipant( + groupCall.voip, + userId: userId, + deviceId: deviceId, + ) + ], + ); + } + } + + @override + Future onNewParticipant( + GroupCallSession groupCall, + List anyJoined, + ) async { + if (!e2eeEnabled) return; + if (groupCall.voip.enableSFUE2EEKeyRatcheting) { + await _ratchetLocalParticipantKey(groupCall, anyJoined); + } else { + await _makeNewSenderKey(groupCall, true); + } + } + + @override + Future onLeftParticipant( + GroupCallSession groupCall, + List anyLeft, + ) async { + _encryptionKeysMap.removeWhere((key, value) => anyLeft.contains(key)); + + // debounce it because people leave at the same time + if (_memberLeaveEncKeyRotateDebounceTimer != null) { + _memberLeaveEncKeyRotateDebounceTimer!.cancel(); + } + _memberLeaveEncKeyRotateDebounceTimer = + Timer(CallTimeouts.makeKeyDelay, () async { + await _makeNewSenderKey(groupCall, true); + }); + } + + @override + Future dispose(GroupCallSession groupCall) async { + // only remove our own, to save requesting if we join again, yes the other side + // will send it anyway but welp + _encryptionKeysMap.remove(groupCall.localParticipant!); + _currentLocalKeyIndex = 0; + _latestLocalKeyIndex = 0; + _memberLeaveEncKeyRotateDebounceTimer?.cancel(); + } + + @override + List>? getCurrentFeeds() { + return null; + } + + @override + bool operator ==(Object other) => + identical(this, other) || + other is LiveKitBackend && + type == other.type && + livekitServiceUrl == other.livekitServiceUrl && + livekitAlias == other.livekitAlias; + + @override + int get hashCode => + type.hashCode ^ livekitServiceUrl.hashCode ^ livekitAlias.hashCode; + + /// get everything else from your livekit sdk in your client + @override + Future initLocalStream(GroupCallSession groupCall, + {WrappedMediaStream? stream}) async { + return null; + } + + @override + CallParticipant? get activeSpeaker => null; + + /// these are unimplemented on purpose so that you know you have + /// used the wrong method + @override + bool get isLocalVideoMuted => + throw UnimplementedError('Use livekit sdk for this'); + + @override + bool get isMicrophoneMuted => + throw UnimplementedError('Use livekit sdk for this'); + + @override + WrappedMediaStream? get localScreenshareStream => + throw UnimplementedError('Use livekit sdk for this'); + + @override + WrappedMediaStream? get localUserMediaStream => + throw UnimplementedError('Use livekit sdk for this'); + + @override + List get screenShareStreams => + throw UnimplementedError('Use livekit sdk for this'); + + @override + List get userMediaStreams => + throw UnimplementedError('Use livekit sdk for this'); + + @override + Future setDeviceMuted( + GroupCallSession groupCall, bool muted, MediaInputKind kind) async { + return; + } + + @override + Future setScreensharingEnabled(GroupCallSession groupCall, bool enabled, + String desktopCapturerSourceId) async { + return; + } + + @override + Future setupP2PCallWithNewMember(GroupCallSession groupCall, + CallParticipant rp, CallMembership mem) async { + return; + } + + @override + Future setupP2PCallsWithExistingMembers( + GroupCallSession groupCall) async { + return; + } + + @override + Future updateMediaDeviceForCalls() async { + return; + } +} diff --git a/lib/src/voip/backend/mesh_backend.dart b/lib/src/voip/backend/mesh_backend.dart new file mode 100644 index 00000000..755a1d06 --- /dev/null +++ b/lib/src/voip/backend/mesh_backend.dart @@ -0,0 +1,877 @@ +import 'dart:async'; + +import 'package:collection/collection.dart'; +import 'package:webrtc_interface/webrtc_interface.dart'; + +import 'package:matrix/matrix.dart'; +import 'package:matrix/src/utils/cached_stream_controller.dart'; +import 'package:matrix/src/voip/models/call_membership.dart'; +import 'package:matrix/src/voip/models/call_options.dart'; +import 'package:matrix/src/voip/utils/stream_helper.dart'; +import 'package:matrix/src/voip/utils/user_media_constraints.dart'; + +class MeshBackend extends CallBackend { + MeshBackend({ + super.type = 'mesh', + }); + + final List _callSessions = []; + + /// participant:volume + final Map _audioLevelsMap = {}; + + StreamSubscription? _callSubscription; + + Timer? _activeSpeakerLoopTimeout; + + final CachedStreamController onStreamAdd = + CachedStreamController(); + + final CachedStreamController onStreamRemoved = + CachedStreamController(); + + final CachedStreamController onGroupCallFeedsChanged = + CachedStreamController(); + + @override + Map toJson() { + return { + 'type': type, + }; + } + + CallParticipant? _activeSpeaker; + WrappedMediaStream? _localUserMediaStream; + WrappedMediaStream? _localScreenshareStream; + final List _userMediaStreams = []; + final List _screenshareStreams = []; + + List _getLocalStreams() { + final feeds = []; + + if (localUserMediaStream != null) { + feeds.add(localUserMediaStream!); + } + + if (localScreenshareStream != null) { + feeds.add(localScreenshareStream!); + } + + return feeds; + } + + Future _getUserMedia( + GroupCallSession groupCall, CallType type) async { + final mediaConstraints = { + 'audio': UserMediaConstraints.micMediaConstraints, + 'video': type == CallType.kVideo + ? UserMediaConstraints.camMediaConstraints + : false, + }; + + try { + return await groupCall.voip.delegate.mediaDevices + .getUserMedia(mediaConstraints); + } catch (e) { + groupCall.setState(GroupCallState.localCallFeedUninitialized); + rethrow; + } + } + + Future _getDisplayMedia(GroupCallSession groupCall) async { + final mediaConstraints = { + 'audio': false, + 'video': true, + }; + try { + return await groupCall.voip.delegate.mediaDevices + .getDisplayMedia(mediaConstraints); + } catch (e, s) { + Logs().e('[VOIP] _getDisplayMedia failed because,', e, s); + rethrow; + } + } + + CallSession? _getCallForParticipant( + GroupCallSession groupCall, CallParticipant participant) { + return _callSessions.singleWhereOrNull((call) => + call.groupCallId == groupCall.groupCallId && + CallParticipant( + groupCall.voip, + userId: call.remoteUserId!, + deviceId: call.remoteDeviceId, + ) == + participant); + } + + Future _addCall(GroupCallSession groupCall, CallSession call) async { + _callSessions.add(call); + await _initCall(groupCall, call); + groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged); + } + + /// init a peer call from group calls. + Future _initCall(GroupCallSession groupCall, CallSession call) async { + if (call.remoteUserId == null) { + throw Exception( + 'Cannot init call without proper invitee user and device Id'); + } + + call.onCallStateChanged.stream.listen(((event) async { + await _onCallStateChanged(call, event); + })); + + call.onCallReplaced.stream.listen((CallSession newCall) async { + await _replaceCall(groupCall, call, newCall); + }); + + call.onCallStreamsChanged.stream.listen((call) async { + await call.tryRemoveStopedStreams(); + await _onStreamsChanged(groupCall, call); + }); + + call.onCallHangupNotifierForGroupCalls.stream.listen((event) async { + await _onCallHangup(groupCall, call); + }); + + call.onStreamAdd.stream.listen((stream) { + if (!stream.isLocal()) { + onStreamAdd.add(stream); + } + }); + + call.onStreamRemoved.stream.listen((stream) { + if (!stream.isLocal()) { + onStreamRemoved.add(stream); + } + }); + } + + Future _replaceCall( + GroupCallSession groupCall, + CallSession existingCall, + CallSession replacementCall, + ) async { + final existingCallIndex = _callSessions + .indexWhere((element) => element.callId == existingCall.callId); + + if (existingCallIndex == -1) { + throw Exception('Couldn\'t find call to replace'); + } + + _callSessions.removeAt(existingCallIndex); + _callSessions.add(replacementCall); + + await _disposeCall(groupCall, existingCall, CallErrorCode.replaced); + await _initCall(groupCall, replacementCall); + + groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged); + } + + /// Removes a peer call from group calls. + Future _removeCall(GroupCallSession groupCall, CallSession call, + CallErrorCode hangupReason) async { + await _disposeCall(groupCall, call, hangupReason); + + _callSessions.removeWhere((element) => call.callId == element.callId); + + groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged); + } + + Future _disposeCall(GroupCallSession groupCall, CallSession call, + CallErrorCode hangupReason) async { + if (call.remoteUserId == null) { + throw Exception( + 'Cannot init call without proper invitee user and device Id'); + } + + if (call.hangupReason == CallErrorCode.replaced) { + return; + } + + if (call.state != CallState.kEnded) { + // no need to emit individual handleCallEnded on group calls + // also prevents a loop of hangup and onCallHangupNotifierForGroupCalls + await call.hangup(reason: hangupReason, shouldEmit: false); + } + + final usermediaStream = _getUserMediaStreamByParticipantId( + CallParticipant( + groupCall.voip, + userId: call.remoteUserId!, + deviceId: call.remoteDeviceId, + ).id, + ); + + if (usermediaStream != null) { + await _removeUserMediaStream(groupCall, usermediaStream); + } + + final screenshareStream = _getScreenshareStreamByParticipantId( + CallParticipant( + groupCall.voip, + userId: call.remoteUserId!, + deviceId: call.remoteDeviceId, + ).id, + ); + + if (screenshareStream != null) { + await _removeScreenshareStream(groupCall, screenshareStream); + } + } + + Future _onStreamsChanged( + GroupCallSession groupCall, CallSession call) async { + if (call.remoteUserId == null) { + throw Exception( + 'Cannot init call without proper invitee user and device Id'); + } + + final currentUserMediaStream = _getUserMediaStreamByParticipantId( + CallParticipant( + groupCall.voip, + userId: call.remoteUserId!, + deviceId: call.remoteDeviceId, + ).id, + ); + + final remoteUsermediaStream = call.remoteUserMediaStream; + final remoteStreamChanged = remoteUsermediaStream != currentUserMediaStream; + + if (remoteStreamChanged) { + if (currentUserMediaStream == null && remoteUsermediaStream != null) { + await _addUserMediaStream(groupCall, remoteUsermediaStream); + } else if (currentUserMediaStream != null && + remoteUsermediaStream != null) { + await _replaceUserMediaStream( + groupCall, currentUserMediaStream, remoteUsermediaStream); + } else if (currentUserMediaStream != null && + remoteUsermediaStream == null) { + await _removeUserMediaStream(groupCall, currentUserMediaStream); + } + } + + final currentScreenshareStream = + _getScreenshareStreamByParticipantId(CallParticipant( + groupCall.voip, + userId: call.remoteUserId!, + deviceId: call.remoteDeviceId, + ).id); + final remoteScreensharingStream = call.remoteScreenSharingStream; + final remoteScreenshareStreamChanged = + remoteScreensharingStream != currentScreenshareStream; + + if (remoteScreenshareStreamChanged) { + if (currentScreenshareStream == null && + remoteScreensharingStream != null) { + _addScreenshareStream(groupCall, remoteScreensharingStream); + } else if (currentScreenshareStream != null && + remoteScreensharingStream != null) { + await _replaceScreenshareStream( + groupCall, currentScreenshareStream, remoteScreensharingStream); + } else if (currentScreenshareStream != null && + remoteScreensharingStream == null) { + await _removeScreenshareStream(groupCall, currentScreenshareStream); + } + } + + onGroupCallFeedsChanged.add(groupCall); + } + + WrappedMediaStream? _getUserMediaStreamByParticipantId(String participantId) { + final stream = _userMediaStreams + .where((stream) => stream.participant.id == participantId); + if (stream.isNotEmpty) { + return stream.first; + } + return null; + } + + void _onActiveSpeakerLoop(GroupCallSession groupCall) async { + CallParticipant? nextActiveSpeaker; + // idc about screen sharing atm. + final userMediaStreamsCopyList = + List.from(_userMediaStreams); + for (final stream in userMediaStreamsCopyList) { + if (stream.participant.isLocal && stream.pc == null) { + continue; + } + + final List statsReport = await stream.pc!.getStats(); + statsReport + .removeWhere((element) => !element.values.containsKey('audioLevel')); + + // https://www.w3.org/TR/webrtc-stats/#summary + final otherPartyAudioLevel = statsReport + .singleWhereOrNull((element) => + element.type == 'inbound-rtp' && + element.values['kind'] == 'audio') + ?.values['audioLevel']; + if (otherPartyAudioLevel != null) { + _audioLevelsMap[stream.participant] = otherPartyAudioLevel; + } + + // https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-source + // firefox does not seem to have this though. Works on chrome and android + final ownAudioLevel = statsReport + .singleWhereOrNull((element) => + element.type == 'media-source' && + element.values['kind'] == 'audio') + ?.values['audioLevel']; + if (groupCall.localParticipant != null && + ownAudioLevel != null && + _audioLevelsMap[groupCall.localParticipant] != ownAudioLevel) { + _audioLevelsMap[groupCall.localParticipant!] = ownAudioLevel; + } + } + + double maxAudioLevel = double.negativeInfinity; + // TODO: we probably want a threshold here? + _audioLevelsMap.forEach((key, value) { + if (value > maxAudioLevel) { + nextActiveSpeaker = key; + maxAudioLevel = value; + } + }); + + if (nextActiveSpeaker != null && _activeSpeaker != nextActiveSpeaker) { + _activeSpeaker = nextActiveSpeaker; + groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged); + } + _activeSpeakerLoopTimeout?.cancel(); + _activeSpeakerLoopTimeout = Timer( + CallConstants.activeSpeakerInterval, + () => _onActiveSpeakerLoop(groupCall), + ); + } + + WrappedMediaStream? _getScreenshareStreamByParticipantId( + String participantId) { + final stream = _screenshareStreams + .where((stream) => stream.participant.id == participantId); + if (stream.isNotEmpty) { + return stream.first; + } + return null; + } + + void _addScreenshareStream( + GroupCallSession groupCall, WrappedMediaStream stream) { + _screenshareStreams.add(stream); + onStreamAdd.add(stream); + groupCall.onGroupCallEvent + .add(GroupCallStateChange.screenshareStreamsChanged); + } + + Future _replaceScreenshareStream( + GroupCallSession groupCall, + WrappedMediaStream existingStream, + WrappedMediaStream replacementStream, + ) async { + final streamIndex = _screenshareStreams.indexWhere( + (stream) => stream.participant.id == existingStream.participant.id); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find screenshare stream to replace'); + } + + _screenshareStreams.replaceRange(streamIndex, 1, [replacementStream]); + + await existingStream.dispose(); + groupCall.onGroupCallEvent + .add(GroupCallStateChange.screenshareStreamsChanged); + } + + Future _removeScreenshareStream( + GroupCallSession groupCall, + WrappedMediaStream stream, + ) async { + final streamIndex = _screenshareStreams + .indexWhere((stream) => stream.participant.id == stream.participant.id); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find screenshare stream to remove'); + } + + _screenshareStreams.removeWhere( + (element) => element.participant.id == stream.participant.id); + + onStreamRemoved.add(stream); + + if (stream.isLocal()) { + await stopMediaStream(stream.stream); + } + + groupCall.onGroupCallEvent + .add(GroupCallStateChange.screenshareStreamsChanged); + } + + Future _onCallStateChanged(CallSession call, CallState state) async { + final audioMuted = localUserMediaStream?.isAudioMuted() ?? true; + if (call.localUserMediaStream != null && + call.isMicrophoneMuted != audioMuted) { + await call.setMicrophoneMuted(audioMuted); + } + + final videoMuted = localUserMediaStream?.isVideoMuted() ?? true; + + if (call.localUserMediaStream != null && + call.isLocalVideoMuted != videoMuted) { + await call.setLocalVideoMuted(videoMuted); + } + } + + Future _onCallHangup( + GroupCallSession groupCall, + CallSession call, + ) async { + if (call.hangupReason == CallErrorCode.replaced) { + return; + } + await _onStreamsChanged(groupCall, call); + await _removeCall(groupCall, call, call.hangupReason!); + } + + Future _addUserMediaStream( + GroupCallSession groupCall, + WrappedMediaStream stream, + ) async { + _userMediaStreams.add(stream); + onStreamAdd.add(stream); + groupCall.onGroupCallEvent + .add(GroupCallStateChange.userMediaStreamsChanged); + } + + Future _replaceUserMediaStream( + GroupCallSession groupCall, + WrappedMediaStream existingStream, + WrappedMediaStream replacementStream, + ) async { + final streamIndex = _userMediaStreams.indexWhere( + (stream) => stream.participant.id == existingStream.participant.id); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find user media stream to replace'); + } + + _userMediaStreams.replaceRange(streamIndex, 1, [replacementStream]); + + await existingStream.dispose(); + groupCall.onGroupCallEvent + .add(GroupCallStateChange.userMediaStreamsChanged); + } + + Future _removeUserMediaStream( + GroupCallSession groupCall, + WrappedMediaStream stream, + ) async { + final streamIndex = _userMediaStreams.indexWhere( + (element) => element.participant.id == stream.participant.id); + + if (streamIndex == -1) { + throw Exception('Couldn\'t find user media stream to remove'); + } + + _userMediaStreams.removeWhere( + (element) => element.participant.id == stream.participant.id); + _audioLevelsMap.remove(stream.participant); + onStreamRemoved.add(stream); + + if (stream.isLocal()) { + await stopMediaStream(stream.stream); + } + + groupCall.onGroupCallEvent + .add(GroupCallStateChange.userMediaStreamsChanged); + + if (_activeSpeaker == stream.participant && _userMediaStreams.isNotEmpty) { + _activeSpeaker = _userMediaStreams[0].participant; + groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged); + } + } + + @override + bool get e2eeEnabled => false; + + @override + CallParticipant? get activeSpeaker => _activeSpeaker; + + @override + WrappedMediaStream? get localUserMediaStream => _localUserMediaStream; + + @override + WrappedMediaStream? get localScreenshareStream => _localScreenshareStream; + + @override + List get userMediaStreams => + List.unmodifiable(_userMediaStreams); + + @override + List get screenShareStreams => + List.unmodifiable(_screenshareStreams); + + @override + Future updateMediaDeviceForCalls() async { + for (final call in _callSessions) { + await call.updateMediaDeviceForCall(); + } + } + + /// Initializes the local user media stream. + /// The media stream must be prepared before the group call enters. + /// if you allow the user to configure their camera and such ahead of time, + /// you can pass that `stream` on to this function. + /// This allows you to configure the camera before joining the call without + /// having to reopen the stream and possibly losing settings. + @override + Future initLocalStream(GroupCallSession groupCall, + {WrappedMediaStream? stream}) async { + if (groupCall.state != GroupCallState.localCallFeedUninitialized) { + throw Exception( + 'Cannot initialize local call feed in the ${groupCall.state} state.'); + } + + groupCall.setState(GroupCallState.initializingLocalCallFeed); + + WrappedMediaStream localWrappedMediaStream; + + if (stream == null) { + MediaStream stream; + + try { + stream = await _getUserMedia(groupCall, CallType.kVideo); + } catch (error) { + groupCall.setState(GroupCallState.localCallFeedUninitialized); + rethrow; + } + + localWrappedMediaStream = WrappedMediaStream( + stream: stream, + participant: groupCall.localParticipant!, + room: groupCall.room, + client: groupCall.client, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isGroupCall: true, + voip: groupCall.voip, + ); + } else { + localWrappedMediaStream = stream; + } + + _localUserMediaStream = localWrappedMediaStream; + await _addUserMediaStream(groupCall, localWrappedMediaStream); + + groupCall.setState(GroupCallState.localCallFeedInitialized); + + _activeSpeaker = null; + + return localWrappedMediaStream; + } + + @override + Future setDeviceMuted( + GroupCallSession groupCall, bool muted, MediaInputKind kind) async { + if (!await hasMediaDevice(groupCall.voip.delegate, kind)) { + return; + } + + if (localUserMediaStream != null) { + switch (kind) { + case MediaInputKind.audioinput: + localUserMediaStream!.setAudioMuted(muted); + setTracksEnabled( + localUserMediaStream!.stream!.getAudioTracks(), !muted); + for (final call in _callSessions) { + await call.setMicrophoneMuted(muted); + } + break; + case MediaInputKind.videoinput: + localUserMediaStream!.setVideoMuted(muted); + setTracksEnabled( + localUserMediaStream!.stream!.getVideoTracks(), !muted); + for (final call in _callSessions) { + await call.setLocalVideoMuted(muted); + } + break; + default: + } + } + + groupCall.onGroupCallEvent.add(GroupCallStateChange.localMuteStateChanged); + return; + } + + Future _onIncomingCall( + GroupCallSession groupCall, CallSession newCall) async { + // The incoming calls may be for another room, which we will ignore. + if (newCall.room.id != groupCall.room.id) { + return; + } + + if (newCall.state != CallState.kRinging) { + Logs().w('Incoming call no longer in ringing state. Ignoring.'); + return; + } + + if (newCall.groupCallId == null || + newCall.groupCallId != groupCall.groupCallId) { + Logs().v( + 'Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call'); + await newCall.reject(); + return; + } + + final existingCall = _getCallForParticipant( + groupCall, + CallParticipant( + groupCall.voip, + userId: newCall.remoteUserId!, + deviceId: newCall.remoteDeviceId, + ), + ); + + if (existingCall != null && existingCall.callId == newCall.callId) { + return; + } + + Logs().v( + 'GroupCallSession: incoming call from: ${newCall.remoteUserId}${newCall.remoteDeviceId}${newCall.remotePartyId}'); + + // Check if the user calling has an existing call and use this call instead. + if (existingCall != null) { + await _replaceCall(groupCall, existingCall, newCall); + } else { + await _addCall(groupCall, newCall); + } + + await newCall.answerWithStreams(_getLocalStreams()); + } + + @override + Future setScreensharingEnabled( + GroupCallSession groupCall, + bool enabled, + String desktopCapturerSourceId, + ) async { + if (enabled == (localScreenshareStream != null)) { + return; + } + + if (enabled) { + try { + Logs().v('Asking for screensharing permissions...'); + final stream = await _getDisplayMedia(groupCall); + for (final track in stream.getTracks()) { + // screen sharing should only have 1 video track anyway, so this only + // fires once + track.onEnded = () async { + await setScreensharingEnabled(groupCall, false, ''); + }; + } + Logs().v( + 'Screensharing permissions granted. Setting screensharing enabled on all calls'); + _localScreenshareStream = WrappedMediaStream( + stream: stream, + participant: groupCall.localParticipant!, + room: groupCall.room, + client: groupCall.client, + purpose: SDPStreamMetadataPurpose.Screenshare, + audioMuted: stream.getAudioTracks().isEmpty, + videoMuted: stream.getVideoTracks().isEmpty, + isGroupCall: true, + voip: groupCall.voip, + ); + + _addScreenshareStream(groupCall, localScreenshareStream!); + + groupCall.onGroupCallEvent + .add(GroupCallStateChange.localScreenshareStateChanged); + for (final call in _callSessions) { + await call.addLocalStream( + await localScreenshareStream!.stream!.clone(), + localScreenshareStream!.purpose); + } + + await groupCall.sendMemberStateEvent(); + + return; + } catch (e, s) { + Logs().e('[VOIP] Enabling screensharing error', e, s); + groupCall.onGroupCallEvent.add(GroupCallStateChange.error); + return; + } + } else { + for (final call in _callSessions) { + await call.removeLocalStream(call.localScreenSharingStream!); + } + await stopMediaStream(localScreenshareStream?.stream); + await _removeScreenshareStream(groupCall, localScreenshareStream!); + _localScreenshareStream = null; + + await groupCall.sendMemberStateEvent(); + + groupCall.onGroupCallEvent + .add(GroupCallStateChange.localMuteStateChanged); + return; + } + } + + @override + Future dispose(GroupCallSession groupCall) async { + if (localUserMediaStream != null) { + await _removeUserMediaStream(groupCall, localUserMediaStream!); + _localUserMediaStream = null; + } + + if (localScreenshareStream != null) { + await stopMediaStream(localScreenshareStream!.stream); + await _removeScreenshareStream(groupCall, localScreenshareStream!); + _localScreenshareStream = null; + } + + // removeCall removes it from `_callSessions` later. + final callsCopy = _callSessions.toList(); + + for (final call in callsCopy) { + await _removeCall(groupCall, call, CallErrorCode.userHangup); + } + + _activeSpeaker = null; + _activeSpeakerLoopTimeout?.cancel(); + await _callSubscription?.cancel(); + } + + @override + bool get isLocalVideoMuted { + if (localUserMediaStream != null) { + return localUserMediaStream!.isVideoMuted(); + } + + return true; + } + + @override + bool get isMicrophoneMuted { + if (localUserMediaStream != null) { + return localUserMediaStream!.isAudioMuted(); + } + + return true; + } + + @override + Future setupP2PCallsWithExistingMembers( + GroupCallSession groupCall) async { + for (final call in _callSessions) { + await _onIncomingCall(groupCall, call); + } + + _callSubscription = groupCall.voip.onIncomingCall.stream.listen( + (newCall) => _onIncomingCall(groupCall, newCall), + ); + + _onActiveSpeakerLoop(groupCall); + } + + @override + Future setupP2PCallWithNewMember( + GroupCallSession groupCall, + CallParticipant rp, + CallMembership mem, + ) async { + final existingCall = _getCallForParticipant(groupCall, rp); + if (existingCall != null) { + if (existingCall.remoteSessionId != mem.membershipId) { + await existingCall.hangup(reason: CallErrorCode.unknownError); + } else { + Logs().e( + '[VOIP] onMemberStateChanged Not updating _participants list, already have a ongoing call with ${rp.id}'); + return; + } + } + + // Only initiate a call with a participant who has a id that is lexicographically + // less than your own. Otherwise, that user will call you. + if (groupCall.localParticipant!.id.compareTo(rp.id) > 0) { + Logs().i('[VOIP] Waiting for ${rp.id} to send call invite.'); + return; + } + + final opts = CallOptions( + callId: genCallID(), + room: groupCall.room, + voip: groupCall.voip, + dir: CallDirection.kOutgoing, + localPartyId: groupCall.voip.currentSessionId, + groupCallId: groupCall.groupCallId, + type: CallType.kVideo, + iceServers: await groupCall.voip.getIceServers(), + ); + final newCall = groupCall.voip.createNewCall(opts); + + /// both invitee userId and deviceId are set here because there can be + /// multiple devices from same user in a call, so we specifiy who the + /// invite is for + /// + /// MOVE TO CREATENEWCALL? + newCall.remoteUserId = mem.userId; + newCall.remoteDeviceId = mem.deviceId; + // party id set to when answered + newCall.remoteSessionId = mem.membershipId; + + await newCall.placeCallWithStreams(_getLocalStreams(), + requestScreenSharing: mem.feeds?.any((element) => + element['purpose'] == SDPStreamMetadataPurpose.Screenshare) ?? + false); + + await _addCall(groupCall, newCall); + } + + @override + List>? getCurrentFeeds() { + return _getLocalStreams() + .map((feed) => ({ + 'purpose': feed.purpose, + })) + .toList(); + } + + @override + bool operator ==(Object other) => + identical(this, other) || other is MeshBackend && type == other.type; + @override + int get hashCode => type.hashCode; + + /// get everything is livekit specific mesh calls shouldn't be affected by these + @override + Future onCallEncryption(GroupCallSession groupCall, String userId, + String deviceId, Map content) async { + return; + } + + @override + Future onCallEncryptionKeyRequest(GroupCallSession groupCall, + String userId, String deviceId, Map content) async { + return; + } + + @override + Future onLeftParticipant( + GroupCallSession groupCall, List anyLeft) async { + return; + } + + @override + Future onNewParticipant( + GroupCallSession groupCall, List anyJoined) async { + return; + } + + @override + Future requestEncrytionKey(GroupCallSession groupCall, + List remoteParticipants) async { + return; + } +} diff --git a/lib/src/voip/call.dart b/lib/src/voip/call_session.dart similarity index 70% rename from lib/src/voip/call.dart rename to lib/src/voip/call_session.dart index 97cc738a..d16e420e 100644 --- a/lib/src/voip/call.dart +++ b/lib/src/voip/call_session.dart @@ -25,288 +25,16 @@ import 'package:webrtc_interface/webrtc_interface.dart'; import 'package:matrix/matrix.dart'; import 'package:matrix/src/utils/cached_stream_controller.dart'; +import 'package:matrix/src/voip/models/call_options.dart'; +import 'package:matrix/src/voip/models/voip_id.dart'; +import 'package:matrix/src/voip/utils/stream_helper.dart'; import 'package:matrix/src/voip/utils/user_media_constraints.dart'; -/// https://github.com/matrix-org/matrix-doc/pull/2746 -/// version 1 -const String voipProtoVersion = '1'; - -class CallTimeouts { - /// The default life time for call events, in millisecond. - static const defaultCallEventLifetime = Duration(seconds: 10); - - /// The length of time a call can be ringing for. - static const callInviteLifetime = Duration(seconds: 60); - - /// The delay for ice gathering. - static const iceGatheringDelay = Duration(milliseconds: 200); - - /// Delay before createOffer. - static const delayBeforeOffer = Duration(milliseconds: 100); -} - -extension RTCIceCandidateExt on RTCIceCandidate { - bool get isValid => - sdpMLineIndex != null && - sdpMid != null && - candidate != null && - candidate!.isNotEmpty; -} - -/// Wrapped MediaStream, used to adapt Widget to display -class WrappedMediaStream { - MediaStream? stream; - final String userId; - final Room room; - - /// Current stream type, usermedia or screen-sharing - String purpose; - bool audioMuted; - bool videoMuted; - final Client client; - VideoRenderer renderer; - final bool isWeb; - final bool isGroupCall; - final RTCPeerConnection? pc; - - /// for debug - String get title => '$displayName:$purpose:a[$audioMuted]:v[$videoMuted]'; - bool stopped = false; - - final CachedStreamController onMuteStateChanged = - CachedStreamController(); - - void Function(MediaStream stream)? onNewStream; - - WrappedMediaStream( - {this.stream, - this.pc, - required this.renderer, - required this.room, - required this.userId, - required this.purpose, - required this.client, - required this.audioMuted, - required this.videoMuted, - required this.isWeb, - required this.isGroupCall}); - - /// Initialize the video renderer - Future initialize() async { - await renderer.initialize(); - renderer.srcObject = stream; - renderer.onResize = () { - Logs().i( - 'onResize [${stream!.id.substring(0, 8)}] ${renderer.videoWidth} x ${renderer.videoHeight}'); - }; - } - - Future dispose() async { - renderer.srcObject = null; - - /// libwebrtc does not provide a way to clone MediaStreams. So stopping the - /// local stream here would break calls with all other participants if anyone - /// leaves. The local stream is manually disposed when user leaves. On web - /// streams are actually cloned. - if (!isGroupCall || isWeb) { - await stopMediaStream(stream); - } - - stream = null; - await renderer.dispose(); - } - - Future disposeRenderer() async { - renderer.srcObject = null; - await renderer.dispose(); - } - - Uri? get avatarUrl => getUser().avatarUrl; - - String get avatarName => - getUser().calcDisplayname(mxidLocalPartFallback: false); - - String? get displayName => getUser().displayName; - - User getUser() { - return room.unsafeGetUserFromMemoryOrFallback(userId); - } - - bool isLocal() { - return userId == client.userID; - } - - bool isAudioMuted() { - return (stream != null && stream!.getAudioTracks().isEmpty) || audioMuted; - } - - bool isVideoMuted() { - return (stream != null && stream!.getVideoTracks().isEmpty) || videoMuted; - } - - void setNewStream(MediaStream newStream) { - stream = newStream; - renderer.srcObject = stream; - if (onNewStream != null) { - onNewStream?.call(stream!); - } - } - - void setAudioMuted(bool muted) { - audioMuted = muted; - onMuteStateChanged.add(this); - } - - void setVideoMuted(bool muted) { - videoMuted = muted; - onMuteStateChanged.add(this); - } -} - -// Call state -enum CallState { - /// The call is inilalized but not yet started - kFledgling, - - /// The first time an invite is sent, the local has createdOffer - kInviteSent, - - /// getUserMedia or getDisplayMedia has been called, - /// but MediaStream has not yet been returned - kWaitLocalMedia, - - /// The local has createdOffer - kCreateOffer, - - /// Received a remote offer message and created a local Answer - kCreateAnswer, - - /// Answer sdp is set, but ice is not connected - kConnecting, - - /// WebRTC media stream is connected - kConnected, - - /// The call was received, but no processing has been done yet. - kRinging, - - /// End of call - kEnded, -} - -class CallErrorCode { - /// The user chose to end the call - static String UserHangup = 'user_hangup'; - - /// An error code when the local client failed to create an offer. - static String LocalOfferFailed = 'local_offer_failed'; - - /// An error code when there is no local mic/camera to use. This may be because - /// the hardware isn't plugged in, or the user has explicitly denied access. - static String NoUserMedia = 'no_user_media'; - - /// Error code used when a call event failed to send - /// because unknown devices were present in the room - static String UnknownDevices = 'unknown_devices'; - - /// Error code used when we fail to send the invite - /// for some reason other than there being unknown devices - static String SendInvite = 'send_invite'; - - /// An answer could not be created - - static String CreateAnswer = 'create_answer'; - - /// Error code used when we fail to send the answer - /// for some reason other than there being unknown devices - - static String SendAnswer = 'send_answer'; - - /// The session description from the other side could not be set - static String SetRemoteDescription = 'set_remote_description'; - - /// The session description from this side could not be set - static String SetLocalDescription = 'set_local_description'; - - /// A different device answered the call - static String AnsweredElsewhere = 'answered_elsewhere'; - - /// No media connection could be established to the other party - static String IceFailed = 'ice_failed'; - - /// The invite timed out whilst waiting for an answer - static String InviteTimeout = 'invite_timeout'; - - /// The call was replaced by another call - static String Replaced = 'replaced'; - - /// Signalling for the call could not be sent (other than the initial invite) - static String SignallingFailed = 'signalling_timeout'; - - /// The remote party is busy - static String UserBusy = 'user_busy'; - - /// We transferred the call off to somewhere else - static String Transfered = 'transferred'; -} - -class CallError extends Error { - final String code; - final String msg; - final dynamic err; - CallError(this.code, this.msg, this.err); - - @override - String toString() { - return '[$code] $msg, err: ${err.toString()}'; - } -} - -enum CallEvent { - /// The call was hangup by the local|remote user. - kHangup, - - /// The call state has changed - kState, - - /// The call got some error. - kError, - - /// Call transfer - kReplaced, - - /// The value of isLocalOnHold() has changed - kLocalHoldUnhold, - - /// The value of isRemoteOnHold() has changed - kRemoteHoldUnhold, - - /// Feeds have changed - kFeedsChanged, - - /// For sip calls. support in the future. - kAssertedIdentityChanged, -} - -enum CallType { kVoice, kVideo } - -enum CallDirection { kIncoming, kOutgoing } - -enum CallParty { kLocal, kRemote } - -/// Initialization parameters of the call session. -class CallOptions { - late String callId; - String? groupCallId; - late CallType type; - late CallDirection dir; - late String localPartyId; - late VoIP voip; - late Room room; - late List> iceServers; -} - -/// A call session object +/// Parses incoming matrix events to the apropriate webrtc layer underneath using +/// a `WebRTCDelegate`. This class is also responsible for sending any outgoing +/// matrix events if required (f.ex m.call.answer). +/// +/// Handles p2p calls as well individual mesh group call peer connections. class CallSession { CallSession(this.opts); CallOptions opts; @@ -316,42 +44,69 @@ class CallSession { String? get groupCallId => opts.groupCallId; String get callId => opts.callId; String get localPartyId => opts.localPartyId; - @Deprecated('Use room.getLocalizedDisplayname() instead') - String? get displayName => room.displayname; + CallDirection get direction => opts.dir; - CallState state = CallState.kFledgling; + + CallState get state => _state; + CallState _state = CallState.kFledgling; + bool get isOutgoing => direction == CallDirection.kOutgoing; + bool get isRinging => state == CallState.kRinging; + RTCPeerConnection? pc; - List remoteCandidates = []; - List localCandidates = []; - late AssertedIdentity remoteAssertedIdentity; + + final _remoteCandidates = []; + final _localCandidates = []; + + AssertedIdentity? get remoteAssertedIdentity => _remoteAssertedIdentity; + AssertedIdentity? _remoteAssertedIdentity; + bool get callHasEnded => state == CallState.kEnded; - bool iceGatheringFinished = false; - bool inviteOrAnswerSent = false; - bool localHold = false; - bool remoteOnHold = false; + + bool _iceGatheringFinished = false; + + bool _inviteOrAnswerSent = false; + + bool get localHold => _localHold; + bool _localHold = false; + + bool get remoteOnHold => _remoteOnHold; + bool _remoteOnHold = false; + bool _answeredByUs = false; - bool speakerOn = false; - bool makingOffer = false; - bool ignoreOffer = false; - String facingMode = 'user'; + + bool _speakerOn = false; + + bool _makingOffer = false; + + bool _ignoreOffer = false; + bool get answeredByUs => _answeredByUs; + Client get client => opts.room.client; - String? remotePartyId; - String? opponentDeviceId; - String? opponentSessionId; - String? invitee; - User? remoteUser; - late CallParty hangupParty; - String? hangupReason; - late CallError lastError; - CallSession? successor; - bool waitForLocalAVStream = false; - int toDeviceSeq = 0; - int candidateSendTries = 0; + + /// The local participant in the call, with id userId + deviceId + CallParticipant? get localParticipant => voip.localParticipant; + + /// The ID of the user being called. If omitted, any user in the room can answer. + String? remoteUserId; + + User? get remoteUser => remoteUserId != null + ? room.unsafeGetUserFromMemoryOrFallback(remoteUserId!) + : null; + + /// The ID of the device being called. If omitted, any device for the remoteUserId in the room can answer. + String? remoteDeviceId; + String? remoteSessionId; // same + String? remotePartyId; // random string + + CallErrorCode? hangupReason; + CallSession? _successor; + int _toDeviceSeq = 0; + int _candidateSendTries = 0; bool get isGroupCall => groupCallId != null; - bool missedCall = true; + bool _missedCall = true; final CachedStreamController onCallStreamsChanged = CachedStreamController(); @@ -365,7 +120,7 @@ class CallSession { final CachedStreamController onCallStateChanged = CachedStreamController(); - final CachedStreamController onCallEventChanged = + final CachedStreamController onCallEventChanged = CachedStreamController(); final CachedStreamController onStreamAdd = @@ -374,14 +129,21 @@ class CallSession { final CachedStreamController onStreamRemoved = CachedStreamController(); - SDPStreamMetadata? remoteSDPStreamMetadata; - List usermediaSenders = []; - List screensharingSenders = []; - List streams = []; + SDPStreamMetadata? _remoteSDPStreamMetadata; + final List _usermediaSenders = []; + final List _screensharingSenders = []; + final List _streams = []; + List get getLocalStreams => - streams.where((element) => element.isLocal()).toList(); + _streams.where((element) => element.isLocal()).toList(); List get getRemoteStreams => - streams.where((element) => !element.isLocal()).toList(); + _streams.where((element) => !element.isLocal()).toList(); + + bool get isLocalVideoMuted => localUserMediaStream?.isVideoMuted() ?? false; + + bool get isMicrophoneMuted => localUserMediaStream?.isAudioMuted() ?? false; + + bool get screensharingEnabled => localScreenSharingStream != null; WrappedMediaStream? get localUserMediaStream { final stream = getLocalStreams.where( @@ -433,8 +195,8 @@ class CallSession { null; } - Timer? inviteTimer; - Timer? ringingTimer; + Timer? _inviteTimer; + Timer? _ringingTimer; // outgoing call Future initOutboundCall(CallType type) async { @@ -454,16 +216,17 @@ class CallSession { final prevCallId = voip.incomingCallRoomId[room.id]; if (prevCallId != null) { // This is probably an outbound call, but we already have a incoming invite, so let's terminate it. - final prevCall = voip.calls[prevCallId]; + final prevCall = + voip.calls[VoipId(roomId: room.id, callId: prevCallId)]; if (prevCall != null) { - if (prevCall.inviteOrAnswerSent) { + if (prevCall._inviteOrAnswerSent) { Logs().d('[glare] invite or answer sent, lex compare now'); if (callId.compareTo(prevCall.callId) > 0) { Logs().d( '[glare] new call $callId needs to be canceled because the older one ${prevCall.callId} has a smaller lex'); - await hangup(); - voip.currentCID = prevCall.callId; - return; + await hangup(reason: CallErrorCode.unknownError); + voip.currentCID = + VoipId(roomId: room.id, callId: prevCall.callId); } else { Logs().d( '[glare] nice, lex of newer call $callId is smaller auto accept this here'); @@ -478,7 +241,7 @@ class CallSession { } else { Logs().d( '[glare] ${prevCall.callId} was still preparing prev call, nvm now cancel it'); - await prevCall.hangup(); + await prevCall.hangup(reason: CallErrorCode.unknownError); } } } @@ -506,37 +269,36 @@ class CallSession { setCallState(CallState.kRinging); - ringingTimer = Timer(CallTimeouts.callInviteLifetime, () { + _ringingTimer = Timer(CallTimeouts.callInviteLifetime, () { if (state == CallState.kRinging) { Logs().v('[VOIP] Call invite has expired. Hanging up.'); - hangupParty = CallParty.kRemote; // effectively - fireCallEvent(CallEvent.kHangup); - hangup(reason: CallErrorCode.InviteTimeout); + + fireCallEvent(CallStateChange.kHangup); + hangup(reason: CallErrorCode.inviteTimeout); } - ringingTimer?.cancel(); - ringingTimer = null; + _ringingTimer?.cancel(); + _ringingTimer = null; }); } Future answerWithStreams(List callFeeds) async { - if (inviteOrAnswerSent) return; - Logs().d('nswering call $callId'); + if (_inviteOrAnswerSent) return; + Logs().d('answering call $callId'); await gotCallFeedsForAnswer(callFeeds); } Future replacedBy(CallSession newCall) async { if (state == CallState.kWaitLocalMedia) { Logs().v('Telling new call to wait for local media'); - newCall.waitForLocalAVStream = true; } else if (state == CallState.kCreateOffer || state == CallState.kInviteSent) { Logs().v('Handing local stream to new call'); await newCall.gotCallFeedsForAnswer(getLocalStreams); } - successor = newCall; + _successor = newCall; onCallReplaced.add(newCall); // ignore: unawaited_futures - hangup(reason: CallErrorCode.Replaced); + hangup(reason: CallErrorCode.replaced); } Future sendAnswer(RTCSessionDescription answer) async { @@ -559,8 +321,6 @@ class CallSession { Future gotCallFeedsForAnswer(List callFeeds) async { if (state == CallState.kEnded) return; - waitForLocalAVStream = false; - for (final element in callFeeds) { await addLocalStream(await element.stream!.clone(), element.purpose); } @@ -568,22 +328,25 @@ class CallSession { await answer(); } - Future placeCallWithStreams(List callFeeds, - [bool requestScreenshareFeed = false]) async { - opts.dir = CallDirection.kOutgoing; - - voip.calls[callId] = this; - + Future placeCallWithStreams( + List callFeeds, { + bool requestScreenSharing = false, + }) async { // create the peer connection now so it can be gathering candidates while we get user // media (assuming a candidate pool size is configured) await _preparePeerConnection(); - await gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); + await gotCallFeedsForInvite( + callFeeds, + requestScreenSharing: requestScreenSharing, + ); } - Future gotCallFeedsForInvite(List callFeeds, - [bool requestScreenshareFeed = false]) async { - if (successor != null) { - await successor!.gotCallFeedsForAnswer(callFeeds); + Future gotCallFeedsForInvite( + List callFeeds, { + bool requestScreenSharing = false, + }) async { + if (_successor != null) { + await _successor!.gotCallFeedsForAnswer(callFeeds); return; } if (state == CallState.kEnded) { @@ -595,7 +358,7 @@ class CallSession { await addLocalStream(await element.stream!.clone(), element.purpose); } - if (requestScreenshareFeed) { + if (requestScreenSharing) { await pc!.addTransceiver( kind: RTCRtpMediaType.RTCRtpMediaTypeVideo, init: @@ -617,13 +380,15 @@ class CallSession { if (direction == CallDirection.kOutgoing) { setCallState(CallState.kConnecting); await pc!.setRemoteDescription(answer); - for (final candidate in remoteCandidates) { + for (final candidate in _remoteCandidates) { await pc!.addCandidate(candidate); } } - - /// Send select_answer event. - await sendSelectCallAnswer(opts.room, callId, localPartyId, remotePartyId!); + if (remotePartyId != null) { + /// Send select_answer event. + await sendSelectCallAnswer( + opts.room, callId, localPartyId, remotePartyId!); + } } Future onNegotiateReceived( @@ -633,11 +398,11 @@ class CallSession { // Here we follow the perfect negotiation logic from // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation final offerCollision = ((description.type == 'offer') && - (makingOffer || + (_makingOffer || pc!.signalingState != RTCSignalingState.RTCSignalingStateStable)); - ignoreOffer = !polite && offerCollision; - if (ignoreOffer) { + _ignoreOffer = !polite && offerCollision; + if (_ignoreOffer) { Logs().i('Ignoring colliding negotiate event because we\'re impolite'); return; } @@ -655,8 +420,8 @@ class CallSession { try { answer = await pc!.createAnswer({}); } catch (e) { - await terminate(CallParty.kLocal, CallErrorCode.CreateAnswer, true); - return; + await terminate(CallParty.kLocal, CallErrorCode.createAnswer, true); + rethrow; } await sendCallNegotiate( @@ -676,30 +441,27 @@ class CallSession { final newLocalOnHold = await isLocalOnHold(); if (prevLocalOnHold != newLocalOnHold) { - localHold = newLocalOnHold; - fireCallEvent(CallEvent.kLocalHoldUnhold); + _localHold = newLocalOnHold; + fireCallEvent(CallStateChange.kLocalHoldUnhold); } } - Future updateAudioDevice([MediaStreamTrack? track]) async { - final sender = usermediaSenders - .firstWhereOrNull((element) => element.track!.kind == 'audio'); - await sender?.track?.stop(); - if (track != null) { - await sender?.replaceTrack(track); - } else { - final stream = - await voip.delegate.mediaDevices.getUserMedia({'audio': true}); - final audioTrack = stream.getAudioTracks().firstOrNull; - if (audioTrack != null) { - await sender?.replaceTrack(audioTrack); - } - } + Future updateMediaDeviceForCall() async { + await updateMediaDevice( + voip.delegate, + MediaKind.audio, + _usermediaSenders, + ); + await updateMediaDevice( + voip.delegate, + MediaKind.video, + _usermediaSenders, + ); } void _updateRemoteSDPStreamMetadata(SDPStreamMetadata metadata) { - remoteSDPStreamMetadata = metadata; - remoteSDPStreamMetadata!.sdpStreamMetadatas + _remoteSDPStreamMetadata = metadata; + _remoteSDPStreamMetadata?.sdpStreamMetadatas .forEach((streamId, sdpStreamMetadata) { Logs().i( 'Stream purpose update: \nid = "$streamId", \npurpose = "${sdpStreamMetadata.purpose}", \naudio_muted = ${sdpStreamMetadata.audio_muted}, \nvideo_muted = ${sdpStreamMetadata.video_muted}'); @@ -716,14 +478,14 @@ class CallSession { } else { Logs().i('Not found purpose for remote stream $streamId, remove it?'); wpstream.stopped = true; - fireCallEvent(CallEvent.kFeedsChanged); + fireCallEvent(CallStateChange.kFeedsChanged); } } } Future onSDPStreamMetadataReceived(SDPStreamMetadata metadata) async { _updateRemoteSDPStreamMetadata(metadata); - fireCallEvent(CallEvent.kFeedsChanged); + fireCallEvent(CallStateChange.kFeedsChanged); } Future onCandidatesReceived(List candidates) async { @@ -736,25 +498,25 @@ class CallSession { if (!candidate.isValid) { Logs().w( - '[VOIP] onCandidatesReceived => skip invalid candidate $candidate'); + '[VOIP] onCandidatesReceived => skip invalid candidate ${candidate.toMap()}'); continue; } if (direction == CallDirection.kOutgoing && pc != null && await pc!.getRemoteDescription() == null) { - remoteCandidates.add(candidate); + _remoteCandidates.add(candidate); continue; } - if (pc != null && inviteOrAnswerSent && remotePartyId != null) { + if (pc != null && _inviteOrAnswerSent) { try { await pc!.addCandidate(candidate); } catch (e, s) { Logs().e('[VOIP] onCandidatesReceived => ', e, s); } } else { - remoteCandidates.add(candidate); + _remoteCandidates.add(candidate); } } @@ -768,12 +530,10 @@ class CallSession { } void onAssertedIdentityReceived(AssertedIdentity identity) { - remoteAssertedIdentity = identity; - fireCallEvent(CallEvent.kAssertedIdentityChanged); + _remoteAssertedIdentity = identity; + fireCallEvent(CallStateChange.kAssertedIdentityChanged); } - bool get screensharingEnabled => localScreenSharingStream != null; - Future setScreensharingEnabled(bool enabled) async { // Skip if there is nothing to do if (enabled && localScreenSharingStream != null) { @@ -805,14 +565,13 @@ class CallSession { await addLocalStream(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { - fireCallEvent(CallEvent.kError); - lastError = CallError(CallErrorCode.NoUserMedia, - 'Failed to get screen-sharing stream: ', err); + fireCallEvent(CallStateChange.kError); + return false; } } else { try { - for (final sender in screensharingSenders) { + for (final sender in _screensharingSenders) { await pc!.removeTrack(sender); } for (final track in localScreenSharingStream!.stream!.getTracks()) { @@ -820,7 +579,7 @@ class CallSession { } localScreenSharingStream!.stopped = true; await _removeStream(localScreenSharingStream!.stream!); - fireCallEvent(CallEvent.kFeedsChanged); + fireCallEvent(CallStateChange.kFeedsChanged); return false; } catch (e, s) { Logs().e('[VOIP] stopping screen sharing track failed', e, s); @@ -829,59 +588,60 @@ class CallSession { } } - Future addLocalStream(MediaStream stream, String purpose, - {bool addToPeerConnection = true}) async { + Future addLocalStream( + MediaStream stream, + String purpose, { + bool addToPeerConnection = true, + }) async { final existingStream = getLocalStreams.where((element) => element.purpose == purpose); if (existingStream.isNotEmpty) { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - userId: client.userID!, + participant: localParticipant!, room: opts.room, stream: stream, purpose: purpose, client: client, audioMuted: stream.getAudioTracks().isEmpty, videoMuted: stream.getVideoTracks().isEmpty, - isWeb: voip.delegate.isWeb, isGroupCall: groupCallId != null, pc: pc, + voip: voip, ); - await newStream.initialize(); - streams.add(newStream); + _streams.add(newStream); onStreamAdd.add(newStream); } if (addToPeerConnection) { if (purpose == SDPStreamMetadataPurpose.Screenshare) { - screensharingSenders.clear(); + _screensharingSenders.clear(); for (final track in stream.getTracks()) { - screensharingSenders.add(await pc!.addTrack(track, stream)); + _screensharingSenders.add(await pc!.addTrack(track, stream)); } } else if (purpose == SDPStreamMetadataPurpose.Usermedia) { - usermediaSenders.clear(); + _usermediaSenders.clear(); for (final track in stream.getTracks()) { - usermediaSenders.add(await pc!.addTrack(track, stream)); + _usermediaSenders.add(await pc!.addTrack(track, stream)); } } } if (purpose == SDPStreamMetadataPurpose.Usermedia) { - speakerOn = type == CallType.kVideo; + _speakerOn = type == CallType.kVideo; if (!voip.delegate.isWeb && stream.getAudioTracks().isNotEmpty) { final audioTrack = stream.getAudioTracks()[0]; - audioTrack.enableSpeakerphone(speakerOn); + audioTrack.enableSpeakerphone(_speakerOn); } } - fireCallEvent(CallEvent.kFeedsChanged); + fireCallEvent(CallStateChange.kFeedsChanged); } Future _addRemoteStream(MediaStream stream) async { //final userId = remoteUser.id; - final metadata = remoteSDPStreamMetadata!.sdpStreamMetadatas[stream.id]; + final metadata = _remoteSDPStreamMetadata?.sdpStreamMetadatas[stream.id]; if (metadata == null) { Logs().i( 'Ignoring stream with id ${stream.id} because we didn\'t get any metadata about it'); @@ -900,58 +660,60 @@ class CallSession { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - userId: remoteUser!.id, + participant: CallParticipant( + voip, + userId: remoteUserId!, + deviceId: remoteDeviceId, + ), room: opts.room, stream: stream, purpose: purpose, client: client, audioMuted: audioMuted, videoMuted: videoMuted, - isWeb: voip.delegate.isWeb, isGroupCall: groupCallId != null, pc: pc, + voip: voip, ); - await newStream.initialize(); - streams.add(newStream); + _streams.add(newStream); onStreamAdd.add(newStream); } - fireCallEvent(CallEvent.kFeedsChanged); + fireCallEvent(CallStateChange.kFeedsChanged); Logs().i('Pushed remote stream (id="${stream.id}", purpose=$purpose)'); } Future deleteAllStreams() async { - for (final stream in streams) { + for (final stream in _streams) { if (stream.isLocal() || groupCallId == null) { await stream.dispose(); } } - streams.clear(); - fireCallEvent(CallEvent.kFeedsChanged); + _streams.clear(); + fireCallEvent(CallStateChange.kFeedsChanged); } Future deleteFeedByStream(MediaStream stream) async { final index = - streams.indexWhere((element) => element.stream!.id == stream.id); + _streams.indexWhere((element) => element.stream!.id == stream.id); if (index == -1) { Logs().w('Didn\'t find the feed with stream id ${stream.id} to delete'); return; } - final wstream = streams.elementAt(index); + final wstream = _streams.elementAt(index); onStreamRemoved.add(wstream); await deleteStream(wstream); } Future deleteStream(WrappedMediaStream stream) async { await stream.dispose(); - streams.removeAt(streams.indexOf(stream)); - fireCallEvent(CallEvent.kFeedsChanged); + _streams.removeAt(_streams.indexOf(stream)); + fireCallEvent(CallStateChange.kFeedsChanged); } Future removeLocalStream(WrappedMediaStream callFeed) async { final senderArray = callFeed.purpose == SDPStreamMetadataPurpose.Usermedia - ? usermediaSenders - : screensharingSenders; + ? _usermediaSenders + : _screensharingSenders; for (final element in senderArray) { await pc!.removeTrack(element); @@ -968,16 +730,16 @@ class CallSession { } void setCallState(CallState newState) { - state = newState; + _state = newState; onCallStateChanged.add(newState); - fireCallEvent(CallEvent.kState); + fireCallEvent(CallStateChange.kState); } Future setLocalVideoMuted(bool muted) async { if (!muted) { final videoToSend = await hasVideoToSend(); if (!videoToSend) { - if (remoteSDPStreamMetadata == null) return; + if (_remoteSDPStreamMetadata == null) return; await insertVideoTrackToAudioOnlyStream(); } } @@ -1042,23 +804,20 @@ class CallSession { } } // for renderer to be able to show new video track - localUserMediaStream?.renderer.srcObject = stream; + localUserMediaStream?.onStreamChanged + .add(localUserMediaStream!.stream!); } } } - bool get isLocalVideoMuted => localUserMediaStream?.isVideoMuted() ?? false; - Future setMicrophoneMuted(bool muted) async { localUserMediaStream?.setAudioMuted(muted); await updateMuteStatus(); } - bool get isMicrophoneMuted => localUserMediaStream?.isAudioMuted() ?? false; - Future setRemoteOnHold(bool onHold) async { - if (isRemoteOnHold == onHold) return; - remoteOnHold = onHold; + if (remoteOnHold == onHold) return; + _remoteOnHold = onHold; final transceivers = await pc!.getTransceivers(); for (final transceiver in transceivers) { await transceiver.setDirection(onHold @@ -1066,11 +825,9 @@ class CallSession { : TransceiverDirection.SendRecv); } await updateMuteStatus(); - fireCallEvent(CallEvent.kRemoteHoldUnhold); + fireCallEvent(CallStateChange.kRemoteHoldUnhold); } - bool get isRemoteOnHold => remoteOnHold; - Future isLocalOnHold() async { if (state != CallState.kConnected) return false; var callOnHold = true; @@ -1079,8 +836,6 @@ class CallSession { final transceivers = await pc!.getTransceivers(); for (final transceiver in transceivers) { final currentDirection = await transceiver.getCurrentDirection(); - Logs() - .i('transceiver.currentDirection = ${currentDirection?.toString()}'); final trackOnHold = (currentDirection == TransceiverDirection.Inactive || currentDirection == TransceiverDirection.RecvOnly); if (!trackOnHold) { @@ -1091,7 +846,7 @@ class CallSession { } Future answer({String? txid}) async { - if (inviteOrAnswerSent) { + if (_inviteOrAnswerSent) { return; } // stop play ringtone @@ -1101,7 +856,7 @@ class CallSession { setCallState(CallState.kCreateAnswer); final answer = await pc!.createAnswer({}); - for (final candidate in remoteCandidates) { + for (final candidate in _remoteCandidates) { await pc!.addCandidate(candidate); } @@ -1140,7 +895,7 @@ class CallSession { ); Logs().v('[VOIP] answer res => $res'); - inviteOrAnswerSent = true; + _inviteOrAnswerSent = true; _answeredByUs = true; } } @@ -1148,24 +903,25 @@ class CallSession { /// Reject a call /// This used to be done by calling hangup, but is a separate method and protocol /// event as of MSC2746. - Future reject({String? reason, bool shouldEmit = true}) async { + Future reject({CallErrorCode? reason, bool shouldEmit = true}) async { + setCallState(CallState.kEnding); if (state != CallState.kRinging && state != CallState.kFledgling) { Logs().e( '[VOIP] Call must be in \'ringing|fledgling\' state to reject! (current state was: ${state.toString()}) Calling hangup instead'); - await hangup(reason: reason, shouldEmit: shouldEmit); + await hangup(reason: CallErrorCode.userHangup, shouldEmit: shouldEmit); return; } Logs().d('[VOIP] Rejecting call: $callId'); - await terminate(CallParty.kLocal, CallErrorCode.UserHangup, shouldEmit); + await terminate(CallParty.kLocal, CallErrorCode.userHangup, shouldEmit); if (shouldEmit) { - await sendCallReject(room, callId, localPartyId, reason); + await sendCallReject(room, callId, localPartyId); } } - Future hangup({String? reason, bool shouldEmit = true}) async { - await terminate( - CallParty.kLocal, reason ?? CallErrorCode.UserHangup, shouldEmit); - + Future hangup( + {required CallErrorCode reason, bool shouldEmit = true}) async { + setCallState(CallState.kEnding); + await terminate(CallParty.kLocal, reason, shouldEmit); try { final res = await sendHangupCall(room, callId, localPartyId, 'userHangup'); @@ -1183,20 +939,28 @@ class CallSession { return; } } - Logs().e('Unable to find a track to send DTMF on'); + Logs().e('[VOIP] Unable to find a track to send DTMF on'); } Future terminate( CallParty party, - String reason, + CallErrorCode reason, bool shouldEmit, ) async { - Logs().d('[VOIP] terminating call'); - inviteTimer?.cancel(); - inviteTimer = null; + if (state == CallState.kConnected) { + await hangup( + reason: CallErrorCode.userHangup, + shouldEmit: true, + ); + return; + } - ringingTimer?.cancel(); - ringingTimer = null; + Logs().d('[VOIP] terminating call'); + _inviteTimer?.cancel(); + _inviteTimer = null; + + _ringingTimer?.cancel(); + _ringingTimer = null; try { await voip.delegate.stopRingtone(); @@ -1205,7 +969,6 @@ class CallSession { Logs().d('stopping ringtone failed ', e); } - hangupParty = party; hangupReason = reason; // don't see any reason to wrap this with shouldEmit atm, @@ -1215,25 +978,26 @@ class CallSession { if (!isGroupCall) { // when a call crash and this call is already terminated the currentCId is null. // So don't return bc the hangup or reject will not proceed anymore. - if (callId != voip.currentCID && voip.currentCID != null) return; + if (voip.currentCID != null && + voip.currentCID != VoipId(roomId: room.id, callId: callId)) return; voip.currentCID = null; voip.incomingCallRoomId.removeWhere((key, value) => value == callId); } - voip.calls.remove(callId); + voip.calls.removeWhere((key, value) => key.callId == callId); await cleanUp(); if (shouldEmit) { onCallHangupNotifierForGroupCalls.add(this); await voip.delegate.handleCallEnded(this); - fireCallEvent(CallEvent.kHangup); - if ((party == CallParty.kRemote && missedCall)) { + fireCallEvent(CallStateChange.kHangup); + if ((party == CallParty.kRemote && _missedCall)) { await voip.delegate.handleMissedCall(this); } } } - Future onRejectReceived(String? reason) async { + Future onRejectReceived(CallErrorCode? reason) async { Logs().v('[VOIP] Reject received for call ID $callId'); // No need to check party_id for reject because if we'd received either // an answer or reject, we wouldn't be in state InviteSent @@ -1244,9 +1008,9 @@ class CallSession { if (shouldTerminate) { await terminate( - CallParty.kRemote, reason ?? CallErrorCode.UserHangup, true); + CallParty.kRemote, reason ?? CallErrorCode.userHangup, true); } else { - Logs().e('Call is in state: ${state.toString()}: ignoring reject'); + Logs().e('[VOIP] Call is in state: ${state.toString()}: ignoring reject'); } } @@ -1262,7 +1026,7 @@ class CallSession { } catch (err) { Logs().d('Error setting local description! ${err.toString()}'); await terminate( - CallParty.kLocal, CallErrorCode.SetLocalDescription, true); + CallParty.kLocal, CallErrorCode.setLocalDescription, true); return; } @@ -1279,24 +1043,21 @@ class CallSession { ..transferee = false; final metadata = _getLocalSDPStreamMetadata(); if (state == CallState.kCreateOffer) { - Logs().d('[glare] new invite sent about to be called'); - await sendInviteToCall( room, callId, CallTimeouts.callInviteLifetime.inMilliseconds, localPartyId, - null, offer.sdp!, capabilities: callCapabilities, metadata: metadata); // just incase we ended the call but already sent the invite // raraley happens during glares if (state == CallState.kEnded) { - await hangup(reason: CallErrorCode.Replaced); + await hangup(reason: CallErrorCode.replaced); return; } - inviteOrAnswerSent = true; + _inviteOrAnswerSent = true; if (!isGroupCall) { Logs().d('[glare] set callid because new invite sent'); @@ -1305,12 +1066,12 @@ class CallSession { setCallState(CallState.kInviteSent); - inviteTimer = Timer(CallTimeouts.callInviteLifetime, () { + _inviteTimer = Timer(CallTimeouts.callInviteLifetime, () { if (state == CallState.kInviteSent) { - hangup(reason: CallErrorCode.InviteTimeout); + hangup(reason: CallErrorCode.inviteTimeout); } - inviteTimer?.cancel(); - inviteTimer = null; + _inviteTimer?.cancel(); + _inviteTimer = null; }); } else { await sendCallNegotiate( @@ -1327,7 +1088,7 @@ class CallSession { Future onNegotiationNeeded() async { Logs().i('Negotiation is needed!'); - makingOffer = true; + _makingOffer = true; try { // The first addTrack(audio track) on iOS will trigger // onNegotiationNeeded, which causes creatOffer to only include @@ -1340,7 +1101,7 @@ class CallSession { await _getLocalOfferFailed(e); return; } finally { - makingOffer = false; + _makingOffer = false; } } @@ -1352,14 +1113,14 @@ class CallSession { pc!.onIceCandidate = (RTCIceCandidate candidate) async { if (callHasEnded) return; //Logs().v('[VOIP] onIceCandidate => ${candidate.toMap().toString()}'); - localCandidates.add(candidate); + _localCandidates.add(candidate); - if (state == CallState.kRinging || !inviteOrAnswerSent) return; + if (state == CallState.kRinging || !_inviteOrAnswerSent) return; // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) final delay = direction == CallDirection.kIncoming ? 500 : 2000; - if (candidateSendTries == 0) { + if (_candidateSendTries == 0) { Timer(Duration(milliseconds: delay), () { _sendCandidateQueue(); }); @@ -1370,15 +1131,15 @@ class CallSession { Logs().v('[VOIP] IceGatheringState => ${state.toString()}'); if (state == RTCIceGatheringState.RTCIceGatheringStateGathering) { Timer(Duration(seconds: 3), () async { - if (!iceGatheringFinished) { - iceGatheringFinished = true; + if (!_iceGatheringFinished) { + _iceGatheringFinished = true; await _sendCandidateQueue(); } }); } if (state == RTCIceGatheringState.RTCIceGatheringStateComplete) { - if (!iceGatheringFinished) { - iceGatheringFinished = true; + if (!_iceGatheringFinished) { + _iceGatheringFinished = true; await _sendCandidateQueue(); } } @@ -1386,14 +1147,14 @@ class CallSession { pc!.onIceConnectionState = (RTCIceConnectionState state) async { Logs().v('[VOIP] RTCIceConnectionState => ${state.toString()}'); if (state == RTCIceConnectionState.RTCIceConnectionStateConnected) { - localCandidates.clear(); - remoteCandidates.clear(); + _localCandidates.clear(); + _remoteCandidates.clear(); setCallState(CallState.kConnected); // fix any state/race issues we had with sdp packets and cloned streams await updateMuteStatus(); - missedCall = false; + _missedCall = false; } else if (state == RTCIceConnectionState.RTCIceConnectionStateFailed) { - await hangup(reason: CallErrorCode.IceFailed); + await hangup(reason: CallErrorCode.iceFailed); } }; } catch (e) { @@ -1403,15 +1164,15 @@ class CallSession { Future onAnsweredElsewhere() async { Logs().d('Call ID $callId answered elsewhere'); - await terminate(CallParty.kRemote, CallErrorCode.AnsweredElsewhere, true); + await terminate(CallParty.kRemote, CallErrorCode.answeredElsewhere, true); } Future cleanUp() async { try { - for (final stream in streams) { + for (final stream in _streams) { await stream.dispose(); } - streams.clear(); + _streams.clear(); } catch (e, s) { Logs().e('[VOIP] cleaning up streams failed', e, s); } @@ -1429,10 +1190,10 @@ class CallSession { Future updateMuteStatus() async { final micShouldBeMuted = (localUserMediaStream != null && localUserMediaStream!.isAudioMuted()) || - remoteOnHold; + _remoteOnHold; final vidShouldBeMuted = (localUserMediaStream != null && localUserMediaStream!.isVideoMuted()) || - remoteOnHold; + _remoteOnHold; _setTracksEnabled(localUserMediaStream?.stream?.getAudioTracks() ?? [], !micShouldBeMuted); @@ -1440,7 +1201,11 @@ class CallSession { !vidShouldBeMuted); await sendSDPStreamMetadataChanged( - room, callId, localPartyId, _getLocalSDPStreamMetadata()); + room, + callId, + localPartyId, + _getLocalSDPStreamMetadata(), + ); } void _setTracksEnabled(List tracks, bool enabled) { @@ -1454,9 +1219,10 @@ class CallSession { for (final wpstream in getLocalStreams) { if (wpstream.stream != null) { sdpStreamMetadatas[wpstream.stream!.id] = SDPStreamPurpose( - purpose: wpstream.purpose, - audio_muted: wpstream.audioMuted, - video_muted: wpstream.videoMuted); + purpose: wpstream.purpose, + audio_muted: wpstream.audioMuted, + video_muted: wpstream.videoMuted, + ); } } final metadata = SDPStreamMetadata(sdpStreamMetadatas); @@ -1467,11 +1233,11 @@ class CallSession { Future restartIce() async { Logs().v('[VOIP] iceRestart.'); // Needs restart ice on session.pc and renegotiation. - iceGatheringFinished = false; + _iceGatheringFinished = false; final desc = await pc!.createOffer(_getOfferAnswerConstraints(iceRestart: true)); await pc!.setLocalDescription(desc); - localCandidates.clear(); + _localCandidates.clear(); } Future _getUserMedia(CallType type) async { @@ -1485,8 +1251,8 @@ class CallSession { return await voip.delegate.mediaDevices.getUserMedia(mediaConstraints); } catch (e) { await _getUserMediaFailed(e); + rethrow; } - return null; } Future _getDisplayMedia() async { @@ -1529,12 +1295,12 @@ class CallSession { Future tryRemoveStopedStreams() async { final removedStreams = {}; - for (final stream in streams) { + for (final stream in _streams) { if (stream.stopped) { removedStreams[stream.stream!.id] = stream; } } - streams + _streams .removeWhere((stream) => removedStreams.containsKey(stream.stream!.id)); for (final element in removedStreams.entries) { await _removeStream(element.value.stream!); @@ -1544,15 +1310,15 @@ class CallSession { Future _removeStream(MediaStream stream) async { Logs().v('Removing feed with stream id ${stream.id}'); - final it = streams.where((element) => element.stream!.id == stream.id); + final it = _streams.where((element) => element.stream!.id == stream.id); if (it.isEmpty) { Logs().v('Didn\'t find the feed with stream id ${stream.id} to delete'); return; } final wpstream = it.first; - streams.removeWhere((element) => element.stream!.id == stream.id); + _streams.removeWhere((element) => element.stream!.id == stream.id); onStreamRemoved.add(wpstream); - fireCallEvent(CallEvent.kFeedsChanged); + fireCallEvent(CallStateChange.kFeedsChanged); await wpstream.dispose(); } @@ -1570,82 +1336,74 @@ class CallSession { long time to wait to collect all the canidates, set the timeout for collection canidates to speed up the connection. */ - final candidatesQueue = localCandidates; + final candidatesQueue = _localCandidates; try { if (candidatesQueue.isNotEmpty) { final candidates = >[]; for (final element in candidatesQueue) { candidates.add(element.toMap()); } - localCandidates = []; + _localCandidates.clear(); final res = await sendCallCandidates( opts.room, callId, localPartyId, candidates); Logs().v('[VOIP] sendCallCandidates res => $res'); } } catch (e) { Logs().v('[VOIP] sendCallCandidates e => ${e.toString()}'); - candidateSendTries++; - localCandidates = candidatesQueue; + _candidateSendTries++; + _localCandidates.clear(); + _localCandidates.addAll(candidatesQueue); - if (candidateSendTries > 5) { + if (_candidateSendTries > 5) { Logs().d( - 'Failed to send candidates on attempt $candidateSendTries Giving up on this call.'); - lastError = - CallError(CallErrorCode.SignallingFailed, 'Signalling failed', e); - await hangup(reason: CallErrorCode.SignallingFailed); + 'Failed to send candidates on attempt $_candidateSendTries Giving up on this call.'); + await hangup(reason: CallErrorCode.iceTimeout); return; } - final delay = 500 * pow(2, candidateSendTries); + final delay = 500 * pow(2, _candidateSendTries); Timer(Duration(milliseconds: delay as int), () { _sendCandidateQueue(); }); } } - void fireCallEvent(CallEvent event) { + void fireCallEvent(CallStateChange event) { onCallEventChanged.add(event); - Logs().i('CallEvent: ${event.toString()}'); + Logs().i('CallStateChange: ${event.toString()}'); switch (event) { - case CallEvent.kFeedsChanged: + case CallStateChange.kFeedsChanged: onCallStreamsChanged.add(this); break; - case CallEvent.kState: + case CallStateChange.kState: Logs().i('CallState: ${state.toString()}'); break; - case CallEvent.kError: + case CallStateChange.kError: break; - case CallEvent.kHangup: + case CallStateChange.kHangup: break; - case CallEvent.kReplaced: + case CallStateChange.kReplaced: break; - case CallEvent.kLocalHoldUnhold: + case CallStateChange.kLocalHoldUnhold: break; - case CallEvent.kRemoteHoldUnhold: + case CallStateChange.kRemoteHoldUnhold: break; - case CallEvent.kAssertedIdentityChanged: + case CallStateChange.kAssertedIdentityChanged: break; } } Future _getLocalOfferFailed(dynamic err) async { Logs().e('Failed to get local offer ${err.toString()}'); - fireCallEvent(CallEvent.kError); - lastError = CallError( - CallErrorCode.LocalOfferFailed, 'Failed to get local offer!', err); - await terminate(CallParty.kLocal, CallErrorCode.LocalOfferFailed, true); + fireCallEvent(CallStateChange.kError); + + await terminate(CallParty.kLocal, CallErrorCode.localOfferFailed, true); } Future _getUserMediaFailed(dynamic err) async { - if (state != CallState.kConnected) { - Logs().w('Failed to get user media - ending call ${err.toString()}'); - fireCallEvent(CallEvent.kError); - lastError = CallError( - CallErrorCode.NoUserMedia, - 'Couldn\'t start capturing media! Is your microphone set up and does this app have permission?', - err); - await terminate(CallParty.kLocal, CallErrorCode.NoUserMedia, true); - } + Logs().w('Failed to get user media - ending call ${err.toString()}'); + fireCallEvent(CallStateChange.kError); + await terminate(CallParty.kLocal, CallErrorCode.userMediaFailed, true); } Future onSelectAnswerReceived(String? selectedPartyId) async { @@ -1663,7 +1421,7 @@ class CallSession { Logs().w( 'Got select_answer for party ID $selectedPartyId: we are party ID $localPartyId.'); // The other party has picked somebody else's answer - await terminate(CallParty.kRemote, CallErrorCode.AnsweredElsewhere, true); + await terminate(CallParty.kRemote, CallErrorCode.answeredElsewhere, true); } } @@ -1677,8 +1435,8 @@ class CallSession { /// [invitee] The user ID of the person who is being invited. Invites without an invitee field are defined to be /// intended for any member of the room other than the sender of the event. /// [party_id] The party ID for call, Can be set to client.deviceId. - Future sendInviteToCall(Room room, String callId, int lifetime, - String party_id, String? invitee, String sdp, + Future sendInviteToCall( + Room room, String callId, int lifetime, String party_id, String sdp, {String type = 'offer', String version = voipProtoVersion, String? txid, @@ -1687,17 +1445,23 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, 'lifetime': lifetime, 'offer': {'sdp': sdp, 'type': type}, - if (invitee != null) 'invitee': invitee, + if (remoteUserId != null) + 'invitee': + remoteUserId!, // TODO: rename this to invitee_user_id? breaks spec though + if (remoteDeviceId != null) 'invitee_device_id': remoteDeviceId!, + if (remoteDeviceId != null) + 'device_id': client + .deviceID!, // Having a remoteDeviceId means you are doing to-device events, so you want to send your deviceId too if (capabilities != null) 'capabilities': capabilities.toJson(), if (metadata != null) sdpStreamMetadataKey: metadata.toJson(), }; return await _sendCallContent( room, - EventTypes.CallInvite, + isGroupCall ? EventTypes.GroupCallMemberInvite : EventTypes.CallInvite, content, txid: txid, ); @@ -1718,14 +1482,16 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, 'selected_party_id': selected_party_id, }; return await _sendCallContent( room, - EventTypes.CallSelectAnswer, + isGroupCall + ? EventTypes.GroupCallMemberSelectAnswer + : EventTypes.CallSelectAnswer, content, txid: txid, ); @@ -1735,20 +1501,18 @@ class CallSession { /// [callId] is a unique identifier for the call. /// [version] is the version of the VoIP specification this message adheres to. This specification is version 1. /// [party_id] The party ID for call, Can be set to client.deviceId. - Future sendCallReject( - Room room, String callId, String party_id, String? reason, + Future sendCallReject(Room room, String callId, String party_id, {String version = voipProtoVersion, String? txid}) async { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, - if (reason != null) 'reason': reason, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, }; return await _sendCallContent( room, - EventTypes.CallReject, + isGroupCall ? EventTypes.GroupCallMemberReject : EventTypes.CallReject, content, txid: txid, ); @@ -1769,7 +1533,7 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, 'lifetime': lifetime, 'description': {'sdp': sdp, 'type': type}, @@ -1778,7 +1542,9 @@ class CallSession { }; return await _sendCallContent( room, - EventTypes.CallNegotiate, + isGroupCall + ? EventTypes.GroupCallMemberNegotiate + : EventTypes.CallNegotiate, content, txid: txid, ); @@ -1815,13 +1581,15 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, 'candidates': candidates, }; return await _sendCallContent( room, - EventTypes.CallCandidates, + isGroupCall + ? EventTypes.GroupCallMemberCandidates + : EventTypes.CallCandidates, content, txid: txid, ); @@ -1843,7 +1611,7 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, 'answer': {'sdp': sdp, 'type': type}, if (capabilities != null) 'capabilities': capabilities.toJson(), @@ -1851,7 +1619,7 @@ class CallSession { }; return await _sendCallContent( room, - EventTypes.CallAnswer, + isGroupCall ? EventTypes.GroupCallMemberAnswer : EventTypes.CallAnswer, content, txid: txid, ); @@ -1867,13 +1635,13 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, if (hangupCause != null) 'reason': hangupCause, }; return await _sendCallContent( room, - EventTypes.CallHangup, + isGroupCall ? EventTypes.GroupCallMemberHangup : EventTypes.CallHangup, content, txid: txid, ); @@ -1899,13 +1667,15 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, sdpStreamMetadataKey: metadata.toJson(), }; return await _sendCallContent( room, - EventTypes.CallSDPStreamMetadataChangedPrefix, + isGroupCall + ? EventTypes.GroupCallMemberSDPStreamMetadataChanged + : EventTypes.CallSDPStreamMetadataChanged, content, txid: txid, ); @@ -1923,13 +1693,15 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, ...callReplaces.toJson(), }; return await _sendCallContent( room, - EventTypes.CallReplaces, + isGroupCall + ? EventTypes.GroupCallMemberReplaces + : EventTypes.CallReplaces, content, txid: txid, ); @@ -1947,13 +1719,15 @@ class CallSession { final content = { 'call_id': callId, 'party_id': party_id, - if (groupCallId != null) 'conf_id': groupCallId, + if (groupCallId != null) 'conf_id': groupCallId!, 'version': version, 'asserted_identity': assertedIdentity.toJson(), }; return await _sendCallContent( room, - EventTypes.CallAssertedIdentity, + isGroupCall + ? EventTypes.GroupCallMemberAssertedIdentity + : EventTypes.CallAssertedIdentity, content, txid: txid, ); @@ -1962,43 +1736,44 @@ class CallSession { Future _sendCallContent( Room room, String type, - Map content, { + Map content, { String? txid, }) async { + Logs().d('[VOIP] sending content type $type, with conf: $content'); txid ??= VoIP.customTxid ?? client.generateUniqueTransactionId(); final mustEncrypt = room.encrypted && client.encryptionEnabled; // opponentDeviceId is only set for a few events during group calls, - // therefore only group calls use to-device messages for some events - if (opponentDeviceId != null) { - final toDeviceSeq = this.toDeviceSeq++; + // therefore only group calls use to-device messages for call events + if (isGroupCall && remoteDeviceId != null) { + final toDeviceSeq = _toDeviceSeq++; + final Map data = { + ...content, + 'seq': toDeviceSeq, + if (remoteSessionId != null) 'dest_session_id': remoteSessionId!, + 'sender_session_id': voip.currentSessionId, + 'room_id': room.id, + }; if (mustEncrypt) { - await client.sendToDeviceEncrypted( - [ - client.userDeviceKeys[invitee ?? remoteUser!.id]! - .deviceKeys[opponentDeviceId]! - ], - type, - { - ...content, - 'device_id': client.deviceID!, - 'seq': toDeviceSeq, - 'dest_session_id': opponentSessionId, - 'sender_session_id': client.groupCallSessionId, - }); + await client.userDeviceKeysLoading; + if (client.userDeviceKeys[remoteUserId]?.deviceKeys[remoteDeviceId] != + null) { + await client.sendToDeviceEncrypted([ + client.userDeviceKeys[remoteUserId]!.deviceKeys[remoteDeviceId]! + ], type, data); + } else { + Logs().w( + '[VOIP] _sendCallContent missing device keys for $remoteUserId'); + } } else { - final data = >>{}; - data[invitee ?? remoteUser!.id] = { - opponentDeviceId!: { - ...content, - 'device_id': client.deviceID!, - 'seq': toDeviceSeq, - 'dest_session_id': opponentSessionId, - 'sender_session_id': client.groupCallSessionId, - } - }; - await client.sendToDevice(type, txid, data); + await client.sendToDevice( + type, + txid, + { + remoteUserId!: {remoteDeviceId!: data} + }, + ); } return ''; } else { diff --git a/lib/src/voip/group_call.dart b/lib/src/voip/group_call.dart deleted file mode 100644 index 167c56cc..00000000 --- a/lib/src/voip/group_call.dart +++ /dev/null @@ -1,1308 +0,0 @@ -/* - * Famedly Matrix SDK - * Copyright (C) 2021 Famedly GmbH - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General License for more details. - * - * You should have received a copy of the GNU Affero General License - * along with this program. If not, see . - */ - -import 'dart:async'; -import 'dart:core'; - -import 'package:collection/collection.dart'; -import 'package:webrtc_interface/webrtc_interface.dart'; - -import 'package:matrix/matrix.dart'; -import 'package:matrix/src/utils/cached_stream_controller.dart'; -import 'package:matrix/src/voip/utils/user_media_constraints.dart'; - -/// TODO(@duan): Need to add voice activity detection mechanism -/// const int SPEAKING_THRESHOLD = -60; // dB - -class GroupCallIntent { - static String Ring = 'm.ring'; - static String Prompt = 'm.prompt'; - static String Room = 'm.room'; -} - -class GroupCallType { - static String Video = 'm.video'; - static String Voice = 'm.voice'; -} - -class GroupCallTerminationReason { - static String CallEnded = 'call_ended'; -} - -class GroupCallEvent { - static String GroupCallStateChanged = 'group_call_state_changed'; - static String ActiveSpeakerChanged = 'active_speaker_changed'; - static String CallsChanged = 'calls_changed'; - static String UserMediaStreamsChanged = 'user_media_feeds_changed'; - static String ScreenshareStreamsChanged = 'screenshare_feeds_changed'; - static String LocalScreenshareStateChanged = - 'local_screenshare_state_changed'; - static String LocalMuteStateChanged = 'local_mute_state_changed'; - static String ParticipantsChanged = 'participants_changed'; - static String Error = 'error'; -} - -class GroupCallErrorCode { - static String NoUserMedia = 'no_user_media'; - static String UnknownDevice = 'unknown_device'; -} - -class GroupCallError extends Error { - final String code; - final String msg; - final dynamic err; - GroupCallError(this.code, this.msg, this.err); - - @override - String toString() { - return 'Group Call Error: [$code] $msg, err: ${err.toString()}'; - } -} - -abstract class ISendEventResponse { - String? event_id; -} - -class IGroupCallRoomMemberFeed { - String? purpose; - // TODO: Sources for adaptive bitrate - IGroupCallRoomMemberFeed.fromJson(Map json) { - purpose = json['purpose']; - } - Map toJson() { - final data = {}; - data['purpose'] = purpose; - return data; - } -} - -class IGroupCallRoomMemberDevice { - String? device_id; - String? session_id; - int? expires_ts; - - List feeds = []; - IGroupCallRoomMemberDevice.fromJson(Map json) { - device_id = json['device_id']; - session_id = json['session_id']; - expires_ts = json['expires_ts']; - - if (json['feeds'] != null) { - feeds = (json['feeds'] as List) - .map((feed) => IGroupCallRoomMemberFeed.fromJson(feed)) - .toList(); - } - } - - Map toJson() { - final data = {}; - data['device_id'] = device_id; - data['session_id'] = session_id; - data['expires_ts'] = expires_ts; - data['feeds'] = feeds.map((feed) => feed.toJson()).toList(); - return data; - } -} - -class IGroupCallRoomMemberCallState { - String? call_id; - List? foci; - List devices = []; - IGroupCallRoomMemberCallState.fromJson(Map json) { - call_id = json['m.call_id']; - if (json['m.foci'] != null) { - foci = (json['m.foci'] as List).cast(); - } - if (json['m.devices'] != null) { - devices = (json['m.devices'] as List) - .map((device) => IGroupCallRoomMemberDevice.fromJson(device)) - .toList(); - } - } - Map toJson() { - final data = {}; - data['m.call_id'] = call_id; - if (foci != null) { - data['m.foci'] = foci; - } - if (devices.isNotEmpty) { - data['m.devices'] = devices.map((e) => e.toJson()).toList(); - } - return data; - } -} - -class IGroupCallRoomMemberState { - List calls = []; - IGroupCallRoomMemberState.fromJson(MatrixEvent event) { - if (event.content['m.calls'] != null) { - for (final call in (event.content['m.calls'] as List)) { - calls.add(IGroupCallRoomMemberCallState.fromJson(call)); - } - } - } -} - -class GroupCallState { - static String LocalCallFeedUninitialized = 'local_call_feed_uninitialized'; - static String InitializingLocalCallFeed = 'initializing_local_call_feed'; - static String LocalCallFeedInitialized = 'local_call_feed_initialized'; - static String Entering = 'entering'; - static String Entered = 'entered'; - static String Ended = 'ended'; -} - -abstract class ICallHandlers { - Function(List feeds)? onCallFeedsChanged; - Function(CallState state, CallState oldState)? onCallStateChanged; - Function(CallSession call)? onCallHangup; - Function(CallSession newCall)? onCallReplaced; -} - -class GroupCall { - // Config - - static const updateExpireTsTimerDuration = Duration(seconds: 15); - static const expireTsBumpDuration = Duration(seconds: 45); - static const activeSpeakerInterval = Duration(seconds: 5); - - final Client client; - final VoIP voip; - final Room room; - final String intent; - final String type; - String state = GroupCallState.LocalCallFeedUninitialized; - StreamSubscription? _callSubscription; - final Map audioLevelsMap = {}; - String? activeSpeaker; // userId - WrappedMediaStream? localUserMediaStream; - WrappedMediaStream? localScreenshareStream; - String? localDesktopCapturerSourceId; - List callSessions = []; - List participants = []; - List userMediaStreams = []; - List screenshareStreams = []; - late String groupCallId; - - GroupCallError? lastError; - - Map callHandlers = {}; - - Timer? activeSpeakerLoopTimeout; - - Timer? resendMemberStateEventTimer; - - final CachedStreamController onGroupCallFeedsChanged = - CachedStreamController(); - - final CachedStreamController onGroupCallState = - CachedStreamController(); - - final CachedStreamController onGroupCallEvent = - CachedStreamController(); - - final CachedStreamController onStreamAdd = - CachedStreamController(); - - final CachedStreamController onStreamRemoved = - CachedStreamController(); - - GroupCall({ - String? groupCallId, - required this.client, - required this.voip, - required this.room, - required this.type, - required this.intent, - }) { - this.groupCallId = groupCallId ?? genCallID(); - } - - Future create() async { - voip.groupCalls[groupCallId] = this; - voip.groupCalls[room.id] = this; - - await client.setRoomStateWithKey( - room.id, - EventTypes.GroupCallPrefix, - groupCallId, - { - 'm.intent': intent, - 'm.type': type, - }, - ); - - return this; - } - - bool get terminated => - room - .getState(EventTypes.GroupCallPrefix, groupCallId) - ?.content - .containsKey('m.terminated') ?? - false; - - String get avatarName => - getUser().calcDisplayname(mxidLocalPartFallback: false); - - String? get displayName => getUser().displayName; - - User getUser() { - return room.unsafeGetUserFromMemoryOrFallback(client.userID!); - } - - Event? getMemberStateEvent(String userId) { - final event = room.getState(EventTypes.GroupCallMemberPrefix, userId); - if (event != null) { - return room.callMemberStateForIdIsExpired(event, groupCallId) - ? null - : event; - } - return null; - } - - Future> getAllMemberStateEvents() async { - final List events = []; - final roomStates = await client.getRoomState(room.id); - roomStates.sort((a, b) => a.originServerTs.compareTo(b.originServerTs)); - for (final value in roomStates) { - if (value.type == EventTypes.GroupCallMemberPrefix && - !room.callMemberStateForIdIsExpired(value, groupCallId)) { - events.add(value); - } - } - return events; - } - - void setState(String newState) { - state = newState; - onGroupCallState.add(newState); - onGroupCallEvent.add(GroupCallEvent.GroupCallStateChanged); - } - - List getLocalStreams() { - final feeds = []; - - if (localUserMediaStream != null) { - feeds.add(localUserMediaStream!); - } - - if (localScreenshareStream != null) { - feeds.add(localScreenshareStream!); - } - - return feeds; - } - - bool hasLocalParticipant() { - final userId = client.userID; - return participants.indexWhere((member) => member.id == userId) != -1; - } - - Future _getUserMedia(CallType type) async { - final mediaConstraints = { - 'audio': UserMediaConstraints.micMediaConstraints, - 'video': type == CallType.kVideo - ? UserMediaConstraints.camMediaConstraints - : false, - }; - try { - return await voip.delegate.mediaDevices.getUserMedia(mediaConstraints); - } catch (e) { - setState(GroupCallState.LocalCallFeedUninitialized); - } - return Null as MediaStream; - } - - Future _getDisplayMedia() async { - try { - return await voip.delegate.mediaDevices - .getDisplayMedia(UserMediaConstraints.screenMediaConstraints); - } catch (e, s) { - Logs().e('[VOIP] _getDisplayMedia failed because,', e, s); - } - return Null as MediaStream; - } - - /// Initializes the local user media stream. - /// The media stream must be prepared before the group call enters. - /// if you allow the user to configure their camera and such ahead of time, - /// you can pass that `stream` on to this function. - /// This allows you to configure the camera before joining the call without - /// having to reopen the stream and possibly losing settings. - Future initLocalStream( - {WrappedMediaStream? stream}) async { - if (state != GroupCallState.LocalCallFeedUninitialized) { - throw Exception('Cannot initialize local call feed in the $state state.'); - } - - setState(GroupCallState.InitializingLocalCallFeed); - - WrappedMediaStream localWrappedMediaStream; - - if (stream == null) { - MediaStream stream; - - try { - stream = await _getUserMedia( - type == GroupCallType.Video ? CallType.kVideo : CallType.kVoice); - } catch (error) { - setState(GroupCallState.LocalCallFeedUninitialized); - rethrow; - } - - final userId = client.userID; - localWrappedMediaStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - stream: stream, - userId: userId!, - room: room, - client: client, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().isEmpty, - videoMuted: stream.getVideoTracks().isEmpty, - isWeb: voip.delegate.isWeb, - isGroupCall: true, - ); - } else { - localWrappedMediaStream = stream; - } - - localUserMediaStream = localWrappedMediaStream; - await localUserMediaStream!.initialize(); - await addUserMediaStream(localWrappedMediaStream); - - setState(GroupCallState.LocalCallFeedInitialized); - - return localWrappedMediaStream; - } - - Future updateAudioDevice() async { - final stream = - await voip.delegate.mediaDevices.getUserMedia({'audio': true}); - final audioTrack = stream.getAudioTracks().first; - for (final call in callSessions) { - await call.updateAudioDevice(audioTrack); - } - } - - void updateLocalUsermediaStream(WrappedMediaStream stream) { - if (localUserMediaStream != null) { - final oldStream = localUserMediaStream!.stream; - localUserMediaStream!.setNewStream(stream.stream!); - // ignore: discarded_futures - stopMediaStream(oldStream); - } - } - - /// enter the group call. - Future enter({WrappedMediaStream? stream}) async { - if (!(state == GroupCallState.LocalCallFeedUninitialized || - state == GroupCallState.LocalCallFeedInitialized)) { - throw Exception('Cannot enter call in the $state state'); - } - - if (state == GroupCallState.LocalCallFeedUninitialized) { - await initLocalStream(stream: stream); - } - await _addParticipant( - (await room.requestUser(client.userID!, ignoreErrors: true))!); - - await sendMemberStateEvent(); - - activeSpeaker = null; - - setState(GroupCallState.Entered); - - Logs().v('Entered group call $groupCallId'); - - _callSubscription = voip.onIncomingCall.stream.listen(onIncomingCall); - - for (final call in callSessions) { - await onIncomingCall(call); - } - - // Set up participants for the members currently in the room. - // Other members will be picked up by the RoomState.members event. - - final memberStateEvents = await getAllMemberStateEvents(); - - for (final memberState in memberStateEvents) { - await onMemberStateChanged(memberState); - } - - onActiveSpeakerLoop(); - - voip.currentGroupCID = groupCallId; - - await voip.delegate.handleNewGroupCall(this); - } - - Future dispose() async { - if (localUserMediaStream != null) { - await removeUserMediaStream(localUserMediaStream!); - localUserMediaStream = null; - } - - if (localScreenshareStream != null) { - await stopMediaStream(localScreenshareStream!.stream); - await removeScreenshareStream(localScreenshareStream!); - localScreenshareStream = null; - localDesktopCapturerSourceId = null; - } - - await _removeParticipant(client.userID!); - - await removeMemberStateEvent(); - - final callsCopy = callSessions.toList(); - - for (final call in callsCopy) { - await removeCall(call, CallErrorCode.UserHangup); - } - - activeSpeaker = null; - activeSpeakerLoopTimeout?.cancel(); - await _callSubscription?.cancel(); - } - - Future leave() async { - await dispose(); - setState(GroupCallState.LocalCallFeedUninitialized); - voip.currentGroupCID = null; - await voip.delegate.handleGroupCallEnded(this); - final justLeftGroupCall = voip.groupCalls.tryGet(room.id); - // terminate group call if empty - if (justLeftGroupCall != null && - justLeftGroupCall.intent != 'm.room' && - justLeftGroupCall.participants.isEmpty && - room.canCreateGroupCall) { - await terminate(); - } else { - Logs().d( - '[VOIP] left group call but cannot terminate. participants: ${participants.length}, pl: ${room.canCreateGroupCall}'); - } - } - - /// terminate group call. - Future terminate({bool emitStateEvent = true}) async { - final existingStateEvent = - room.getState(EventTypes.GroupCallPrefix, groupCallId); - await dispose(); - participants = []; - voip.groupCalls.remove(room.id); - voip.groupCalls.remove(groupCallId); - if (emitStateEvent) { - await client.setRoomStateWithKey( - room.id, EventTypes.GroupCallPrefix, groupCallId, { - ...existingStateEvent!.content, - 'm.terminated': GroupCallTerminationReason.CallEnded, - }); - Logs().d('[VOIP] Group call $groupCallId was killed'); - } - await voip.delegate.handleGroupCallEnded(this); - setState(GroupCallState.Ended); - } - - bool get isLocalVideoMuted { - if (localUserMediaStream != null) { - return localUserMediaStream!.isVideoMuted(); - } - - return true; - } - - bool get isMicrophoneMuted { - if (localUserMediaStream != null) { - return localUserMediaStream!.isAudioMuted(); - } - - return true; - } - - Future setMicrophoneMuted(bool muted) async { - if (!await hasAudioDevice()) { - return false; - } - - if (localUserMediaStream != null) { - localUserMediaStream!.setAudioMuted(muted); - setTracksEnabled(localUserMediaStream!.stream!.getAudioTracks(), !muted); - } - - for (final call in callSessions) { - await call.setMicrophoneMuted(muted); - } - - onGroupCallEvent.add(GroupCallEvent.LocalMuteStateChanged); - return true; - } - - Future setLocalVideoMuted(bool muted) async { - if (!await hasVideoDevice()) { - return false; - } - - if (localUserMediaStream != null) { - localUserMediaStream!.setVideoMuted(muted); - setTracksEnabled(localUserMediaStream!.stream!.getVideoTracks(), !muted); - } - - for (final call in callSessions) { - await call.setLocalVideoMuted(muted); - } - - onGroupCallEvent.add(GroupCallEvent.LocalMuteStateChanged); - return true; - } - - bool get screensharingEnabled => isScreensharing(); - - Future setScreensharingEnabled( - bool enabled, - String desktopCapturerSourceId, - ) async { - if (enabled == isScreensharing()) { - return enabled; - } - - if (enabled) { - try { - Logs().v('Asking for screensharing permissions...'); - final stream = await _getDisplayMedia(); - for (final track in stream.getTracks()) { - // screen sharing should only have 1 video track anyway, so this only - // fires once - track.onEnded = () async { - await setScreensharingEnabled(false, ''); - }; - } - Logs().v( - 'Screensharing permissions granted. Setting screensharing enabled on all calls'); - localDesktopCapturerSourceId = desktopCapturerSourceId; - localScreenshareStream = WrappedMediaStream( - renderer: voip.delegate.createRenderer(), - stream: stream, - userId: client.userID!, - room: room, - client: client, - purpose: SDPStreamMetadataPurpose.Screenshare, - audioMuted: stream.getAudioTracks().isEmpty, - videoMuted: stream.getVideoTracks().isEmpty, - isWeb: voip.delegate.isWeb, - isGroupCall: true, - ); - - addScreenshareStream(localScreenshareStream!); - await localScreenshareStream!.initialize(); - - onGroupCallEvent.add(GroupCallEvent.LocalScreenshareStateChanged); - for (final call in callSessions) { - await call.addLocalStream( - await localScreenshareStream!.stream!.clone(), - localScreenshareStream!.purpose); - } - - await sendMemberStateEvent(); - - return true; - } catch (e, s) { - Logs().e('Enabling screensharing error', e, s); - lastError = GroupCallError(GroupCallErrorCode.NoUserMedia, - 'Failed to get screen-sharing stream: ', e); - onGroupCallEvent.add(GroupCallEvent.Error); - return false; - } - } else { - for (final call in callSessions) { - await call.removeLocalStream(call.localScreenSharingStream!); - } - - await stopMediaStream(localScreenshareStream?.stream); - await removeScreenshareStream(localScreenshareStream!); - localScreenshareStream = null; - localDesktopCapturerSourceId = null; - await sendMemberStateEvent(); - onGroupCallEvent.add(GroupCallEvent.LocalMuteStateChanged); - return false; - } - } - - bool isScreensharing() { - return localScreenshareStream != null; - } - - Future onIncomingCall(CallSession newCall) async { - // The incoming calls may be for another room, which we will ignore. - if (newCall.room.id != room.id) { - return; - } - - if (newCall.state != CallState.kRinging) { - Logs().w('Incoming call no longer in ringing state. Ignoring.'); - return; - } - - if (newCall.groupCallId == null || newCall.groupCallId != groupCallId) { - Logs().v( - 'Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call'); - await newCall.reject(); - return; - } - - final opponentMemberId = newCall.remoteUser!.id; - final existingCall = getCallByUserId(opponentMemberId); - - if (existingCall != null && existingCall.callId == newCall.callId) { - return; - } - - Logs().v('GroupCall: incoming call from: $opponentMemberId'); - - // Check if the user calling has an existing call and use this call instead. - if (existingCall != null) { - await replaceCall(existingCall, newCall); - } else { - await addCall(newCall); - } - - await newCall.answerWithStreams(getLocalStreams()); - } - - Future sendMemberStateEvent() async { - final deviceId = client.deviceID; - await updateMemberCallState( - IGroupCallRoomMemberCallState.fromJson( - { - 'm.call_id': groupCallId, - 'm.devices': [ - { - 'device_id': deviceId, - 'session_id': client.groupCallSessionId, - 'expires_ts': DateTime.now() - .add(expireTsBumpDuration) - .millisecondsSinceEpoch, - 'feeds': getLocalStreams() - .map((feed) => ({ - 'purpose': feed.purpose, - })) - .toList(), - // TODO: Add data channels - }, - ], - // TODO 'm.foci' - }, - ), - ); - - if (resendMemberStateEventTimer != null) { - resendMemberStateEventTimer!.cancel(); - } - resendMemberStateEventTimer = - Timer.periodic(updateExpireTsTimerDuration, ((timer) async { - Logs().d('updating member event with timer'); - return await sendMemberStateEvent(); - })); - } - - Future removeMemberStateEvent() { - if (resendMemberStateEventTimer != null) { - Logs().d('resend member event timer cancelled'); - resendMemberStateEventTimer!.cancel(); - resendMemberStateEventTimer = null; - } - return updateMemberCallState(); - } - - Future updateMemberCallState( - [IGroupCallRoomMemberCallState? memberCallState]) async { - final localUserId = client.userID; - - final currentStateEvent = getMemberStateEvent(localUserId!); - var calls = []; - - if (currentStateEvent != null) { - final memberStateEvent = - IGroupCallRoomMemberState.fromJson(currentStateEvent); - final unCheckedCalls = memberStateEvent.calls; - - // don't keep pushing stale devices every update - final validCalls = []; - for (final call in unCheckedCalls) { - final validDevices = []; - for (final device in call.devices) { - if (device.expires_ts != null && - device.expires_ts! > - DateTime.now() - // safety buffer just incase we were slow to process a - // call event, if the device is actually dead it should - // get removed pretty soon - .add(Duration(seconds: 10)) - .millisecondsSinceEpoch) { - validDevices.add(device); - } - } - if (validDevices.isNotEmpty) { - validCalls.add(call); - } - } - - calls = validCalls; - - final existingCallIndex = - calls.indexWhere((element) => groupCallId == element.call_id); - - if (existingCallIndex != -1) { - if (memberCallState != null) { - calls[existingCallIndex] = memberCallState; - } else { - calls.removeAt(existingCallIndex); - } - } else if (memberCallState != null) { - calls.add(memberCallState); - } - } else if (memberCallState != null) { - calls.add(memberCallState); - } - final content = { - 'm.calls': calls.map((e) => e.toJson()).toList(), - }; - - await client.setRoomStateWithKey( - room.id, EventTypes.GroupCallMemberPrefix, localUserId, content); - } - - Future onMemberStateChanged(MatrixEvent event) async { - // The member events may be received for another room, which we will ignore. - if (event.roomId != room.id) { - return; - } - - final user = await room.requestUser(event.stateKey!); - - if (user == null) { - return; - } - - final callsState = IGroupCallRoomMemberState.fromJson(event); - - if (callsState is List) { - Logs() - .w('Ignoring member state from ${user.id} member not in any calls.'); - await _removeParticipant(user.id); - return; - } - - // Currently we only support a single call per room. So grab the first call. - IGroupCallRoomMemberCallState? callState; - - if (callsState.calls.isNotEmpty) { - final index = callsState.calls - .indexWhere((element) => element.call_id == groupCallId); - if (index != -1) { - callState = callsState.calls[index]; - } - } - - if (callState == null) { - Logs().w( - 'Room member ${user.id} does not have a valid m.call_id set. Ignoring.'); - await _removeParticipant(user.id); - return; - } - - final callId = callState.call_id; - if (callId != null && callId != groupCallId) { - Logs().w( - 'Call id $callId does not match group call id $groupCallId, ignoring.'); - await _removeParticipant(user.id); - return; - } - - await _addParticipant(user); - - // Don't process your own member. - final localUserId = client.userID; - - if (user.id == localUserId) { - return; - } - - if (state != GroupCallState.Entered) { - return; - } - - // Only initiate a call with a user who has a userId that is lexicographically - // less than your own. Otherwise, that user will call you. - if (localUserId!.compareTo(user.id) > 0) { - Logs().i('Waiting for ${user.id} to send call invite.'); - return; - } - - final existingCall = getCallByUserId(user.id); - - if (existingCall != null) { - return; - } - - final opponentDevice = await getDeviceForMember(user.id); - - if (opponentDevice == null) { - Logs().w('No opponent device found for ${user.id}, ignoring.'); - lastError = GroupCallError( - '400', - GroupCallErrorCode.UnknownDevice, - 'Outgoing Call: No opponent device found for ${user.id}, ignoring.', - ); - onGroupCallEvent.add(GroupCallEvent.Error); - return; - } - - final opts = CallOptions() - ..callId = genCallID() - ..room = room - ..voip = voip - ..dir = CallDirection.kOutgoing - ..localPartyId = client.deviceID! - ..groupCallId = groupCallId - ..type = CallType.kVideo - ..iceServers = await voip.getIceSevers(); - - final newCall = voip.createNewCall(opts); - newCall.opponentDeviceId = opponentDevice.device_id; - newCall.opponentSessionId = opponentDevice.session_id; - newCall.remoteUser = await room.requestUser(user.id, ignoreErrors: true); - newCall.invitee = user.id; - - final requestScreenshareFeed = opponentDevice.feeds.indexWhere( - (IGroupCallRoomMemberFeed feed) => - feed.purpose == SDPStreamMetadataPurpose.Screenshare) != - -1; - - await newCall.placeCallWithStreams( - getLocalStreams(), requestScreenshareFeed); - - await addCall(newCall); - } - - Future getDeviceForMember(String userId) async { - final memberStateEvent = getMemberStateEvent(userId); - if (memberStateEvent == null) { - return null; - } - - final memberState = IGroupCallRoomMemberState.fromJson(memberStateEvent); - - final memberGroupCallState = - memberState.calls.where(((call) => call.call_id == groupCallId)); - - if (memberGroupCallState.isEmpty) { - return null; - } - - final memberDevices = memberGroupCallState.first.devices; - - if (memberDevices.isEmpty) { - return null; - } - - /// NOTE: For now we only support one device so we use the device id in - /// the first source. - return memberDevices[0]; - } - - CallSession? getCallByUserId(String userId) { - final value = callSessions.where((item) => item.remoteUser!.id == userId); - if (value.isNotEmpty) { - return value.first; - } - return null; - } - - Future addCall(CallSession call) async { - callSessions.add(call); - await initCall(call); - onGroupCallEvent.add(GroupCallEvent.CallsChanged); - } - - Future replaceCall( - CallSession existingCall, CallSession replacementCall) async { - final existingCallIndex = - callSessions.indexWhere((element) => element == existingCall); - - if (existingCallIndex == -1) { - throw Exception('Couldn\'t find call to replace'); - } - - callSessions.removeAt(existingCallIndex); - callSessions.add(replacementCall); - - await disposeCall(existingCall, CallErrorCode.Replaced); - await initCall(replacementCall); - - onGroupCallEvent.add(GroupCallEvent.CallsChanged); - } - - /// Removes a peer call from group calls. - Future removeCall(CallSession call, String hangupReason) async { - await disposeCall(call, hangupReason); - - callSessions.removeWhere((element) => call.callId == element.callId); - - onGroupCallEvent.add(GroupCallEvent.CallsChanged); - } - - /// init a peer call from group calls. - Future initCall(CallSession call) async { - final opponentMemberId = call.opponentDeviceId; - - if (opponentMemberId == null) { - throw Exception('Cannot init call without user id'); - } - - call.onCallStateChanged.stream.listen(((event) async { - await onCallStateChanged(call, event); - })); - - call.onCallReplaced.stream.listen((CallSession newCall) async { - await replaceCall(call, newCall); - }); - - call.onCallStreamsChanged.stream.listen((call) async { - await call.tryRemoveStopedStreams(); - await onStreamsChanged(call); - }); - - call.onCallHangupNotifierForGroupCalls.stream.listen((event) async { - await onCallHangup(call); - }); - - call.onStreamAdd.stream.listen((stream) { - if (!stream.isLocal()) { - onStreamAdd.add(stream); - } - }); - - call.onStreamRemoved.stream.listen((stream) { - if (!stream.isLocal()) { - onStreamRemoved.add(stream); - } - }); - } - - Future disposeCall(CallSession call, String hangupReason) async { - final opponentMemberId = call.opponentDeviceId; - - if (opponentMemberId == null) { - throw Exception('Cannot dispose call without user id'); - } - - callHandlers.remove(opponentMemberId); - - if (call.hangupReason == CallErrorCode.Replaced) { - return; - } - - if (call.state != CallState.kEnded) { - // no need to emit individual handleCallEnded on group calls - // also prevents a loop of hangup and onCallHangupNotifierForGroupCalls - await call.hangup(reason: hangupReason, shouldEmit: false); - } - - final usermediaStream = getUserMediaStreamByUserId(opponentMemberId); - - if (usermediaStream != null) { - await removeUserMediaStream(usermediaStream); - } - - final screenshareStream = getScreenshareStreamByUserId(opponentMemberId); - - if (screenshareStream != null) { - await removeScreenshareStream(screenshareStream); - } - } - - String? getCallUserId(CallSession call) { - return call.remoteUser?.id ?? call.invitee; - } - - Future onStreamsChanged(CallSession call) async { - final opponentMemberId = getCallUserId(call); - - if (opponentMemberId == null) { - throw Exception('Cannot change call streams without user id'); - } - - final currentUserMediaStream = getUserMediaStreamByUserId(opponentMemberId); - final remoteUsermediaStream = call.remoteUserMediaStream; - final remoteStreamChanged = remoteUsermediaStream != currentUserMediaStream; - - if (remoteStreamChanged) { - if (currentUserMediaStream == null && remoteUsermediaStream != null) { - await addUserMediaStream(remoteUsermediaStream); - } else if (currentUserMediaStream != null && - remoteUsermediaStream != null) { - await replaceUserMediaStream( - currentUserMediaStream, remoteUsermediaStream); - } else if (currentUserMediaStream != null && - remoteUsermediaStream == null) { - await removeUserMediaStream(currentUserMediaStream); - } - } - - final currentScreenshareStream = - getScreenshareStreamByUserId(opponentMemberId); - final remoteScreensharingStream = call.remoteScreenSharingStream; - final remoteScreenshareStreamChanged = - remoteScreensharingStream != currentScreenshareStream; - - if (remoteScreenshareStreamChanged) { - if (currentScreenshareStream == null && - remoteScreensharingStream != null) { - addScreenshareStream(remoteScreensharingStream); - } else if (currentScreenshareStream != null && - remoteScreensharingStream != null) { - await replaceScreenshareStream( - currentScreenshareStream, remoteScreensharingStream); - } else if (currentScreenshareStream != null && - remoteScreensharingStream == null) { - await removeScreenshareStream(currentScreenshareStream); - } - } - - onGroupCallFeedsChanged.add(this); - } - - Future onCallStateChanged(CallSession call, CallState state) async { - final audioMuted = localUserMediaStream?.isAudioMuted() ?? true; - if (call.localUserMediaStream != null && - call.isMicrophoneMuted != audioMuted) { - await call.setMicrophoneMuted(audioMuted); - } - - final videoMuted = localUserMediaStream?.isVideoMuted() ?? true; - - if (call.localUserMediaStream != null && - call.isLocalVideoMuted != videoMuted) { - await call.setLocalVideoMuted(videoMuted); - } - } - - Future onCallHangup(CallSession call) async { - if (call.hangupReason == CallErrorCode.Replaced) { - return; - } - await onStreamsChanged(call); - await removeCall(call, call.hangupReason!); - } - - WrappedMediaStream? getUserMediaStreamByUserId(String userId) { - final stream = userMediaStreams.where((stream) => stream.userId == userId); - if (stream.isNotEmpty) { - return stream.first; - } - return null; - } - - Future addUserMediaStream(WrappedMediaStream stream) async { - userMediaStreams.add(stream); - //callFeed.measureVolumeActivity(true); - onStreamAdd.add(stream); - onGroupCallEvent.add(GroupCallEvent.UserMediaStreamsChanged); - } - - Future replaceUserMediaStream(WrappedMediaStream existingStream, - WrappedMediaStream replacementStream) async { - final streamIndex = userMediaStreams - .indexWhere((stream) => stream.userId == existingStream.userId); - - if (streamIndex == -1) { - throw Exception('Couldn\'t find user media stream to replace'); - } - - userMediaStreams.replaceRange(streamIndex, 1, [replacementStream]); - - await existingStream.dispose(); - //replacementStream.measureVolumeActivity(true); - onGroupCallEvent.add(GroupCallEvent.UserMediaStreamsChanged); - } - - Future removeUserMediaStream(WrappedMediaStream stream) async { - final streamIndex = - userMediaStreams.indexWhere((stream) => stream.userId == stream.userId); - - if (streamIndex == -1) { - throw Exception('Couldn\'t find user media stream to remove'); - } - - userMediaStreams.removeWhere((element) => element.userId == stream.userId); - audioLevelsMap.remove(stream.userId); - onStreamRemoved.add(stream); - - if (stream.isLocal()) { - await stream.disposeRenderer(); - await stopMediaStream(stream.stream); - } - - onGroupCallEvent.add(GroupCallEvent.UserMediaStreamsChanged); - - if (activeSpeaker == stream.userId && userMediaStreams.isNotEmpty) { - activeSpeaker = userMediaStreams[0].userId; - onGroupCallEvent.add(GroupCallEvent.ActiveSpeakerChanged); - } - } - - void onActiveSpeakerLoop() async { - String? nextActiveSpeaker; - // idc about screen sharing atm. - final userMediaStreamsCopyList = List.from(userMediaStreams); - for (final stream in userMediaStreamsCopyList) { - if (stream.userId == client.userID && stream.pc == null) { - continue; - } - - final List statsReport = await stream.pc!.getStats(); - statsReport - .removeWhere((element) => !element.values.containsKey('audioLevel')); - - // https://www.w3.org/TR/webrtc-stats/#summary - final otherPartyAudioLevel = statsReport - .singleWhereOrNull((element) => - element.type == 'inbound-rtp' && - element.values['kind'] == 'audio') - ?.values['audioLevel']; - if (otherPartyAudioLevel != null) { - audioLevelsMap[stream.userId] = otherPartyAudioLevel; - } - - // https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-source - // firefox does not seem to have this though. Works on chrome and android - final ownAudioLevel = statsReport - .singleWhereOrNull((element) => - element.type == 'media-source' && - element.values['kind'] == 'audio') - ?.values['audioLevel']; - if (ownAudioLevel != null && - audioLevelsMap[client.userID] != ownAudioLevel) { - audioLevelsMap[client.userID!] = ownAudioLevel; - } - } - - double maxAudioLevel = double.negativeInfinity; - // TODO: we probably want a threshold here? - audioLevelsMap.forEach((key, value) { - if (value > maxAudioLevel) { - nextActiveSpeaker = key; - maxAudioLevel = value; - } - }); - - if (nextActiveSpeaker != null && activeSpeaker != nextActiveSpeaker) { - activeSpeaker = nextActiveSpeaker; - onGroupCallEvent.add(GroupCallEvent.ActiveSpeakerChanged); - } - activeSpeakerLoopTimeout?.cancel(); - activeSpeakerLoopTimeout = - Timer(activeSpeakerInterval, onActiveSpeakerLoop); - } - - WrappedMediaStream? getScreenshareStreamByUserId(String userId) { - final stream = - screenshareStreams.where((stream) => stream.userId == userId); - if (stream.isNotEmpty) { - return stream.first; - } - return null; - } - - void addScreenshareStream(WrappedMediaStream stream) { - screenshareStreams.add(stream); - onStreamAdd.add(stream); - onGroupCallEvent.add(GroupCallEvent.ScreenshareStreamsChanged); - } - - Future replaceScreenshareStream(WrappedMediaStream existingStream, - WrappedMediaStream replacementStream) async { - final streamIndex = screenshareStreams - .indexWhere((stream) => stream.userId == existingStream.userId); - - if (streamIndex == -1) { - throw Exception('Couldn\'t find screenshare stream to replace'); - } - - screenshareStreams.replaceRange(streamIndex, 1, [replacementStream]); - - await existingStream.dispose(); - onGroupCallEvent.add(GroupCallEvent.ScreenshareStreamsChanged); - } - - Future removeScreenshareStream(WrappedMediaStream stream) async { - final streamIndex = screenshareStreams - .indexWhere((stream) => stream.userId == stream.userId); - - if (streamIndex == -1) { - throw Exception('Couldn\'t find screenshare stream to remove'); - } - - screenshareStreams - .removeWhere((element) => element.userId == stream.userId); - - onStreamRemoved.add(stream); - - if (stream.isLocal()) { - await stream.disposeRenderer(); - await stopMediaStream(stream.stream); - } - - onGroupCallEvent.add(GroupCallEvent.ScreenshareStreamsChanged); - } - - Future _addParticipant(User user) async { - if (participants.indexWhere((m) => m.id == user.id) != -1) { - return; - } - - participants.add(user); - - onGroupCallEvent.add(GroupCallEvent.ParticipantsChanged); - - final callsCopylist = List.from(callSessions); - - for (final call in callsCopylist) { - await call.updateMuteStatus(); - } - } - - Future _removeParticipant(String userid) async { - final index = participants.indexWhere((m) => m.id == userid); - - if (index == -1) { - return; - } - - participants.removeAt(index); - - onGroupCallEvent.add(GroupCallEvent.ParticipantsChanged); - - final callsCopylist = List.from(callSessions); - - for (final call in callsCopylist) { - await call.updateMuteStatus(); - } - } -} diff --git a/lib/src/voip/group_call_session.dart b/lib/src/voip/group_call_session.dart new file mode 100644 index 00000000..3535a06f --- /dev/null +++ b/lib/src/voip/group_call_session.dart @@ -0,0 +1,272 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2021 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General License for more details. + * + * You should have received a copy of the GNU Affero General License + * along with this program. If not, see . + */ + +import 'dart:async'; +import 'dart:core'; + +import 'package:matrix/matrix.dart'; +import 'package:matrix/src/utils/cached_stream_controller.dart'; +import 'package:matrix/src/voip/models/call_membership.dart'; +import 'package:matrix/src/voip/models/voip_id.dart'; +import 'package:matrix/src/voip/utils/stream_helper.dart'; + +/// Holds methods for managing a group call. This class is also responsible for +/// holding and managing the individual `CallSession`s in a group call. +class GroupCallSession { + // Config + final Client client; + final VoIP voip; + final Room room; + + /// is a list of backend to allow passing multiple backend in the future + /// we use the first backend everywhere as of now + final CallBackend backend; + + /// something like normal calls or thirdroom + final String? application; + + /// either room scoped or user scoped calls + final String? scope; + + GroupCallState state = GroupCallState.localCallFeedUninitialized; + + CallParticipant? get localParticipant => voip.localParticipant; + + List get participants => List.unmodifiable(_participants); + final List _participants = []; + + String groupCallId; + + final CachedStreamController onGroupCallState = + CachedStreamController(); + + final CachedStreamController onGroupCallEvent = + CachedStreamController(); + + Timer? _resendMemberStateEventTimer; + + factory GroupCallSession.withAutoGenId( + Room room, + VoIP voip, + CallBackend backend, + String? application, + String? scope, + String? groupCallId, + ) { + return GroupCallSession( + client: room.client, + room: room, + voip: voip, + backend: backend, + application: application ?? 'm.call', + scope: scope ?? 'm.room', + groupCallId: groupCallId ?? genCallID(), + ); + } + + GroupCallSession({ + required this.client, + required this.room, + required this.voip, + required this.backend, + required this.groupCallId, + required this.application, + required this.scope, + }); + + String get avatarName => + _getUser().calcDisplayname(mxidLocalPartFallback: false); + + String? get displayName => _getUser().displayName; + + User _getUser() { + return room.unsafeGetUserFromMemoryOrFallback(client.userID!); + } + + void setState(GroupCallState newState) { + state = newState; + onGroupCallState.add(newState); + onGroupCallEvent.add(GroupCallStateChange.groupCallStateChanged); + } + + bool hasLocalParticipant() { + return _participants.contains(localParticipant); + } + + /// enter the group call. + Future enter({WrappedMediaStream? stream}) async { + if (!(state == GroupCallState.localCallFeedUninitialized || + state == GroupCallState.localCallFeedInitialized)) { + throw Exception('Cannot enter call in the $state state'); + } + + if (state == GroupCallState.localCallFeedUninitialized) { + await backend.initLocalStream(this, stream: stream); + } + + await sendMemberStateEvent(); + + setState(GroupCallState.entered); + + Logs().v('Entered group call $groupCallId'); + + // Set up _participants for the members currently in the call. + // Other members will be picked up by the RoomState.members event. + await onMemberStateChanged(); + + await backend.setupP2PCallsWithExistingMembers(this); + + voip.currentGroupCID = VoipId(roomId: room.id, callId: groupCallId); + + await voip.delegate.handleNewGroupCall(this); + } + + Future leave() async { + await removeMemberStateEvent(); + await backend.dispose(this); + setState(GroupCallState.localCallFeedUninitialized); + voip.currentGroupCID = null; + _participants.clear(); + voip.groupCalls.remove(VoipId(roomId: room.id, callId: groupCallId)); + await voip.delegate.handleGroupCallEnded(this); + _resendMemberStateEventTimer?.cancel(); + setState(GroupCallState.ended); + } + + Future sendMemberStateEvent() async { + await room.updateFamedlyCallMemberStateEvent( + CallMembership( + userId: client.userID!, + roomId: room.id, + callId: groupCallId, + application: application, + scope: scope, + backend: backend, + deviceId: client.deviceID!, + expiresTs: DateTime.now() + .add(CallTimeouts.expireTsBumpDuration) + .millisecondsSinceEpoch, + membershipId: voip.currentSessionId, + feeds: backend.getCurrentFeeds(), + ), + ); + + if (_resendMemberStateEventTimer != null) { + _resendMemberStateEventTimer!.cancel(); + } + _resendMemberStateEventTimer = Timer.periodic( + CallTimeouts.updateExpireTsTimerDuration, ((timer) async { + Logs().d('sendMemberStateEvent updating member event with timer'); + if (state != GroupCallState.ended || + state != GroupCallState.localCallFeedUninitialized) { + await sendMemberStateEvent(); + } else { + Logs().d( + '[VOIP] deteceted groupCall in state $state, removing state event'); + await removeMemberStateEvent(); + } + })); + } + + Future removeMemberStateEvent() { + if (_resendMemberStateEventTimer != null) { + Logs().d('resend member event timer cancelled'); + _resendMemberStateEventTimer!.cancel(); + _resendMemberStateEventTimer = null; + } + return room.removeFamedlyCallMemberEvent( + groupCallId, + client.deviceID!, + application: application, + scope: scope, + ); + } + + /// compltetely rebuilds the local _participants list + Future onMemberStateChanged() async { + if (state != GroupCallState.entered) { + Logs().d( + '[VOIP] early return onMemberStateChanged, group call state is not Entered. Actual state: ${state.toString()} '); + return; + } + + // The member events may be received for another room, which we will ignore. + final mems = + room.getCallMembershipsFromRoom().values.expand((element) => element); + final memsForCurrentGroupCall = mems.where((element) { + return element.callId == groupCallId && + !element.isExpired && + element.application == application && + element.scope == scope && + element.roomId == room.id; // sanity checks + }).toList(); + + final ignoredMems = + mems.where((element) => !memsForCurrentGroupCall.contains(element)); + + for (final mem in ignoredMems) { + Logs().w( + '[VOIP] Ignored ${mem.userId}\'s mem event ${mem.toJson()} while updating _participants list for callId: $groupCallId, expiry status: ${mem.isExpired}'); + } + + final List newP = []; + + for (final mem in memsForCurrentGroupCall) { + final rp = CallParticipant( + voip, + userId: mem.userId, + deviceId: mem.deviceId, + ); + + newP.add(rp); + + if (rp.isLocal) continue; + + if (state != GroupCallState.entered) { + Logs().w( + '[VOIP] onMemberStateChanged groupCall state is currently $state, skipping member update'); + continue; + } + + await backend.setupP2PCallWithNewMember(this, rp, mem); + } + final newPcopy = List.from(newP); + final oldPcopy = List.from(_participants); + final anyJoined = newPcopy.where((element) => !oldPcopy.contains(element)); + final anyLeft = oldPcopy.where((element) => !newPcopy.contains(element)); + + if (anyJoined.isNotEmpty || anyLeft.isNotEmpty) { + if (anyJoined.isNotEmpty) { + Logs().d('anyJoined: ${anyJoined.map((e) => e.id).toString()}'); + _participants.addAll(anyJoined); + await backend.onNewParticipant(this, anyJoined.toList()); + } + if (anyLeft.isNotEmpty) { + Logs().d('anyLeft: ${anyLeft.map((e) => e.id).toString()}'); + for (final leftp in anyLeft) { + _participants.remove(leftp); + } + await backend.onLeftParticipant(this, anyLeft.toList()); + } + + onGroupCallEvent.add(GroupCallStateChange.participantsChanged); + Logs().d( + '[VOIP] onMemberStateChanged current list: ${_participants.map((e) => e.id).toString()}'); + } + } +} diff --git a/lib/src/voip/voip_content.dart b/lib/src/voip/models/call_events.dart similarity index 95% rename from lib/src/voip/voip_content.dart rename to lib/src/voip/models/call_events.dart index c39dce7f..59783c09 100644 --- a/lib/src/voip/voip_content.dart +++ b/lib/src/voip/models/call_events.dart @@ -60,12 +60,12 @@ class CallReplaces { target_user: CallReplacesTarget.fromJson(json['target_user']), ); - Map toJson() => { - if (replacement_id != null) 'replacement_id': replacement_id, + Map toJson() => { + if (replacement_id != null) 'replacement_id': replacement_id!, if (target_user != null) 'target_user': target_user!.toJson(), - if (create_call != null) 'create_call': create_call, - if (await_call != null) 'await_call': await_call, - if (target_room != null) 'target_room': target_room, + if (create_call != null) 'create_call': create_call!, + if (await_call != null) 'await_call': await_call!, + if (target_room != null) 'target_room': target_room!, }; } diff --git a/lib/src/voip/models/call_membership.dart b/lib/src/voip/models/call_membership.dart new file mode 100644 index 00000000..77c8735b --- /dev/null +++ b/lib/src/voip/models/call_membership.dart @@ -0,0 +1,124 @@ +import 'package:matrix/matrix.dart'; + +class FamedlyCallMemberEvent { + final List memberships; + + FamedlyCallMemberEvent({required this.memberships}); + + Map toJson() { + return {'memberships': memberships.map((e) => e.toJson()).toList()}; + } + + factory FamedlyCallMemberEvent.fromJson(Event event) { + final List callMemberships = []; + final memberships = event.content.tryGetList('memberships'); + if (memberships != null && memberships.isNotEmpty) { + for (final mem in memberships) { + if (isValidMemEvent(mem)) { + final callMem = + CallMembership.fromJson(mem, event.senderId, event.room.id); + if (callMem != null) callMemberships.add(callMem); + } + } + } + return FamedlyCallMemberEvent(memberships: callMemberships); + } +} + +class CallMembership { + final String userId; + final String callId; + final String? application; + final String? scope; + final CallBackend backend; + final String deviceId; + final int expiresTs; + final String membershipId; + final List? feeds; + + final String roomId; + + CallMembership({ + required this.userId, + required this.callId, + required this.backend, + required this.deviceId, + required this.expiresTs, + required this.roomId, + required this.membershipId, + this.application = 'm.call', + this.scope = 'm.room', + this.feeds, + }); + + Map toJson() { + return { + 'call_id': callId, + 'application': application, + 'scope': scope, + 'foci_active': [backend.toJson()], + 'device_id': deviceId, + 'expires_ts': expiresTs, + 'expires': 7200000, // element compatibiltiy remove asap + 'membershipID': membershipId, // sessionId + if (feeds != null) 'feeds': feeds, + }; + } + + static CallMembership? fromJson(Map json, String userId, String roomId) { + try { + return CallMembership( + userId: userId, + roomId: roomId, + callId: json['call_id'], + application: json['application'], + scope: json['scope'], + backend: (json['foci_active'] as List) + .map((e) => CallBackend.fromJson(e)) + .first, + deviceId: json['device_id'], + expiresTs: json['expires_ts'], + membershipId: + json['membershipID'] ?? 'someone_forgot_to_set_the_membershipID', + feeds: json['feeds'], + ); + } catch (e, s) { + Logs().e('[VOIP] call membership parsing failed. $json', e, s); + return null; + } + } + + @override + bool operator ==(other) => + identical(this, other) || + other is CallMembership && + runtimeType == other.runtimeType && + userId == other.userId && + roomId == other.roomId && + callId == other.callId && + application == other.application && + scope == other.scope && + backend.type == other.backend.type && + deviceId == other.deviceId && + membershipId == other.membershipId; + + @override + int get hashCode => + userId.hashCode ^ + roomId.hashCode ^ + callId.hashCode ^ + application.hashCode ^ + scope.hashCode ^ + backend.type.hashCode ^ + deviceId.hashCode ^ + membershipId.hashCode; + + // with a buffer of 1 minute just incase we were slow to process a + // call event, if the device is actually dead it should + // get removed pretty soon + bool get isExpired => + expiresTs < + DateTime.now() + .subtract(CallTimeouts.expireTsBumpDuration) + .millisecondsSinceEpoch; +} diff --git a/lib/src/voip/models/call_options.dart b/lib/src/voip/models/call_options.dart new file mode 100644 index 00000000..a2a59edd --- /dev/null +++ b/lib/src/voip/models/call_options.dart @@ -0,0 +1,26 @@ +import 'package:matrix/matrix.dart'; + +/// Initialization parameters of the call session. +class CallOptions { + final String callId; + final CallType type; + final CallDirection dir; + + /// client.deviceID + final String localPartyId; + final VoIP voip; + final Room room; + final List> iceServers; + final String? groupCallId; + + CallOptions({ + required this.callId, + required this.type, + required this.dir, + required this.localPartyId, + required this.voip, + required this.room, + required this.iceServers, + this.groupCallId, + }); +} diff --git a/lib/src/voip/models/call_participant.dart b/lib/src/voip/models/call_participant.dart new file mode 100644 index 00000000..7ba2b490 --- /dev/null +++ b/lib/src/voip/models/call_participant.dart @@ -0,0 +1,39 @@ +import 'package:matrix/matrix.dart'; + +class CallParticipant { + final VoIP voip; + final String userId; + final String? deviceId; + + CallParticipant( + this.voip, { + required this.userId, + this.deviceId, + }); + + bool get isLocal => + userId == voip.client.userID && deviceId == voip.client.deviceID; + + String get id { + String pid = userId; + if (deviceId != null) { + pid += ':$deviceId'; + } + return pid; + } + + @override + String toString() { + return id; + } + + @override + bool operator ==(Object other) => + identical(this, other) || + other is CallParticipant && + userId == other.userId && + deviceId == other.deviceId; + + @override + int get hashCode => userId.hashCode ^ deviceId.hashCode; +} diff --git a/lib/src/voip/models/key_provider.dart b/lib/src/voip/models/key_provider.dart new file mode 100644 index 00000000..8fc12177 --- /dev/null +++ b/lib/src/voip/models/key_provider.dart @@ -0,0 +1,55 @@ +import 'dart:typed_data'; + +import 'package:matrix/matrix.dart'; + +enum E2EEKeyMode { + kNone, + kSharedKey, + kPerParticipant, +} + +abstract class EncryptionKeyProvider { + Future onSetEncryptionKey( + CallParticipant participant, Uint8List key, int index); + + Future onRatchetKey(CallParticipant participant, int index); + + Future onExportKey(CallParticipant participant, int index); +} + +class EncryptionKeyEntry { + final int index; + final String key; + EncryptionKeyEntry(this.index, this.key); + + factory EncryptionKeyEntry.fromJson(Map json) => + EncryptionKeyEntry( + json['index'] as int, + json['key'] as String, + ); + + Map toJson() => { + 'index': index, + 'key': key, + }; +} + +class EncryptionKeysEventContent { + // Get the participant info from todevice message params + final List keys; + final String callId; + EncryptionKeysEventContent(this.keys, this.callId); + + factory EncryptionKeysEventContent.fromJson(Map json) => + EncryptionKeysEventContent( + (json['keys'] as List) + .map( + (e) => EncryptionKeyEntry.fromJson(e as Map)) + .toList(), + json['call_id'] as String); + + Map toJson() => { + 'keys': keys.map((e) => e.toJson()).toList(), + 'call_id': callId, + }; +} diff --git a/lib/src/voip/models/voip_id.dart b/lib/src/voip/models/voip_id.dart new file mode 100644 index 00000000..19523b4b --- /dev/null +++ b/lib/src/voip/models/voip_id.dart @@ -0,0 +1,24 @@ +class VoipId { + final String roomId; + final String callId; + + String get id => '$roomId:$callId'; + + factory VoipId.fromId(String id) { + final int lastIndex = id.lastIndexOf(':'); + return VoipId( + roomId: id.substring(0, lastIndex), + callId: id.substring(lastIndex + 1), + ); + } + + VoipId({required this.roomId, required this.callId}); + + @override + bool operator ==(Object other) => + identical(this, other) || + other is VoipId && roomId == other.roomId && callId == other.callId; + + @override + int get hashCode => roomId.hashCode ^ callId.hashCode; +} diff --git a/lib/src/voip/models/webrtc_delegate.dart b/lib/src/voip/models/webrtc_delegate.dart new file mode 100644 index 00000000..ea6e2d1e --- /dev/null +++ b/lib/src/voip/models/webrtc_delegate.dart @@ -0,0 +1,25 @@ +import 'package:webrtc_interface/webrtc_interface.dart'; + +import 'package:matrix/matrix.dart'; + +/// Delegate WebRTC basic functionality. +abstract class WebRTCDelegate { + MediaDevices get mediaDevices; + Future createPeerConnection( + Map configuration, + [Map constraints = const {}]); + Future playRingtone(); + Future stopRingtone(); + Future handleNewCall(CallSession session); + Future handleCallEnded(CallSession session); + Future handleMissedCall(CallSession session); + Future handleNewGroupCall(GroupCallSession groupCall); + Future handleGroupCallEnded(GroupCallSession groupCall); + bool get isWeb; + + /// This should be set to false if any calls in the client are in kConnected + /// state. If another room tries to call you during a connected call this fires + /// a handleMissedCall + bool get canHandleNewCall; + EncryptionKeyProvider? get keyProvider; +} diff --git a/lib/src/voip/utils.dart b/lib/src/voip/utils.dart deleted file mode 100644 index 182843e2..00000000 --- a/lib/src/voip/utils.dart +++ /dev/null @@ -1,47 +0,0 @@ -import 'dart:async'; - -import 'package:random_string/random_string.dart'; -import 'package:webrtc_interface/webrtc_interface.dart'; - -import 'package:matrix/matrix.dart'; - -Future stopMediaStream(MediaStream? stream) async { - if (stream != null) { - for (final track in stream.getTracks()) { - try { - await track.stop(); - } catch (e, s) { - Logs().e('[VOIP] stopping track ${track.id} failed', e, s); - } - } - try { - await stream.dispose(); - } catch (e, s) { - Logs().e('[VOIP] disposing stream ${stream.id} failed', e, s); - } - } -} - -void setTracksEnabled(List tracks, bool enabled) { - for (final element in tracks) { - element.enabled = enabled; - } -} - -Future hasAudioDevice() async { - //TODO(duan): implement this, check if there is any audio device - return true; -} - -Future hasVideoDevice() async { - //TODO(duan): implement this, check if there is any video device - return true; -} - -String roomAliasFromRoomName(String roomName) { - return roomName.trim().replaceAll('-', '').toLowerCase(); -} - -String genCallID() { - return '${DateTime.now().millisecondsSinceEpoch}${randomAlphaNumeric(16)}'; -} diff --git a/lib/src/voip/conn_tester.dart b/lib/src/voip/utils/conn_tester.dart similarity index 96% rename from lib/src/voip/conn_tester.dart rename to lib/src/voip/utils/conn_tester.dart index 889334dc..d501de16 100644 --- a/lib/src/voip/conn_tester.dart +++ b/lib/src/voip/utils/conn_tester.dart @@ -12,7 +12,7 @@ class ConnectionTester { TurnServerCredentials? _turnServerCredentials; Future verifyTurnServer() async { - final iceServers = await getIceSevers(); + final iceServers = await getIceServers(); final configuration = { 'iceServers': iceServers, 'sdpSemantics': 'unified-plan', @@ -95,7 +95,7 @@ class ConnectionTester { return iterations; } - Future>> getIceSevers() async { + Future>> getIceServers() async { if (_turnServerCredentials == null) { try { _turnServerCredentials = await client.getTurnServer(); diff --git a/lib/src/voip/utils/famedly_call_extension.dart b/lib/src/voip/utils/famedly_call_extension.dart new file mode 100644 index 00000000..1a37a7c2 --- /dev/null +++ b/lib/src/voip/utils/famedly_call_extension.dart @@ -0,0 +1,167 @@ +import 'package:collection/collection.dart'; + +import 'package:matrix/matrix.dart'; +import 'package:matrix/src/voip/models/call_membership.dart'; + +extension FamedlyCallMemberEventsExtension on Room { + /// a map of every users famedly call event, holds the memberships list + /// returns sorted according to originTs (oldest to newest) + Map getFamedlyCallEvents() { + final Map mappedEvents = {}; + final famedlyCallMemberStates = + states.tryGetMap(EventTypes.GroupCallMember); + + if (famedlyCallMemberStates == null) return {}; + final sortedEvents = famedlyCallMemberStates.values + .sorted((a, b) => a.originServerTs.compareTo(b.originServerTs)); + + for (final element in sortedEvents) { + mappedEvents + .addAll({element.senderId: FamedlyCallMemberEvent.fromJson(element)}); + } + return mappedEvents; + } + + /// extracts memberships list form a famedly call event and maps it to a userid + /// returns sorted (oldest to newest) + Map> getCallMembershipsFromRoom() { + final parsedMemberEvents = getFamedlyCallEvents(); + final Map> memberships = {}; + for (final element in parsedMemberEvents.entries) { + memberships.addAll({element.key: element.value.memberships}); + } + return memberships; + } + + /// returns a list of memberships in the room for `user` + List getCallMembershipsForUser(String userId) { + final parsedMemberEvents = getCallMembershipsFromRoom(); + final mem = parsedMemberEvents.tryGet>(userId); + return mem ?? []; + } + + /// returns the user count (not sessions, yet) for the group call with id: `groupCallId`. + /// returns 0 if group call not found + int groupCallParticipantCount(String groupCallId) { + int participantCount = 0; + // userid:membership + final memberships = getCallMembershipsFromRoom(); + + memberships.forEach((key, value) { + for (final membership in value) { + if (membership.callId == groupCallId && !membership.isExpired) { + participantCount++; + } + } + }); + + return participantCount; + } + + bool get hasActiveGroupCall { + if (activeGroupCallIds.isNotEmpty) { + return true; + } + return false; + } + + /// list of active group call ids + List get activeGroupCallIds { + final Set ids = {}; + final memberships = getCallMembershipsFromRoom(); + + memberships.forEach((key, value) { + for (final mem in value) { + if (!mem.isExpired) ids.add(mem.callId); + } + }); + return ids.toList(); + } + + /// passing no `CallMembership` removes it from the state event. + Future updateFamedlyCallMemberStateEvent( + CallMembership callMembership) async { + final ownMemberships = getCallMembershipsForUser(client.userID!); + + // do not bother removing other deviceId expired events because we have no + // ownership over them + ownMemberships + .removeWhere((element) => client.deviceID! == element.deviceId); + + ownMemberships.removeWhere((e) => e == callMembership); + + ownMemberships.add(callMembership); + + final newContent = { + 'memberships': List.from(ownMemberships.map((e) => e.toJson())) + }; + + await setFamedlyCallMemberEvent(newContent); + } + + Future removeFamedlyCallMemberEvent( + String groupCallId, + String deviceId, { + String? application = 'm.call', + String? scope = 'm.room', + }) async { + final ownMemberships = getCallMembershipsForUser(client.userID!); + + ownMemberships.removeWhere((mem) => + mem.callId == groupCallId && + mem.deviceId == deviceId && + mem.application == application && + mem.scope == scope); + + final newContent = { + 'memberships': List.from(ownMemberships.map((e) => e.toJson())) + }; + await setFamedlyCallMemberEvent(newContent); + } + + Future setFamedlyCallMemberEvent(Map newContent) async { + if (groupCallsEnabledForEveryone) { + await client.setRoomStateWithKey( + id, + EventTypes.GroupCallMember, + client.userID!, + newContent, + ); + } else { + Logs().w( + '[VOIP] cannot send ${EventTypes.GroupCallMember} events in room: $id, fix your PLs'); + } + } + + /// returns a list of memberships from a famedly call matrix event + List getCallMembershipsFromEvent(MatrixEvent event) { + if (event.roomId != id) return []; + return getCallMembershipsFromEventContent( + event.content, event.senderId, event.roomId!); + } + + /// returns a list of memberships from a famedly call matrix event + List getCallMembershipsFromEventContent( + Map content, String senderId, String roomId) { + final mems = content.tryGetList('memberships'); + final callMems = []; + for (final m in mems ?? []) { + final mem = CallMembership.fromJson(m, senderId, roomId); + if (mem != null) callMems.add(mem); + } + return callMems; + } +} + +bool isValidMemEvent(Map event) { + if (event['call_id'] is String && + event['device_id'] is String && + event['expires_ts'] is num && + event['foci_active'] is List) { + return true; + } else { + Logs() + .w('[VOIP] FamedlyCallMemberEvent ignoring unclean membership $event'); + return false; + } +} diff --git a/lib/src/voip/utils/rtc_candidate_extension.dart b/lib/src/voip/utils/rtc_candidate_extension.dart new file mode 100644 index 00000000..8c62c8c8 --- /dev/null +++ b/lib/src/voip/utils/rtc_candidate_extension.dart @@ -0,0 +1,9 @@ +import 'package:webrtc_interface/webrtc_interface.dart'; + +extension RTCIceCandidateExt on RTCIceCandidate { + bool get isValid => + sdpMLineIndex != null && + sdpMid != null && + candidate != null && + candidate!.isNotEmpty; +} diff --git a/lib/src/voip/utils/stream_helper.dart b/lib/src/voip/utils/stream_helper.dart new file mode 100644 index 00000000..61da7c5f --- /dev/null +++ b/lib/src/voip/utils/stream_helper.dart @@ -0,0 +1,65 @@ +import 'package:collection/collection.dart'; +import 'package:random_string/random_string.dart'; +import 'package:webrtc_interface/webrtc_interface.dart'; + +import 'package:matrix/matrix.dart'; + +Future stopMediaStream(MediaStream? stream) async { + if (stream != null) { + for (final track in stream.getTracks()) { + try { + await track.stop(); + } catch (e, s) { + Logs().e('[VOIP] stopping track ${track.id} failed', e, s); + } + } + try { + await stream.dispose(); + } catch (e, s) { + Logs().e('[VOIP] disposing stream ${stream.id} failed', e, s); + } + } +} + +void setTracksEnabled(List tracks, bool enabled) { + for (final element in tracks) { + element.enabled = enabled; + } +} + +Future hasMediaDevice( + WebRTCDelegate delegate, MediaInputKind mediaInputKind) async { + final devices = await delegate.mediaDevices.enumerateDevices(); + return devices + .where((device) => device.kind == mediaInputKind.name) + .isNotEmpty; +} + +Future updateMediaDevice( + WebRTCDelegate delegate, + MediaKind kind, + List userRtpSenders, [ + MediaStreamTrack? track, +]) async { + final sender = userRtpSenders + .firstWhereOrNull((element) => element.track!.kind == kind.name); + await sender?.track?.stop(); + if (track != null) { + await sender?.replaceTrack(track); + } else { + final stream = await delegate.mediaDevices.getUserMedia({kind.name: true}); + MediaStreamTrack? track; + if (kind == MediaKind.audio) { + track = stream.getAudioTracks().firstOrNull; + } else if (kind == MediaKind.video) { + track = stream.getVideoTracks().firstOrNull; + } + if (track != null) { + await sender?.replaceTrack(track); + } + } +} + +String genCallID() { + return '${DateTime.now().millisecondsSinceEpoch}${randomAlphaNumeric(16)}'; +} diff --git a/lib/src/voip/utils/types.dart b/lib/src/voip/utils/types.dart new file mode 100644 index 00000000..09455726 --- /dev/null +++ b/lib/src/voip/utils/types.dart @@ -0,0 +1,187 @@ +// ignore_for_file: constant_identifier_names + +enum EncryptionKeyTypes { remote, local } + +// Call state +enum CallState { + /// The call is inilalized but not yet started + kFledgling, + + /// The first time an invite is sent, the local has createdOffer + kInviteSent, + + /// getUserMedia or getDisplayMedia has been called, + /// but MediaStream has not yet been returned + kWaitLocalMedia, + + /// The local has createdOffer + kCreateOffer, + + /// Received a remote offer message and created a local Answer + kCreateAnswer, + + /// Answer sdp is set, but ice is not connected + kConnecting, + + /// WebRTC media stream is connected + kConnected, + + /// The call was received, but no processing has been done yet. + kRinging, + + /// Ending a call + kEnding, + + /// End of call + kEnded, +} + +enum CallErrorCode { + /// The user chose to end the call + userHangup('user_hangup'), + + /// An error code when the local client failed to create an offer. + localOfferFailed('local_offer_failed'), + + /// An error code when there is no local mic/camera to use. This may be because + /// the hardware isn't plugged in, or the user has explicitly denied access. + userMediaFailed('user_media_failed'), + + /// Error code used when a call event failed to send + /// because unknown devices were present in the room + unknownDevice('unknown_device'), + + /// An answer could not be created + createAnswer('create_answer'), + + /// The session description from the other side could not be set + + setRemoteDescription('set_remote_description'), + + /// The session description from this side could not be set + setLocalDescription('set_local_description'), + + /// A different device answered the call + answeredElsewhere('answered_elsewhere'), + + /// No media connection could be established to the other party + iceFailed('ice_failed'), + + /// The invite timed out whilst waiting for an answer + inviteTimeout('invite_timeout'), + + /// The call was replaced by another call + replaced('replaced'), + + /// Signalling for the call could not be sent (other than the initial invite) + iceTimeout('ice_timeout'), + + /// The remote party is busy + userBusy('user_busy'), + + /// We transferred the call off to somewhere else + transferred('transferred'), + + /// Some other failure occurred that meant the client was unable to continue + /// the call rather than the user choosing to end it. + unknownError('unknown_error'); + + final String reason; + + const CallErrorCode(this.reason); +} + +class CallError extends Error { + final CallErrorCode code; + final String msg; + final dynamic err; + CallError(this.code, this.msg, this.err); + + @override + String toString() { + return '[$code] $msg, err: ${err.toString()}'; + } +} + +enum CallStateChange { + /// The call was hangup by the local|remote user. + kHangup, + + /// The call state has changed + kState, + + /// The call got some error. + kError, + + /// Call transfer + kReplaced, + + /// The value of isLocalOnHold() has changed + kLocalHoldUnhold, + + /// The value of isRemoteOnHold() has changed + kRemoteHoldUnhold, + + /// Feeds have changed + kFeedsChanged, + + /// For sip calls. support in the future. + kAssertedIdentityChanged, +} + +enum CallType { kVoice, kVideo } + +enum CallDirection { kIncoming, kOutgoing } + +enum CallParty { kLocal, kRemote } + +enum MediaInputKind { videoinput, audioinput } + +enum MediaKind { video, audio } + +enum GroupCallErrorCode { + /// An error code when there is no local mic/camera to use. This may be because + /// the hardware isn't plugged in, or the user has explicitly denied access. + userMediaFailed('user_media_failed'), + + /// Some other failure occurred that meant the client was unable to continue + /// the call rather than the user choosing to end it. + unknownError('unknownError'); + + final String reason; + + const GroupCallErrorCode(this.reason); +} + +class GroupCallError extends Error { + final GroupCallErrorCode code; + final String msg; + final dynamic err; + GroupCallError(this.code, this.msg, this.err); + + @override + String toString() { + return 'Group Call Error: [$code] $msg, err: ${err.toString()}'; + } +} + +enum GroupCallStateChange { + groupCallStateChanged, + activeSpeakerChanged, + callsChanged, + userMediaStreamsChanged, + screenshareStreamsChanged, + localScreenshareStateChanged, + localMuteStateChanged, + participantsChanged, + error +} + +enum GroupCallState { + localCallFeedUninitialized, + initializingLocalCallFeed, + localCallFeedInitialized, + entering, + entered, + ended +} diff --git a/lib/src/voip/utils/voip_constants.dart b/lib/src/voip/utils/voip_constants.dart new file mode 100644 index 00000000..f093902c --- /dev/null +++ b/lib/src/voip/utils/voip_constants.dart @@ -0,0 +1,62 @@ +import 'package:matrix/matrix.dart'; + +/// https://github.com/matrix-org/matrix-doc/pull/2746 +/// version 1 +const String voipProtoVersion = '1'; + +class CallTimeouts { + /// The default life time for call events, in millisecond. + static const defaultCallEventLifetime = Duration(seconds: 10); + + /// The length of time a call can be ringing for. + static const callInviteLifetime = Duration(seconds: 60); + + /// The delay for ice gathering. + static const iceGatheringDelay = Duration(milliseconds: 200); + + /// Delay before createOffer. + static const delayBeforeOffer = Duration(milliseconds: 100); + + /// How often to update the expiresTs + static const updateExpireTsTimerDuration = Duration(minutes: 2); + + /// the expiresTs bump + static const expireTsBumpDuration = Duration(minutes: 6); + + /// Update the active speaker value + static const activeSpeakerInterval = Duration(seconds: 5); + + // source: element call? + /// A delay after a member leaves before we create and publish a new key, because people + /// tend to leave calls at the same time + static const makeKeyDelay = Duration(seconds: 2); + + /// The delay between creating and sending a new key and starting to encrypt with it. This gives others + /// a chance to receive the new key to minimise the chance they don't get media they can't decrypt. + /// The total time between a member leaving and the call switching to new keys is therefore + /// makeKeyDelay + useKeyDelay + static const useKeyDelay = Duration(seconds: 4); +} + +class CallConstants { + static final callEventsRegxp = RegExp( + r'm.call.|org.matrix.call.|org.matrix.msc3401.call.|com.famedly.call.'); + + static const callEndedEventTypes = { + EventTypes.CallAnswer, + EventTypes.CallHangup, + EventTypes.CallReject, + EventTypes.CallReplaces, + }; + static const omitWhenCallEndedTypes = { + EventTypes.CallInvite, + EventTypes.CallCandidates, + EventTypes.CallNegotiate, + EventTypes.CallSDPStreamMetadataChanged, + EventTypes.CallSDPStreamMetadataChangedPrefix, + }; + + static const updateExpireTsTimerDuration = Duration(seconds: 15); + static const expireTsBumpDuration = Duration(seconds: 45); + static const activeSpeakerInterval = Duration(seconds: 5); +} diff --git a/lib/src/voip/utils/wrapped_media_stream.dart b/lib/src/voip/utils/wrapped_media_stream.dart new file mode 100644 index 00000000..e8eb0f3a --- /dev/null +++ b/lib/src/voip/utils/wrapped_media_stream.dart @@ -0,0 +1,100 @@ +import 'package:webrtc_interface/webrtc_interface.dart'; + +import 'package:matrix/matrix.dart'; +import 'package:matrix/src/utils/cached_stream_controller.dart'; +import 'package:matrix/src/voip/utils/stream_helper.dart'; + +/// Wrapped MediaStream, used to adapt Widget to display +class WrappedMediaStream { + MediaStream? stream; + final CallParticipant participant; + final Room room; + final VoIP voip; + + /// Current stream type, usermedia or screen-sharing + String purpose; + bool audioMuted; + bool videoMuted; + final Client client; + final bool isGroupCall; + final RTCPeerConnection? pc; + + /// for debug + String get title => + '${client.userID!}:${client.deviceID!} $displayName:$purpose:a[$audioMuted]:v[$videoMuted]'; + bool stopped = false; + + final CachedStreamController onMuteStateChanged = + CachedStreamController(); + + final CachedStreamController onStreamChanged = + CachedStreamController(); + + WrappedMediaStream({ + this.stream, + this.pc, + required this.room, + required this.participant, + required this.purpose, + required this.client, + required this.audioMuted, + required this.videoMuted, + required this.isGroupCall, + required this.voip, + }); + + String get id => '${stream?.id}: $title'; + + Future dispose() async { + // AOT it + const isWeb = bool.fromEnvironment('dart.library.js_util'); + + // libwebrtc does not provide a way to clone MediaStreams. So stopping the + // local stream here would break calls with all other participants if anyone + // leaves. The local stream is manually disposed when user leaves. On web + // streams are actually cloned. + if (!isGroupCall || isWeb) { + await stopMediaStream(stream); + } + + stream = null; + } + + Uri? get avatarUrl => getUser().avatarUrl; + + String get avatarName => + getUser().calcDisplayname(mxidLocalPartFallback: false); + + String? get displayName => getUser().displayName; + + User getUser() { + return room.unsafeGetUserFromMemoryOrFallback(participant.userId); + } + + bool isLocal() { + return participant == voip.localParticipant; + } + + bool isAudioMuted() { + return (stream != null && stream!.getAudioTracks().isEmpty) || audioMuted; + } + + bool isVideoMuted() { + return (stream != null && stream!.getVideoTracks().isEmpty) || videoMuted; + } + + void setNewStream(MediaStream newStream) { + stream = newStream; + onStreamChanged.add(stream!); + } + + void setAudioMuted(bool muted) { + audioMuted = muted; + onMuteStateChanged.add(this); + } + + void setVideoMuted(bool muted) { + videoMuted = muted; + onMuteStateChanged.add(this); + } +} diff --git a/lib/src/voip/voip.dart b/lib/src/voip/voip.dart index 931449a9..2d18b8b1 100644 --- a/lib/src/voip/voip.dart +++ b/lib/src/voip/voip.dart @@ -1,193 +1,343 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:core'; +import 'package:collection/collection.dart'; import 'package:sdp_transform/sdp_transform.dart' as sdp_transform; import 'package:webrtc_interface/webrtc_interface.dart'; import 'package:matrix/matrix.dart'; import 'package:matrix/src/utils/cached_stream_controller.dart'; +import 'package:matrix/src/utils/crypto/crypto.dart'; +import 'package:matrix/src/voip/models/call_membership.dart'; +import 'package:matrix/src/voip/models/call_options.dart'; +import 'package:matrix/src/voip/models/voip_id.dart'; +import 'package:matrix/src/voip/utils/stream_helper.dart'; -/// Delegate WebRTC basic functionality. -abstract class WebRTCDelegate { - MediaDevices get mediaDevices; - Future createPeerConnection( - Map configuration, - [Map constraints = const {}]); - VideoRenderer createRenderer(); - Future playRingtone(); - Future stopRingtone(); - Future handleNewCall(CallSession session); - Future handleCallEnded(CallSession session); - Future handleMissedCall(CallSession session); - Future handleNewGroupCall(GroupCall groupCall); - Future handleGroupCallEnded(GroupCall groupCall); - bool get isWeb; - - /// This should be set to false if any calls in the client are in kConnected - /// state. If another room tries to call you during a connected call this fires - /// a handleMissedCall - bool get canHandleNewCall => true; -} - +/// The parent highlevel voip class, this trnslates matrix events to webrtc methods via +/// `CallSession` or `GroupCallSession` methods class VoIP { // used only for internal tests, all txids for call events will be overwritten to this static String? customTxid; + /// set to true if you want to use the ratcheting mechanism with your keyprovider + /// remember to set the window size correctly on your keyprovider + /// + /// at client level because reinitializing a `GroupCallSession` and its `KeyProvider` + /// everytime this changed would be a pain + final bool enableSFUE2EEKeyRatcheting; + + /// cached turn creds TurnServerCredentials? _turnServerCredentials; - Map calls = {}; - Map groupCalls = {}; + + Map get calls => _calls; + final Map _calls = {}; + + Map get groupCalls => _groupCalls; + final Map _groupCalls = {}; + final CachedStreamController onIncomingCall = CachedStreamController(); - String? currentCID; - String? currentGroupCID; - String? get localPartyId => client.deviceID; + + VoipId? currentCID; + VoipId? currentGroupCID; + + String get localPartyId => currentSessionId; + final Client client; final WebRTCDelegate delegate; - final StreamController onIncomingGroupCall = StreamController(); - void _handleEvent( - Event event, - Function(String roomId, String senderId, Map content) - func) => - func(event.roomId!, event.senderId, event.content); - Map incomingCallRoomId = {}; + final StreamController onIncomingGroupCall = + StreamController(); - VoIP(this.client, this.delegate) : super() { + CallParticipant? get localParticipant => client.isLogged() + ? CallParticipant( + this, + userId: client.userID!, + deviceId: client.deviceID, + ) + : null; + + /// map of roomIds to the invites they are currently processing or in a call with + /// used for handling glare in p2p calls + Map get incomingCallRoomId => _incomingCallRoomId; + final Map _incomingCallRoomId = {}; + + /// the current instance of voip, changing this will drop any ongoing mesh calls + /// with that sessionId + late String currentSessionId; + VoIP( + this.client, + this.delegate, { + this.enableSFUE2EEKeyRatcheting = false, + }) : super() { + currentSessionId = base64Encode(secureRandomBytes(16)); + Logs().v('set currentSessionId to $currentSessionId'); // to populate groupCalls with already present calls for (final room in client.rooms) { - if (room.activeGroupCallEvents.isNotEmpty) { - for (final groupCall in room.activeGroupCallEvents) { - // ignore: discarded_futures - createGroupCallFromRoomStateEvent(groupCall, - emitHandleNewGroupCall: false); + final memsList = room.getCallMembershipsFromRoom(); + for (final mems in memsList.values) { + for (final mem in mems) { + unawaited(createGroupCallFromRoomStateEvent(mem)); } } } - client.onCallInvite.stream - .listen((event) => _handleEvent(event, onCallInvite)); - client.onCallAnswer.stream - .listen((event) => _handleEvent(event, onCallAnswer)); - client.onCallCandidates.stream - .listen((event) => _handleEvent(event, onCallCandidates)); - client.onCallHangup.stream - .listen((event) => _handleEvent(event, onCallHangup)); - client.onCallReject.stream - .listen((event) => _handleEvent(event, onCallReject)); - client.onCallNegotiate.stream - .listen((event) => _handleEvent(event, onCallNegotiate)); - client.onCallReplaces.stream - .listen((event) => _handleEvent(event, onCallReplaces)); - client.onCallSelectAnswer.stream - .listen((event) => _handleEvent(event, onCallSelectAnswer)); - client.onSDPStreamMetadataChangedReceived.stream.listen( - (event) => _handleEvent(event, onSDPStreamMetadataChangedReceived)); - client.onAssertedIdentityReceived.stream - .listen((event) => _handleEvent(event, onAssertedIdentityReceived)); + /// handles events todevice and matrix events for invite, candidates, hangup, etc. + client.onCallEvents.stream.listen((events) async { + await _handleCallEvents(events); + }); + // handles the com.famedly.call events. client.onRoomState.stream.listen( (event) async { - if ([ - EventTypes.GroupCallPrefix, - EventTypes.GroupCallMemberPrefix, - ].contains(event.type)) { - Logs().v('[VOIP] onRoomState: type ${event.toJson()}.'); - await onRoomStateChanged(event); + if (event.type == EventTypes.GroupCallMember) { + Logs().v('[VOIP] onRoomState: type ${event.toJson()}'); + final mems = event.room.getCallMembershipsFromEvent(event); + for (final mem in mems) { + unawaited(createGroupCallFromRoomStateEvent(mem)); + } + for (final map in groupCalls.entries) { + if (map.key.roomId == event.room.id) { + // because we don't know which call got updated, just update all + // group calls we have entered for that room + await map.value.onMemberStateChanged(); + } + } } }, ); - client.onToDeviceEvent.stream.listen((event) async { - Logs().v('[VOIP] onToDeviceEvent: type ${event.toJson()}.'); - - if (event.type == 'org.matrix.call_duplicate_session') { - Logs().v('[VOIP] onToDeviceEvent: duplicate session.'); - return; - } - - final confId = event.content['conf_id']; - final groupCall = groupCalls[confId]; - if (groupCall == null) { - Logs().d('[VOIP] onToDeviceEvent: groupCall is null.'); - return; - } - final roomId = groupCall.room.id; - final senderId = event.senderId; - final content = event.content; - switch (event.type) { - case EventTypes.CallInvite: - await onCallInvite(roomId, senderId, content); - break; - case EventTypes.CallAnswer: - await onCallAnswer(roomId, senderId, content); - break; - case EventTypes.CallCandidates: - await onCallCandidates(roomId, senderId, content); - break; - case EventTypes.CallHangup: - await onCallHangup(roomId, senderId, content); - break; - case EventTypes.CallReject: - await onCallReject(roomId, senderId, content); - break; - case EventTypes.CallNegotiate: - await onCallNegotiate(roomId, senderId, content); - break; - case EventTypes.CallReplaces: - await onCallReplaces(roomId, senderId, content); - break; - case EventTypes.CallSelectAnswer: - await onCallSelectAnswer(roomId, senderId, content); - break; - case EventTypes.CallSDPStreamMetadataChanged: - case EventTypes.CallSDPStreamMetadataChangedPrefix: - await onSDPStreamMetadataChangedReceived(roomId, senderId, content); - break; - case EventTypes.CallAssertedIdentity: - await onAssertedIdentityReceived(roomId, senderId, content); - break; - } - }); - delegate.mediaDevices.ondevicechange = _onDeviceChange; } + Future _handleCallEvents(List callEvents) async { + // Call invites should be omitted for a call that is already answered, + // has ended, is rejectd or replaced. + final callEventsCopy = List.from(callEvents); + for (final callEvent in callEventsCopy) { + final callId = callEvent.content.tryGet('call_id'); + + if (CallConstants.callEndedEventTypes.contains(callEvent.type)) { + callEvents.removeWhere((event) { + if (CallConstants.omitWhenCallEndedTypes.contains(event.type) && + event.content.tryGet('call_id') == callId) { + Logs().v( + 'Ommit "${event.type}" event for an already terminated call'); + return true; + } + + return false; + }); + } + + // checks for ended events and removes invites for that call id. + if (callEvent is Event) { + // removes expired invites + final age = callEvent.unsigned?.tryGet('age') ?? + (DateTime.now().millisecondsSinceEpoch - + callEvent.originServerTs.millisecondsSinceEpoch); + + callEvents.removeWhere((element) { + if (callEvent.type == EventTypes.CallInvite && + age > + (callEvent.content.tryGet('lifetime') ?? + CallTimeouts.callInviteLifetime.inMilliseconds)) { + Logs().w( + '[VOIP] Ommiting invite event ${callEvent.eventId} as age was older than lifetime'); + return true; + } + return false; + }); + } + } + + // and finally call the respective methods on the clean callEvents list + for (final callEvent in callEvents) { + await _handleCallEvent(callEvent); + } + } + + Future _handleCallEvent(BasicEventWithSender event) async { + // member event updates handled in onRoomState for ease + if (event.type == EventTypes.GroupCallMember) return; + + GroupCallSession? groupCallSession; + Room? room; + final remoteUserId = event.senderId; + String? remoteDeviceId; + + if (event is Event) { + room = event.room; + + /// this can also be sent in p2p calls when they want to call a specific device + remoteDeviceId = event.content.tryGet('invitee_device_id'); + } else if (event is ToDeviceEvent) { + final roomId = event.content.tryGet('room_id'); + final confId = event.content.tryGet('conf_id'); + + /// to-device events specifically, m.call.invite and encryption key sending and requesting + remoteDeviceId = event.content.tryGet('device_id'); + + if (roomId != null && confId != null) { + room = client.getRoomById(roomId); + groupCallSession = groupCalls[VoipId(roomId: roomId, callId: confId)]; + } else { + Logs().w( + '[VOIP] Ignoring to_device event of type ${event.type} but did not find group call for id: $confId'); + return; + } + + if (!event.type.startsWith(EventTypes.GroupCallMemberEncryptionKeys)) { + // livekit calls have their own session deduplication logic so ignore sessionId deduplication for them + final destSessionId = event.content.tryGet('dest_session_id'); + if (destSessionId != currentSessionId) { + Logs().w( + '[VOIP] Ignoring to_device event of type ${event.type} did not match currentSessionId: $currentSessionId, dest_session_id was set to $destSessionId'); + return; + } + } else if (groupCallSession == null || remoteDeviceId == null) { + Logs().w( + '[VOIP] _handleCallEvent ${event.type} recieved but either groupCall ${groupCallSession?.groupCallId} or deviceId $remoteDeviceId was null, ignoring'); + return; + } + } else { + Logs().w( + '[VOIP] _handleCallEvent can only handle Event or ToDeviceEvent, it got ${event.runtimeType}'); + return; + } + + final content = event.content; + + if (room == null) { + Logs().w( + '[VOIP] _handleCallEvent call event does not contain a room_id, ignoring'); + return; + } else if (!event.type + .startsWith(EventTypes.GroupCallMemberEncryptionKeys)) { + // skip webrtc event checks on encryption_keys + final callId = content['call_id'] as String?; + final partyId = content['party_id'] as String?; + if (callId == null && event.type.startsWith('m.call')) { + Logs().w('Ignoring call event ${event.type} because call_id was null'); + return; + } + if (callId != null) { + final call = calls[VoipId(roomId: room.id, callId: callId)]; + if (call == null && event.type != EventTypes.CallInvite) { + Logs().w( + 'Ignoring call event ${event.type} because we do not have the call'); + return; + } else if (call != null) { + // multiple checks to make sure the events sent are from the the + // expected party + if (call.room.id != room.id) { + Logs().w( + 'Ignoring call event ${event.type} for room ${room.id} claiming to be for call in room ${call.room.id}'); + return; + } + if (call.remoteUserId != null && call.remoteUserId != remoteUserId) { + Logs().w( + 'Ignoring call event ${event.type} from sender $remoteUserId, expected sender: ${call.remoteUserId}'); + return; + } + if (call.remotePartyId != null && call.remotePartyId != partyId) { + Logs().w( + 'Ignoring call event ${event.type} from sender with a different party_id $partyId, expected party_id: ${call.remotePartyId}'); + return; + } + if ((call.remotePartyId != null && + call.remotePartyId == localPartyId) || + (remoteUserId == client.userID && + remoteDeviceId == client.deviceID!)) { + Logs().w('Ignoring call event ${event.type} from ourself'); + return; + } + } + } + } + Logs().v( + '[VOIP] Handling event of type: ${event.type}, content ${event.content} from sender ${event.senderId} rp: $remoteUserId:$remoteDeviceId'); + + switch (event.type) { + case EventTypes.CallInvite: + case EventTypes.GroupCallMemberInvite: + await onCallInvite(room, remoteUserId, remoteDeviceId, content); + break; + case EventTypes.CallAnswer: + case EventTypes.GroupCallMemberAnswer: + await onCallAnswer(room, remoteUserId, remoteDeviceId, content); + break; + case EventTypes.CallCandidates: + case EventTypes.GroupCallMemberCandidates: + await onCallCandidates(room, content); + break; + case EventTypes.CallHangup: + case EventTypes.GroupCallMemberHangup: + await onCallHangup(room, content); + break; + case EventTypes.CallReject: + case EventTypes.GroupCallMemberReject: + await onCallReject(room, content); + break; + case EventTypes.CallNegotiate: + case EventTypes.GroupCallMemberNegotiate: + await onCallNegotiate(room, content); + break; + // case EventTypes.CallReplaces: + // await onCallReplaces(room, content); + // break; + case EventTypes.CallSelectAnswer: + case EventTypes.GroupCallMemberSelectAnswer: + await onCallSelectAnswer(room, content); + break; + case EventTypes.CallSDPStreamMetadataChanged: + case EventTypes.CallSDPStreamMetadataChangedPrefix: + case EventTypes.GroupCallMemberSDPStreamMetadataChanged: + await onSDPStreamMetadataChangedReceived(room, content); + break; + case EventTypes.CallAssertedIdentity: + case EventTypes.CallAssertedIdentityPrefix: + case EventTypes.GroupCallMemberAssertedIdentity: + await onAssertedIdentityReceived(room, content); + break; + case EventTypes.GroupCallMemberEncryptionKeys: + await groupCallSession!.backend.onCallEncryption( + groupCallSession, remoteUserId, remoteDeviceId!, content); + break; + case EventTypes.GroupCallMemberEncryptionKeysRequest: + await groupCallSession!.backend.onCallEncryptionKeyRequest( + groupCallSession, remoteUserId, remoteDeviceId!, content); + break; + } + } + Future _onDeviceChange(dynamic _) async { Logs().v('[VOIP] _onDeviceChange'); for (final call in calls.values) { if (call.state == CallState.kConnected && !call.isGroupCall) { - await call.updateAudioDevice(); + await call.updateMediaDeviceForCall(); } } for (final groupCall in groupCalls.values) { - if (groupCall.state == GroupCallState.Entered) { - await groupCall.updateAudioDevice(); + if (groupCall.state == GroupCallState.entered) { + await groupCall.backend.updateMediaDeviceForCalls(); } } } - Future onCallInvite( - String roomId, String senderId, Map content) async { - if (senderId == client.userID) { - // Ignore messages to yourself. - return; - } - + Future onCallInvite(Room room, String remoteUserId, + String? remoteDeviceId, Map content) async { Logs().v( - '[VOIP] onCallInvite $senderId => ${client.userID}, \ncontent => ${content.toString()}'); + '[VOIP] onCallInvite $remoteUserId:$remoteDeviceId => ${client.userID}:${client.deviceID}, \ncontent => ${content.toString()}'); final String callId = content['call_id']; - final String partyId = content['party_id']; final int lifetime = content['lifetime']; final String? confId = content['conf_id']; - // msc3401 group call invites send deviceId and senderSessionId in to device messages - final String? deviceId = content['device_id']; - final String? senderSessionId = content['sender_session_id']; - - final call = calls[callId]; + final call = calls[VoipId(roomId: room.id, callId: callId)]; Logs().d( - '[glare] got new call ${content.tryGet('call_id')} and currently room id is mapped to ${incomingCallRoomId.tryGet(roomId)}'); + '[glare] got new call ${content.tryGet('call_id')} and currently room id is mapped to ${incomingCallRoomId.tryGet(room.id)}'); if (call != null && call.state == CallState.kEnded) { // Session already exist. @@ -195,9 +345,17 @@ class VoIP { return; } - if (content['invitee'] != null && content['invitee'] != client.userID) { + final inviteeUserId = content['invitee']; + if (inviteeUserId != null && inviteeUserId != localParticipant?.userId) { + Logs().w('[VOIP] Ignoring call, meant for user $inviteeUserId'); return; // This invite was meant for another user in the room } + final inviteeDeviceId = content['invitee_device_id']; + if (inviteeDeviceId != null && + inviteeDeviceId != localParticipant?.deviceId) { + Logs().w('[VOIP] Ignoring call, meant for device $inviteeDeviceId'); + return; // This invite was meant for another device in the room + } if (content['capabilities'] != null) { final capabilities = CallCapabilities.fromJson(content['capabilities']); @@ -223,29 +381,36 @@ class VoIP { callType = getCallType(content['offer']['sdp']); } - final room = client.getRoomById(roomId); - - final opts = CallOptions() - ..voip = this - ..callId = callId - ..groupCallId = confId - ..dir = CallDirection.kIncoming - ..type = callType - ..room = room! - ..localPartyId = localPartyId! - ..iceServers = await getIceSevers(); + final opts = CallOptions( + voip: this, + callId: callId, + groupCallId: confId, + dir: CallDirection.kIncoming, + type: callType, + room: room, + localPartyId: localPartyId, + iceServers: await getIceServers(), + ); final newCall = createNewCall(opts); - newCall.remotePartyId = partyId; - newCall.remoteUser = await room.requestUser(senderId); - newCall.opponentDeviceId = deviceId; - newCall.opponentSessionId = senderSessionId; + + /// both invitee userId and deviceId are set here because there can be + /// multiple devices from same user in a call, so we specifiy who the + /// invite is for + newCall.remoteUserId = remoteUserId; + newCall.remoteDeviceId = remoteDeviceId; + newCall.remotePartyId = content['party_id']; + newCall.remoteSessionId = content['sender_session_id']; + + // newCall.remoteSessionId = remoteParticipant.sessionId; + if (!delegate.canHandleNewCall && - (confId == null || confId != currentGroupCID)) { + (confId == null || + currentGroupCID != VoipId(roomId: room.id, callId: confId))) { Logs().v( '[VOIP] onCallInvite: Unable to handle new calls, maybe user is busy.'); // no need to emit here because handleNewCall was never triggered yet - await newCall.reject(reason: CallErrorCode.UserBusy, shouldEmit: false); + await newCall.reject(reason: CallErrorCode.userBusy, shouldEmit: false); await delegate.handleMissedCall(newCall); return; } @@ -270,7 +435,7 @@ class VoIP { // and all this happens inside initWithInvite. If we set currentCID after // initWithInvite, we might set it to callId even after it was reset to null // by terminate. - currentCID = callId; + currentCID = VoipId(roomId: room.id, callId: callId); await newCall.initWithInvite( callType, offer, sdpStreamMetadata, lifetime, confId != null); @@ -286,31 +451,44 @@ class VoIP { } } - Future onCallAnswer( - String roomId, String senderId, Map content) async { + Future onCallAnswer(Room room, String remoteUserId, + String? remoteDeviceId, Map content) async { Logs().v('[VOIP] onCallAnswer => ${content.toString()}'); final String callId = content['call_id']; - final String partyId = content['party_id']; - final call = calls[callId]; + final call = calls[VoipId(roomId: room.id, callId: callId)]; if (call != null) { - if (senderId == client.userID) { - // Ignore messages to yourself. - if (!call.answeredByUs) { - await delegate.stopRingtone(); - } - if (call.state == CallState.kRinging) { - await call.onAnsweredElsewhere(); - } - return; + if (!call.answeredByUs) { + await delegate.stopRingtone(); } - if (call.room.id != roomId) { + if (call.state == CallState.kRinging) { + await call.onAnsweredElsewhere(); + } + + if (call.room.id != room.id) { Logs().w( - 'Ignoring call answer for room $roomId claiming to be for call in room ${call.room.id}'); + 'Ignoring call answer for room ${room.id} claiming to be for call in room ${call.room.id}'); return; } - call.remotePartyId = partyId; - call.remoteUser = await call.room.requestUser(senderId); + + if (call.remoteUserId == null) { + Logs().i( + '[VOIP] you probably called the room without setting a userId in invite, setting the calls remote user id to what I get from m.call.answer now'); + call.remoteUserId = remoteUserId; + } + + if (call.remoteDeviceId == null) { + Logs().i( + '[VOIP] you probably called the room without setting a userId in invite, setting the calls remote user id to what I get from m.call.answer now'); + call.remoteDeviceId = remoteDeviceId; + } + if (call.remotePartyId != null) { + Logs().d( + 'Ignoring call answer from party ${content['party_id']}, we are already with ${call.remotePartyId}'); + return; + } else { + call.remotePartyId = content['party_id']; + } final answer = RTCSessionDescription( content['answer']['sdp'], content['answer']['type']); @@ -325,115 +503,67 @@ class VoIP { } } - Future onCallCandidates( - String roomId, String senderId, Map content) async { - if (senderId == client.userID) { - // Ignore messages to yourself. - return; - } + Future onCallCandidates(Room room, Map content) async { Logs().v('[VOIP] onCallCandidates => ${content.toString()}'); final String callId = content['call_id']; - final call = calls[callId]; + final call = calls[VoipId(roomId: room.id, callId: callId)]; if (call != null) { - if (call.room.id != roomId) { - Logs().w( - 'Ignoring call candidates for room $roomId claiming to be for call in room ${call.room.id}'); - return; - } await call.onCandidatesReceived(content['candidates']); } else { Logs().v('[VOIP] onCallCandidates: Session [$callId] not found!'); } } - Future onCallHangup(String roomId, String _ /*senderId unused*/, - Map content) async { + Future onCallHangup(Room room, Map content) async { // stop play ringtone, if this is an incoming call await delegate.stopRingtone(); Logs().v('[VOIP] onCallHangup => ${content.toString()}'); final String callId = content['call_id']; - final String partyId = content['party_id']; - final call = calls[callId]; + + final call = calls[VoipId(roomId: room.id, callId: callId)]; if (call != null) { - if (call.room.id != roomId) { - Logs().w( - 'Ignoring call hangup for room $roomId claiming to be for call in room ${call.room.id}'); - return; - } - if (call.remotePartyId != null && call.remotePartyId != partyId) { - Logs().w( - 'Ignoring call hangup from sender with a different party_id $partyId for call in room ${call.room.id}'); - return; - } // hangup in any case, either if the other party hung up or we did on another device - await call.terminate(CallParty.kRemote, - content['reason'] ?? CallErrorCode.UserHangup, true); + await call.terminate( + CallParty.kRemote, + CallErrorCode.values.firstWhereOrNull( + (element) => element.reason == content['reason']) ?? + CallErrorCode.userHangup, + true); } else { Logs().v('[VOIP] onCallHangup: Session [$callId] not found!'); } - if (callId == currentCID) { + if (callId == currentCID?.callId) { currentCID = null; } } - Future onCallReject( - String roomId, String senderId, Map content) async { + Future onCallReject(Room room, Map content) async { final String callId = content['call_id']; - final String partyId = content['party_id']; Logs().d('Reject received for call ID $callId'); - final call = calls[callId]; + final call = calls[VoipId(roomId: room.id, callId: callId)]; if (call != null) { - if (call.room.id != roomId) { - Logs().w( - 'Ignoring call reject for room $roomId claiming to be for call in room ${call.room.id}'); - return; - } - if (call.remotePartyId != null && call.remotePartyId != partyId) { - Logs().w( - 'Ignoring call reject from sender with a different party_id $partyId for call in room ${call.room.id}'); - return; - } - await call.onRejectReceived(content['reason']); + await call.onRejectReceived( + CallErrorCode.values.firstWhereOrNull( + (element) => element.reason == content['reason']) ?? + CallErrorCode.userHangup, + ); } else { Logs().v('[VOIP] onCallReject: Session [$callId] not found!'); } } - Future onCallReplaces( - String roomId, String senderId, Map content) async { - if (senderId == client.userID) { - // Ignore messages to yourself. - return; - } - final String callId = content['call_id']; - Logs().d('onCallReplaces received for call ID $callId'); - final call = calls[callId]; - if (call != null) { - if (call.room.id != roomId) { - Logs().w( - 'Ignoring call replace for room $roomId claiming to be for call in room ${call.room.id}'); - return; - } - //TODO: handle replaces - } - } - Future onCallSelectAnswer( - String roomId, String senderId, Map content) async { - if (senderId == client.userID) { - // Ignore messages to yourself. - return; - } + Room room, Map content) async { final String callId = content['call_id']; Logs().d('SelectAnswer received for call ID $callId'); - final call = calls[callId]; final String selectedPartyId = content['selected_party_id']; + final call = calls[VoipId(roomId: room.id, callId: callId)]; if (call != null) { - if (call.room.id != roomId) { + if (call.room.id != room.id) { Logs().w( - 'Ignoring call select answer for room $roomId claiming to be for call in room ${call.room.id}'); + 'Ignoring call select answer for room ${room.id} claiming to be for call in room ${call.room.id}'); return; } await call.onSelectAnswerReceived(selectedPartyId); @@ -441,21 +571,12 @@ class VoIP { } Future onSDPStreamMetadataChangedReceived( - String roomId, String senderId, Map content) async { - if (senderId == client.userID) { - // Ignore messages to yourself. - return; - } + Room room, Map content) async { final String callId = content['call_id']; Logs().d('SDP Stream metadata received for call ID $callId'); - final call = calls[callId]; - if (call != null) { - if (call.room.id != roomId) { - Logs().w( - 'Ignoring call sdp metadata change for room $roomId claiming to be for call in room ${call.room.id}'); - return; - } + final call = calls[VoipId(roomId: room.id, callId: callId)]; + if (call != null) { if (content[sdpStreamMetadataKey] == null) { Logs().d('SDP Stream metadata is null'); return; @@ -466,21 +587,12 @@ class VoIP { } Future onAssertedIdentityReceived( - String roomId, String senderId, Map content) async { - if (senderId == client.userID) { - // Ignore messages to yourself. - return; - } + Room room, Map content) async { final String callId = content['call_id']; Logs().d('Asserted identity received for call ID $callId'); - final call = calls[callId]; - if (call != null) { - if (call.room.id != roomId) { - Logs().w( - 'Ignoring call asserted identity for room $roomId claiming to be for call in room ${call.room.id}'); - return; - } + final call = calls[VoipId(roomId: room.id, callId: callId)]; + if (call != null) { if (content['asserted_identity'] == null) { Logs().d('asserted_identity is null '); return; @@ -490,30 +602,12 @@ class VoIP { } } - Future onCallNegotiate( - String roomId, String senderId, Map content) async { - if (senderId == client.userID) { - // Ignore messages to yourself. - return; - } + Future onCallNegotiate(Room room, Map content) async { final String callId = content['call_id']; Logs().d('Negotiate received for call ID $callId'); - final call = calls[callId]; - if (call != null) { - if (call.room.id != roomId) { - Logs().w( - 'Ignoring call negotiation for room $roomId claiming to be for call in room ${call.room.id}'); - return; - } - if (content['party_id'] != call.remotePartyId) { - Logs().w('Ignoring call negotiation, wrong partyId detected'); - return; - } - if (content['party_id'] == call.localPartyId) { - Logs().w('Ignoring call negotiation echo'); - return; - } + final call = calls[VoipId(roomId: room.id, callId: callId)]; + if (call != null) { // ideally you also check the lifetime here and discard negotiation events // if age of the event was older than the lifetime but as to device events // do not have a unsigned age nor a origin_server_ts there's no easy way to @@ -528,7 +622,7 @@ class VoIP { await call.onNegotiateReceived(metadata, RTCSessionDescription(description['sdp'], description['type'])); } catch (e, s) { - Logs().e('Failed to complete negotiation', e, s); + Logs().e('[VOIP] Failed to complete negotiation', e, s); } } } @@ -540,17 +634,13 @@ class VoIP { return CallType.kVideo; } } catch (e, s) { - Logs().e('Failed to getCallType', e, s); + Logs().e('[VOIP] Failed to getCallType', e, s); } return CallType.kVoice; } - Future requestTurnServerCredentials() async { - return true; - } - - Future>> getIceSevers() async { + Future>> getIceServers() async { if (_turnServerCredentials == null) { try { _turnServerCredentials = await client.getTurnServer(); @@ -574,246 +664,180 @@ class VoIP { /// Make a P2P call to room /// - /// [roomId] The room id to call + /// Pretty important to set the userId, or all the users in the room get a call. + /// Including your own other devices, so just set it to directChatMatrixId /// - /// [type] The type of call to be made. - Future inviteToCall(String roomId, CallType type) async { - final room = client.getRoomById(roomId); - if (room == null) { - Logs().v('[VOIP] Invalid room id [$roomId].'); - return Null as CallSession; - } - final callId = 'cid${DateTime.now().millisecondsSinceEpoch}'; + /// Setting the deviceId would make all other devices for that userId ignore the call + /// Ideally only group calls would need setting both userId and deviceId to allow + /// having 2 devices from the same user in a group call + /// + /// For p2p call, you want to have all the devices of the specified `userId` ring + Future inviteToCall( + Room room, + CallType type, { + String? userId, + String? deviceId, + }) async { + final roomId = room.id; + final callId = genCallID(); if (currentGroupCID == null) { incomingCallRoomId[roomId] = callId; } - final opts = CallOptions() - ..callId = callId - ..type = type - ..dir = CallDirection.kOutgoing - ..room = room - ..voip = this - ..localPartyId = localPartyId! - ..iceServers = await getIceSevers(); - + final opts = CallOptions( + callId: callId, + type: type, + dir: CallDirection.kOutgoing, + room: room, + voip: this, + localPartyId: localPartyId, + iceServers: await getIceServers(), + ); final newCall = createNewCall(opts); - currentCID = callId; + + newCall.remoteUserId = userId; + newCall.remoteDeviceId = deviceId; + + currentCID = VoipId(roomId: roomId, callId: callId); await newCall.initOutboundCall(type).then((_) { delegate.handleNewCall(newCall); }); - currentCID = callId; return newCall; } CallSession createNewCall(CallOptions opts) { final call = CallSession(opts); - calls[opts.callId] = call; + calls[VoipId(roomId: opts.room.id, callId: opts.callId)] = call; return call; } /// Create a new group call in an existing room. /// - /// [roomId] The room id to call + /// [groupCallId] The room id to call /// - /// [type] The type of call to be made. + /// [application] normal group call, thrirdroom, etc /// - /// [intent] The intent of the call. - Future newGroupCall( - String roomId, String type, String intent) async { - if (getGroupCallForRoom(roomId) != null) { - Logs().e('[VOIP] [$roomId] already has an existing group call.'); - return null; + /// [scope] room, between specifc users, etc. + Future _newGroupCall( + String groupCallId, + Room room, + CallBackend backend, + String? application, + String? scope, + ) async { + if (getGroupCallById(room.id, groupCallId) != null) { + Logs().v('[VOIP] [$groupCallId] already exists.'); + return getGroupCallById(room.id, groupCallId)!; } - final room = client.getRoomById(roomId); - if (room == null) { - Logs().v('[VOIP] Invalid room id [$roomId].'); - return null; - } - final groupId = genCallID(); - final groupCall = await GroupCall( - groupCallId: groupId, + + final groupCall = GroupCallSession( + groupCallId: groupCallId, client: client, - voip: this, room: room, - type: type, - intent: intent, - ).create(); - groupCalls[groupId] = groupCall; - groupCalls[roomId] = groupCall; + voip: this, + backend: backend, + application: application, + scope: scope, + ); + + setGroupCallById(groupCall); + return groupCall; } - Future fetchOrCreateGroupCall(String roomId) async { - final groupCall = getGroupCallForRoom(roomId); - final room = client.getRoomById(roomId); - if (room == null) { - Logs().w('Not found room id = $roomId'); - return null; - } + /// Create a new group call in an existing room. + /// + /// [groupCallId] The room id to call + /// + /// [application] normal group call, thrirdroom, etc + /// + /// [scope] room, between specifc users, etc. + + Future fetchOrCreateGroupCall( + String groupCallId, + Room room, + CallBackend backend, + String? application, + String? scope, + ) async { + final groupCall = getGroupCallById(room.id, groupCallId); if (groupCall != null) { if (!room.canJoinGroupCall) { - Logs().w('No permission to join group calls in room $roomId'); - return null; + throw Exception( + 'User is not allowed to join famedly calls in the room'); } return groupCall; } - if (!room.groupCallsEnabled) { + if (!room.groupCallsEnabledForEveryone) { await room.enableGroupCalls(); } - if (room.canCreateGroupCall) { - // The call doesn't exist, but we can create it - - final groupCall = await newGroupCall( - roomId, GroupCallType.Video, GroupCallIntent.Prompt); - if (groupCall != null) { - await groupCall.sendMemberStateEvent(); - } - return groupCall; - } - - final completer = Completer(); - Timer? timer; - final subscription = onIncomingGroupCall.stream.listen((GroupCall call) { - if (call.room.id == roomId) { - timer?.cancel(); - completer.complete(call); - } - }); - - timer = Timer(Duration(seconds: 30), () { - subscription.cancel(); - completer.completeError('timeout'); - }); - - return completer.future; + // The call doesn't exist, but we can create it + return await _newGroupCall( + groupCallId, + room, + backend, + application, + scope, + ); } - GroupCall? getGroupCallForRoom(String roomId) { - return groupCalls[roomId]; + GroupCallSession? getGroupCallById(String roomId, String groupCallId) { + return groupCalls[VoipId(roomId: roomId, callId: groupCallId)]; } - GroupCall? getGroupCallById(String groupCallId) { - return groupCalls[groupCallId]; - } - - Future startGroupCalls() async { - final rooms = client.rooms; - for (final room in rooms) { - await createGroupCallForRoom(room); - } - } - - Future stopGroupCalls() async { - for (final groupCall in groupCalls.values) { - await groupCall.terminate(); - } - groupCalls.clear(); - } - - /// Create a new group call in an existing room. - Future createGroupCallForRoom(Room room) async { - final events = await client.getRoomState(room.id); - events.sort((a, b) => a.originServerTs.compareTo(b.originServerTs)); - - for (final event in events) { - if (event.type == EventTypes.GroupCallPrefix) { - if (event.content['m.terminated'] != null) { - return; - } - await createGroupCallFromRoomStateEvent(event); - } - } - - return; + void setGroupCallById(GroupCallSession groupCallSession) { + groupCalls[VoipId( + roomId: groupCallSession.room.id, + callId: groupCallSession.groupCallId, + )] = groupCallSession; } /// Create a new group call from a room state event. - Future createGroupCallFromRoomStateEvent(MatrixEvent event, - {bool emitHandleNewGroupCall = true}) async { - final roomId = event.roomId; - final content = event.content; + Future createGroupCallFromRoomStateEvent( + CallMembership membership, { + bool emitHandleNewGroupCall = true, + }) async { + if (membership.isExpired) { + Logs().d( + 'Ignoring expired membership in passive groupCall creator. ${membership.toJson()}'); + return; + } - final room = client.getRoomById(roomId!); + final room = client.getRoomById(membership.roomId); if (room == null) { - Logs().w('Couldn\'t find room $roomId for GroupCall'); - return null; + Logs().w('Couldn\'t find room ${membership.roomId} for GroupCallSession'); + return; } - final groupCallId = event.stateKey; - - final callType = content.tryGet('m.type'); - - if (callType == null || - callType != GroupCallType.Video && callType != GroupCallType.Voice) { - Logs().w('Received invalid group call type $callType for room $roomId.'); - return null; + if (membership.application != 'm.call' && membership.scope != 'm.room') { + Logs().w('Received invalid group call application or scope.'); + return; } - final callIntent = content.tryGet('m.intent'); - - if (callIntent == null || - callIntent != GroupCallIntent.Prompt && - callIntent != GroupCallIntent.Room && - callIntent != GroupCallIntent.Ring) { - Logs() - .w('Received invalid group call intent $callType for room $roomId.'); - return null; - } - - final groupCall = GroupCall( + final groupCall = GroupCallSession( client: client, voip: this, room: room, - groupCallId: groupCallId, - type: callType, - intent: callIntent, + backend: membership.backend, + groupCallId: membership.callId, + application: membership.application, + scope: membership.scope, ); - groupCalls[groupCallId!] = groupCall; - groupCalls[room.id] = groupCall; + if (groupCalls.containsKey( + VoipId(roomId: membership.roomId, callId: membership.callId))) { + return; + } + + setGroupCallById(groupCall); onIncomingGroupCall.add(groupCall); if (emitHandleNewGroupCall) { await delegate.handleNewGroupCall(groupCall); } - return groupCall; - } - - Future onRoomStateChanged(MatrixEvent event) async { - final eventType = event.type; - final roomId = event.roomId; - if (eventType == EventTypes.GroupCallPrefix) { - final groupCallId = event.stateKey; - final content = event.content; - final currentGroupCall = groupCalls[groupCallId]; - if (currentGroupCall == null && content['m.terminated'] == null) { - await createGroupCallFromRoomStateEvent(event); - } else if (currentGroupCall != null && - currentGroupCall.groupCallId == groupCallId) { - if (content['m.terminated'] != null) { - await currentGroupCall.terminate(emitStateEvent: false); - } else if (content['m.type'] != currentGroupCall.type) { - // TODO: Handle the callType changing when the room state changes - Logs().w( - 'The group call type changed for room: $roomId. Changing the group call type is currently unsupported.'); - } - } else if (currentGroupCall != null && - currentGroupCall.groupCallId != groupCallId) { - // TODO: Handle new group calls and multiple group calls - Logs().w( - 'Multiple group calls detected for room: $roomId. Multiple group calls are currently unsupported.'); - } - } else if (eventType == EventTypes.GroupCallMemberPrefix) { - final groupCall = groupCalls[roomId]; - if (groupCall == null) { - return; - } - await groupCall.onMemberStateChanged(event); - } } @Deprecated('Call `hasActiveGroupCall` on the room directly instead') diff --git a/lib/src/voip/voip_room_extension.dart b/lib/src/voip/voip_room_extension.dart deleted file mode 100644 index eba4bdbb..00000000 --- a/lib/src/voip/voip_room_extension.dart +++ /dev/null @@ -1,101 +0,0 @@ -import 'package:collection/collection.dart'; - -import 'package:matrix/matrix.dart'; - -extension GroupCallUtils on Room { - /// returns the user count (not sessions, yet) for the group call with id: `groupCallId`. - /// returns 0 if group call not found - int? groupCallParticipantCount(String groupCallId) { - int participantCount = 0; - final groupCallMemberStates = - states.tryGetMap(EventTypes.GroupCallMemberPrefix); - if (groupCallMemberStates != null) { - groupCallMemberStates.forEach((userId, memberStateEvent) { - if (!callMemberStateForIdIsExpired(memberStateEvent, groupCallId)) { - participantCount++; - } - }); - } - return participantCount; - } - - bool get hasActiveGroupCall { - if (activeGroupCallEvents.isNotEmpty) { - return true; - } - return false; - } - - /// list of active group calls - List get activeGroupCallEvents { - final groupCallStates = - states.tryGetMap(EventTypes.GroupCallPrefix); - if (groupCallStates != null) { - groupCallStates.values - .toList() - .sort((a, b) => a.originServerTs.compareTo(b.originServerTs)); - return groupCallStates.values - .where((element) => - !element.content.containsKey('m.terminated') && - callMemberStateIsExpired(element)) - .toList(); - } - return []; - } - - static const staleCallCheckerDuration = Duration(seconds: 30); - - /// checks if a member event has any existing non-expired callId - bool callMemberStateIsExpired(MatrixEvent event) { - final callMemberState = IGroupCallRoomMemberState.fromJson(event); - final calls = callMemberState.calls; - return calls - .where((call) => call.devices.any((d) => - (d.expires_ts ?? 0) + - staleCallCheckerDuration - .inMilliseconds > // buffer for sync glare - DateTime.now().millisecondsSinceEpoch)) - .isEmpty; - } - - /// checks if the member event has `groupCallId` unexpired, if not it checks if - /// the whole event is expired or not - bool callMemberStateForIdIsExpired( - MatrixEvent groupCallMemberStateEvent, String groupCallId) { - final callMemberState = - IGroupCallRoomMemberState.fromJson(groupCallMemberStateEvent); - final calls = callMemberState.calls; - if (calls.isNotEmpty) { - final call = - calls.singleWhereOrNull((call) => call.call_id == groupCallId); - if (call != null) { - return call.devices.where((device) => device.expires_ts != null).every( - (device) => - (device.expires_ts ?? 0) + - staleCallCheckerDuration - .inMilliseconds < // buffer for sync glare - DateTime.now().millisecondsSinceEpoch); - } else { - Logs().d( - '[VOIP] Did not find $groupCallId in member events, probably sync glare'); - return false; - } - } else { - // Last 30 seconds to get yourself together. - // This saves us from accidentally killing calls which were just created and - // whose state event we haven't recieved yet in sync. - // (option 2 was local echo member state events, but reverting them if anything - // fails sounds pain) - final expiredfr = groupCallMemberStateEvent.originServerTs - .add(staleCallCheckerDuration) - .millisecondsSinceEpoch < - DateTime.now().millisecondsSinceEpoch; - - if (!expiredfr) { - Logs().d( - '[VOIP] Last 30 seconds for state event from ${groupCallMemberStateEvent.senderId}'); - } - return expiredfr; - } - } -} diff --git a/pubspec.yaml b/pubspec.yaml index 30028aca..c5c295d9 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -32,7 +32,7 @@ dependencies: sqflite_common: ^2.4.5 sqlite3: ^2.1.0 typed_data: ^1.3.2 - webrtc_interface: ^1.0.13 + webrtc_interface: ^1.2.0 dev_dependencies: build_runner: ^2.4.8 @@ -43,7 +43,4 @@ dev_dependencies: lints: ^3.0.0 sqflite_common_ffi: 2.3.2+1 # v2.3.3 doesn't support dart v3.2.x test: ^1.15.7 - #flutter_test: {sdk: flutter} -#dependency_overrides: -# matrix_api_lite: -# path: ../matrix_api_lite + diff --git a/test/calls_test.dart b/test/calls_test.dart index 7aad93dd..7ecbace7 100644 --- a/test/calls_test.dart +++ b/test/calls_test.dart @@ -2,6 +2,9 @@ import 'package:test/test.dart'; import 'package:webrtc_interface/webrtc_interface.dart'; import 'package:matrix/matrix.dart'; +import 'package:matrix/src/voip/models/call_membership.dart'; +import 'package:matrix/src/voip/models/call_options.dart'; +import 'package:matrix/src/voip/models/voip_id.dart'; import 'fake_client.dart'; import 'webrtc_stub.dart'; @@ -13,25 +16,35 @@ void main() { Logs().level = Level.info; setUp(() async { matrix = await getClient(); + voip = VoIP(matrix, MockWebRTCDelegate()); VoIP.customTxid = '1234'; final id = '!calls:example.com'; - room = matrix.getRoomById(id)!; }); test('Test call methods', () async { - final call = CallSession(CallOptions()..room = room); - await call.sendInviteToCall(room, '1234', 1234, '4567', '7890', 'sdp', + final call = CallSession( + CallOptions( + callId: '1234', + type: CallType.kVoice, + dir: CallDirection.kOutgoing, + localPartyId: '4567', + voip: voip, + room: room, + iceServers: [], + ), + ); + await call.sendInviteToCall(room, '1234', 1234, '4567', 'sdp', txid: '1234'); await call.sendAnswerCall(room, '1234', 'sdp', '4567', txid: '1234'); await call.sendCallCandidates(room, '1234', '4567', [], txid: '1234'); await call.sendSelectCallAnswer(room, '1234', '4567', '6789', txid: '1234'); - await call.sendCallReject(room, '1234', '4567', 'busy', txid: '1234'); + await call.sendCallReject(room, '1234', '4567', txid: '1234'); await call.sendCallNegotiate(room, '1234', 1234, '4567', 'sdp', txid: '1234'); - await call.sendHangupCall(room, '1234', '4567', 'user_hangup', + await call.sendHangupCall(room, '1234', '4567', 'userHangup', txid: '1234'); await call.sendAssertedIdentity( room, @@ -152,12 +165,14 @@ void main() { ) ])) }))); - while (voip.currentCID != 'originTsValidCall') { + while (voip.currentCID != + VoipId(roomId: room.id, callId: 'originTsValidCall')) { // call invite looks valid, call should be created now :D await Future.delayed(Duration(milliseconds: 50)); Logs().d('Waiting for currentCID to update'); } - expect(voip.currentCID, 'originTsValidCall'); + expect(voip.currentCID, + VoipId(roomId: room.id, callId: 'originTsValidCall')); final call = voip.calls[voip.currentCID]!; expect(call.state, CallState.kRinging); await call.answer(txid: '1234'); @@ -219,7 +234,7 @@ void main() { expect(call.state, CallState.kConnected); - await call.hangup(); + await call.hangup(reason: CallErrorCode.userHangup); expect(call.state, CallState.kEnded); expect(voip.currentCID, null); }); @@ -277,12 +292,14 @@ void main() { ) ])) }))); - while (voip.currentCID != 'answer_elseWhere') { + while (voip.currentCID != + VoipId(roomId: room.id, callId: 'answer_elseWhere')) { // call invite looks valid, call should be created now :D await Future.delayed(Duration(milliseconds: 50)); Logs().d('Waiting for currentCID to update'); } - expect(voip.currentCID, 'answer_elseWhere'); + expect( + voip.currentCID, VoipId(roomId: room.id, callId: 'answer_elseWhere')); final call = voip.calls[voip.currentCID]!; expect(call.state, CallState.kRinging); @@ -368,12 +385,13 @@ void main() { ) ])) }))); - while (voip.currentCID != 'reject_call') { + while ( + voip.currentCID != VoipId(roomId: room.id, callId: 'reject_call')) { // call invite looks valid, call should be created now :D await Future.delayed(Duration(milliseconds: 50)); Logs().d('Waiting for currentCID to update'); } - expect(voip.currentCID, 'reject_call'); + expect(voip.currentCID, VoipId(roomId: room.id, callId: 'reject_call')); final call = voip.calls[voip.currentCID]!; expect(call.state, CallState.kRinging); @@ -386,7 +404,11 @@ void main() { test('Glare after invite was sent', () async { expect(voip.currentCID, null); - final firstCall = await voip.inviteToCall(room.id, CallType.kVoice); + final firstCall = await voip.inviteToCall( + room, + CallType.kVoice, + userId: '@alice:testing.com', + ); await firstCall.pc!.onRenegotiationNeeded!.call(); expect(firstCall.state, CallState.kInviteSent); // KABOOM YOU JUST GLARED @@ -412,12 +434,17 @@ void main() { ])) }))); await Future.delayed(Duration(seconds: 3)); - expect(voip.currentCID, firstCall.callId); - await firstCall.hangup(); + expect( + voip.currentCID, VoipId(roomId: room.id, callId: firstCall.callId)); + await firstCall.hangup(reason: CallErrorCode.userBusy); }); test('Glare before invite was sent', () async { expect(voip.currentCID, null); - final firstCall = await voip.inviteToCall(room.id, CallType.kVoice); + final firstCall = await voip.inviteToCall( + room, + CallType.kVoice, + userId: '@alice:testing.com', + ); expect(firstCall.state, CallState.kCreateOffer); // KABOOM YOU JUST GLARED, but this tiem you were still preparing your call // so just cancel that instead @@ -443,7 +470,306 @@ void main() { ])) }))); await Future.delayed(Duration(seconds: 3)); - expect(voip.currentCID, 'zzzz_glare_2nd_call'); + expect(voip.currentCID, + VoipId(roomId: room.id, callId: 'zzzz_glare_2nd_call')); + }); + + test('getFamedlyCallEvents sort order', () { + room.setState( + Event( + content: { + 'memberships': [ + CallMembership( + userId: '@test1:example.com', + callId: '1111', + backend: MeshBackend(), + deviceId: '1111', + expiresTs: DateTime.now() + .add(Duration(hours: 12)) + .millisecondsSinceEpoch, + roomId: room.id, + membershipId: voip.currentSessionId, + ).toJson(), + ] + }, + type: EventTypes.GroupCallMember, + eventId: 'asdfasdf', + senderId: '@test1:example.com', + originServerTs: DateTime.now().add(Duration(hours: 12)), + room: room, + stateKey: '@test1:example.com', + ), + ); + room.setState( + Event( + content: { + 'memberships': [ + CallMembership( + userId: '@test2:example.com', + callId: '1111', + backend: MeshBackend(), + deviceId: '1111', + expiresTs: DateTime.now().millisecondsSinceEpoch, + roomId: room.id, + membershipId: voip.currentSessionId, + ).toJson(), + ] + }, + type: EventTypes.GroupCallMember, + eventId: 'asdfasdf', + senderId: '@test2:example.com', + originServerTs: DateTime.now(), + room: room, + stateKey: '@test2:example.com', + ), + ); + room.setState( + Event( + content: { + 'memberships': [ + CallMembership( + userId: '@test2.0:example.com', + callId: '1111', + backend: MeshBackend(), + deviceId: '1111', + expiresTs: DateTime.now().millisecondsSinceEpoch, + roomId: room.id, + membershipId: voip.currentSessionId, + ).toJson(), + ] + }, + type: EventTypes.GroupCallMember, + eventId: 'asdfasdf', + senderId: '@test2.0:example.com', + originServerTs: DateTime.now(), + room: room, + stateKey: '@test2.0:example.com', + ), + ); + room.setState( + Event( + content: { + 'memberships': [ + CallMembership( + userId: '@test3:example.com', + callId: '1111', + backend: MeshBackend(), + deviceId: '1111', + expiresTs: DateTime.now() + .subtract(Duration(hours: 1)) + .millisecondsSinceEpoch, + roomId: room.id, + membershipId: voip.currentSessionId, + ).toJson(), + ] + }, + type: EventTypes.GroupCallMember, + eventId: 'asdfasdf', + senderId: '@test3:example.com', + originServerTs: DateTime.now().subtract(Duration(hours: 1)), + room: room, + stateKey: '@test3:example.com', + ), + ); + expect(room.getFamedlyCallEvents().entries.elementAt(0).key, + '@test3:example.com'); + expect(room.getFamedlyCallEvents().entries.elementAt(1).key, + '@test2:example.com'); + expect(room.getFamedlyCallEvents().entries.elementAt(2).key, + '@test2.0:example.com'); + expect(room.getFamedlyCallEvents().entries.elementAt(3).key, + '@test1:example.com'); + expect(room.getCallMembershipsFromRoom().entries.elementAt(0).key, + '@test3:example.com'); + expect(room.getCallMembershipsFromRoom().entries.elementAt(1).key, + '@test2:example.com'); + expect(room.getCallMembershipsFromRoom().entries.elementAt(2).key, + '@test2.0:example.com'); + expect(room.getCallMembershipsFromRoom().entries.elementAt(3).key, + '@test1:example.com'); + }); + + test('Enabling group calls', () async { + // users default is 0 and so group calls not enabled + room.setState( + Event( + senderId: '@test:example.com', + type: 'm.room.power_levels', + room: room, + eventId: '123a', + content: { + 'events': {EventTypes.GroupCallMember: 100}, + 'state_default': 50, + 'users_default': 0 + }, + originServerTs: DateTime.now(), + stateKey: '', + ), + ); + expect(room.canJoinGroupCall, false); + expect(room.groupCallsEnabledForEveryone, false); + + room.setState( + Event( + senderId: '@test:example.com', + type: 'm.room.power_levels', + room: room, + eventId: '123a', + content: { + 'events': {EventTypes.GroupCallMember: 27}, + 'state_default': 50, + 'users_default': 49 + }, + originServerTs: DateTime.now(), + stateKey: '', + ), + ); + expect(room.canJoinGroupCall, true); + expect(room.groupCallsEnabledForEveryone, true); + + // state_default 50 and user_default 0, use enableGroupCall + room.setState( + Event( + senderId: '@test:example.com', + type: 'm.room.power_levels', + room: room, + eventId: '123', + content: { + 'state_default': 50, + 'users': {'@test:fakeServer.notExisting': 100}, + 'users_default': 0 + }, + originServerTs: DateTime.now(), + stateKey: ''), + ); + expect(room.canJoinGroupCall, true); // because admin + expect(room.groupCallsEnabledForEveryone, false); + await room.enableGroupCalls(); + expect(room.canJoinGroupCall, true); + expect(room.groupCallsEnabledForEveryone, true); + + // state_default 50 and user_default unspecified, use enableGroupCall + room.setState( + Event( + senderId: '@test:example.com', + type: 'm.room.power_levels', + room: room, + eventId: '123', + content: { + 'state_default': 50, + 'users': {'@test:fakeServer.notExisting': 100}, + }, + originServerTs: DateTime.now(), + stateKey: '', + ), + ); + + expect(room.canJoinGroupCall, true); // because admin + expect(room.groupCallsEnabledForEveryone, false); + await room.enableGroupCalls(); + expect(room.canJoinGroupCall, true); + expect(room.groupCallsEnabledForEveryone, true); + + // state_default is 0 so users should be able to send state events + room.setState( + Event( + senderId: '@test:example.com', + type: 'm.room.power_levels', + room: room, + eventId: '123', + content: { + 'state_default': 0, + 'users': {'@test:fakeServer.notExisting': 100}, + }, + originServerTs: DateTime.now(), + stateKey: '', + ), + ); + expect(room.canJoinGroupCall, true); + expect(room.groupCallsEnabledForEveryone, true); + }); + + test('group call participants count', () { + room.setState( + Event( + senderId: '@test1:example.com', + type: EventTypes.GroupCallMember, + room: room, + eventId: '1234177', + content: { + 'memberships': [ + CallMembership( + userId: '@test1:example.com', + callId: 'participants_count', + backend: MeshBackend(), + deviceId: '1111', + expiresTs: DateTime.now() + .subtract(Duration(hours: 1)) + .millisecondsSinceEpoch, + roomId: room.id, + membershipId: voip.currentSessionId, + ).toJson(), + ] + }, + originServerTs: DateTime.now(), + stateKey: '@test1:example.com'), + ); + + expect(room.groupCallParticipantCount('participants_count'), 0); + expect(room.hasActiveGroupCall, false); + + room.setState( + Event( + senderId: '@test2:example.com', + type: EventTypes.GroupCallMember, + room: room, + eventId: '1234177', + content: { + 'memberships': [ + CallMembership( + userId: '@test2:example.com', + callId: 'participants_count', + backend: MeshBackend(), + deviceId: '1111', + expiresTs: DateTime.now() + .add(Duration(hours: 1)) + .millisecondsSinceEpoch, + roomId: room.id, + membershipId: voip.currentSessionId, + ).toJson(), + ] + }, + originServerTs: DateTime.now(), + stateKey: '@test2:example.com'), + ); + expect(room.groupCallParticipantCount('participants_count'), 1); + expect(room.hasActiveGroupCall, true); + + room.setState( + Event( + senderId: '@test3:example.com', + type: EventTypes.GroupCallMember, + room: room, + eventId: '1231234124123', + content: { + 'memberships': [ + CallMembership( + userId: '@test3:example.com', + callId: 'participants_count', + backend: MeshBackend(), + deviceId: '1111', + expiresTs: DateTime.now().millisecondsSinceEpoch, + roomId: room.id, + membershipId: voip.currentSessionId, + ).toJson(), + ] + }, + originServerTs: DateTime.now(), + stateKey: '@test3:example.com'), + ); + + expect(room.groupCallParticipantCount('participants_count'), 2); + expect(room.hasActiveGroupCall, true); }); }); } diff --git a/test/fake_matrix_api.dart b/test/fake_matrix_api.dart index f8e993e2..355207f8 100644 --- a/test/fake_matrix_api.dart +++ b/test/fake_matrix_api.dart @@ -2581,6 +2581,10 @@ class FakeMatrixApi extends BaseClient { (var reqI) => { 'event_id': '42', }, + '/client/v3/rooms/!calls%3Aexample.com/state/m.room.power_levels': + (var reqI) => { + 'event_id': '42', + }, '/client/v3/directory/list/room/!localpart%3Aexample.com': (var req) => {}, '/client/v3/room_keys/version/5': (var req) => {}, @@ -2600,11 +2604,7 @@ class FakeMatrixApi extends BaseClient { }, '/client/unstable/org.matrix.msc3814.v1/dehydrated_device': (var _) => { 'device_id': 'DEHYDDEV', - }, - '/client/v3/rooms/${Uri.encodeComponent("!localpart:server.abc")}/state/${Uri.encodeComponent("org.matrix.msc3401.call")}/${Uri.encodeComponent("1675856324414gzczMtfzTk0DKgEw")}': - (var req) => { - 'event_id': 'groupCall', - }, + } }, 'DELETE': { '/unknown/token': (var req) => {'errcode': 'M_UNKNOWN_TOKEN'}, diff --git a/test/room_test.dart b/test/room_test.dart index 104331c7..7d8e066a 100644 --- a/test/room_test.dart +++ b/test/room_test.dart @@ -647,179 +647,10 @@ void main() { expect(room.canChangeStateEvent('m.room.power_levels'), false); expect(room.canChangeStateEvent('m.room.member'), false); expect(room.canSendEvent('m.room.message'), true); - final resp = await room.setPower('@test:fakeServer.notExisting', 90); + final resp = await room.setPower('@test:fakeServer.notExisting', 0); expect(resp, '42'); }); - test('Enabling group calls', () async { - expect(room.groupCallsEnabled, false); - - // users default is 0 and so group calls not enabled - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123a', - content: { - 'events': {EventTypes.GroupCallMemberPrefix: 100}, - 'state_default': 50, - 'users_default': 0 - }, - originServerTs: DateTime.now(), - stateKey: '', - ), - ); - expect(room.groupCallsEnabled, false); - - // one of the group call permissions is unspecified in events override - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123a', - content: { - 'events': {EventTypes.GroupCallMemberPrefix: 27}, - 'state_default': 50, - 'users_default': 49 - }, - originServerTs: DateTime.now(), - stateKey: '', - ), - ); - expect(room.groupCallsEnabled, false); - - // only override one of the group calls permission, other one still less - // than users_default and state_default - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123a', - content: { - 'events': { - EventTypes.GroupCallMemberPrefix: 27, - EventTypes.GroupCallPrefix: 0 - }, - 'state_default': 50, - 'users_default': 2 - }, - originServerTs: DateTime.now(), - stateKey: '', - ), - ); - expect(room.groupCallsEnabled, false); - expect(room.canJoinGroupCall, false); - expect(room.canCreateGroupCall, false); - - // state_default 50 and user_default 26, but override evnents present - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123a', - content: { - 'events': { - EventTypes.GroupCallMemberPrefix: 25, - EventTypes.GroupCallPrefix: 25 - }, - 'state_default': 50, - 'users_default': 26 - }, - originServerTs: DateTime.now(), - stateKey: '', - ), - ); - expect(room.groupCallsEnabled, true); - expect(room.canJoinGroupCall, true); - expect(room.canCreateGroupCall, true); - - // state_default 50 and user_default 0, use enableGroupCall - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123', - content: { - 'state_default': 50, - 'users': {'@test:fakeServer.notExisting': 100}, - 'users_default': 0 - }, - originServerTs: DateTime.now(), - stateKey: ''), - ); - expect(room.groupCallsEnabled, false); - expect(room.canJoinGroupCall, false); - expect(room.canCreateGroupCall, false); - await room.enableGroupCalls(); - expect(room.groupCallsEnabled, true); - - // state_default 50 and user_default unspecified, use enableGroupCall - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123', - content: { - 'state_default': 50, - 'users': {'@test:fakeServer.notExisting': 100}, - }, - originServerTs: DateTime.now(), - stateKey: '', - ), - ); - await room.enableGroupCalls(); - expect(room.groupCallsEnabled, true); - expect(room.canJoinGroupCall, true); - expect(room.canCreateGroupCall, true); - - // state_default is 0 so users should be able to send state events - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123', - content: { - 'state_default': 0, - 'users': {'@test:fakeServer.notExisting': 100}, - }, - originServerTs: DateTime.now(), - stateKey: '', - ), - ); - expect(room.groupCallsEnabled, true); - expect(room.canJoinGroupCall, true); - expect(room.canCreateGroupCall, true); - room.setState( - Event( - senderId: '@test:example.com', - type: 'm.room.power_levels', - room: room, - eventId: '123abc', - content: { - 'ban': 50, - 'events': {'m.room.name': 0, 'm.room.power_levels': 100}, - 'events_default': 0, - 'invite': 50, - 'kick': 50, - 'notifications': {'room': 20}, - 'redact': 50, - 'state_default': 50, - 'users': {}, - 'users_default': 0 - }, - originServerTs: DateTime.now(), - stateKey: '', - ), - ); - }); - test('invite', () async { await room.invite('Testname'); }); @@ -1375,7 +1206,7 @@ void main() { expect(room.isSpace, true); expect(room.spaceParents.isEmpty, true); - room.states[EventTypes.spaceParent] = { + room.states[EventTypes.SpaceParent] = { '!1234:example.invalid': Event.fromJson( { 'content': { @@ -1385,7 +1216,7 @@ void main() { 'origin_server_ts': 1432735824653, 'room_id': '!jEsUZKDJdhlrceRyVU:example.org', 'sender': '@example:example.org', - 'type': EventTypes.spaceParent, + 'type': EventTypes.SpaceParent, 'unsigned': {'age': 1234}, 'state_key': '!1234:example.invalid', }, @@ -1395,7 +1226,7 @@ void main() { expect(room.spaceParents.length, 1); expect(room.spaceChildren.isEmpty, true); - room.states[EventTypes.spaceChild] = { + room.states[EventTypes.SpaceChild] = { '!b:example.invalid': Event.fromJson( { 'content': { @@ -1406,7 +1237,7 @@ void main() { 'origin_server_ts': 1432735824653, 'room_id': '!jEsUZKDJdhlrceRyVU:example.org', 'sender': '@example:example.org', - 'type': EventTypes.spaceChild, + 'type': EventTypes.SpaceChild, 'unsigned': {'age': 1234}, 'state_key': '!b:example.invalid', }, @@ -1422,7 +1253,7 @@ void main() { 'origin_server_ts': 1432735824653, 'room_id': '!jEsUZKDJdhlrceRyVU:example.org', 'sender': '@example:example.org', - 'type': EventTypes.spaceChild, + 'type': EventTypes.SpaceChild, 'unsigned': {'age': 1234}, 'state_key': '!c:example.invalid', }, @@ -1437,7 +1268,7 @@ void main() { 'origin_server_ts': 1432735824653, 'room_id': '!jEsUZKDJdhlrceRyVU:example.org', 'sender': '@example:example.org', - 'type': EventTypes.spaceChild, + 'type': EventTypes.SpaceChild, 'unsigned': {'age': 1234}, 'state_key': '!noorder:example.invalid', }, @@ -1453,7 +1284,7 @@ void main() { 'origin_server_ts': 1432735824653, 'room_id': '!jEsUZKDJdhlrceRyVU:example.org', 'sender': '@example:example.org', - 'type': EventTypes.spaceChild, + 'type': EventTypes.SpaceChild, 'unsigned': {'age': 1234}, 'state_key': '!a:example.invalid', }, @@ -1485,7 +1316,6 @@ void main() { test('inviteLink', () async { // ensure we don't rerequest members room.summary.mJoinedMemberCount = 4; - var matrixToLink = await room.matrixToInviteLink(); expect(matrixToLink.toString(), 'https://matrix.to/#/%23testalias%3Aexample.com'); @@ -1504,181 +1334,6 @@ void main() { 'https://matrix.to/#/!localpart%3Aserver.abc?via=example.com&via=test.abc&via=example.org'); }); - test('callMemberStateIsExpired', () { - expect( - room.callMemberStateForIdIsExpired( - Event( - senderId: '@test:example.com', - type: EventTypes.GroupCallMemberPrefix, - room: room, - eventId: '1231234124', - content: { - 'm.calls': [ - { - 'm.call_id': '1674811248673789288k7d60n5976', - 'm.devices': [ - { - 'device_id': 'ZEEGCGPTGI', - 'session_id': 'cbAtVZdLBnJq', - 'm.expires_ts': 1674813039415, - 'feeds': [ - {'purpose': 'm.usermedia'} - ] - } - ] - }, - ], - }, - originServerTs: DateTime.now(), - stateKey: ''), - '1674811248673789288k7d60n5976'), - true); - expect( - room.callMemberStateForIdIsExpired( - Event( - senderId: '@test:example.com', - type: EventTypes.GroupCallMemberPrefix, - room: room, - eventId: '1231234124', - content: { - 'm.calls': [ - { - 'm.call_id': '1674811256006mfqnmsAbzqxjYtWZ', - 'm.devices': [ - { - 'device_id': 'ZEEGCGPTGI', - 'session_id': 'fhovqxwcasdfr', - 'expires_ts': DateTime.now() - .add(Duration(minutes: 1)) - .millisecondsSinceEpoch, - 'feeds': [ - {'purpose': 'm.usermedia'} - ] - } - ] - } - ], - }, - originServerTs: DateTime.now(), - stateKey: ''), - '1674811256006mfqnmsAbzqxjYtWZ'), - false); - }); - - test('group call participants count', () { - room.setState( - Event( - senderId: '@test:example.com', - type: EventTypes.GroupCallMemberPrefix, - room: room, - eventId: '1234177', - content: { - 'm.calls': [ - { - 'm.call_id': '1674811256006mfqnmsAbzqxjYtWZ', - 'm.devices': [ - { - 'device_id': 'ZEEGCGPTGI', - 'session_id': 'fhovqxwcasdfr', - 'expires_ts': DateTime.now() - .add(Duration(minutes: 1)) - .millisecondsSinceEpoch, - 'feeds': [ - {'purpose': 'm.usermedia'} - ] - }, - ] - } - ], - }, - originServerTs: DateTime.now(), - stateKey: '@test:example.com'), - ); - room.setState( - Event( - senderId: '@test0:example.com', - type: EventTypes.GroupCallMemberPrefix, - room: room, - eventId: '1234177', - content: { - 'm.calls': [ - { - 'm.call_id': '1674811256006mfqnmsAbzqxjYtWZ', - 'm.devices': [ - { - 'device_id': 'ZEEGCGPTGI', - 'session_id': 'fhovqxwcasdfr', - 'expires_ts': DateTime.now() - .add(Duration(minutes: 2)) - .millisecondsSinceEpoch, - 'feeds': [ - {'purpose': 'm.usermedia'} - ] - }, - ] - } - ], - }, - originServerTs: DateTime.now(), - stateKey: '@test0:example.com'), - ); - room.setState( - Event( - senderId: '@test2:example.com', - type: EventTypes.GroupCallMemberPrefix, - room: room, - eventId: '1231234124123', - content: { - 'm.calls': [ - { - 'm.call_id': '1674811256006mfqnmsAbzqxjYtWZ', - 'm.devices': [ - { - 'device_id': 'ZEEGCGPTGI', - 'session_id': 'fhovqxwcasdfr', - 'feeds': [ - {'purpose': 'm.usermedia'} - ] - }, - ] - } - ], - }, - originServerTs: DateTime.now(), - stateKey: '@test2:example.com'), - ); - room.setState( - Event( - senderId: '@test3:example.com', - type: EventTypes.GroupCallMemberPrefix, - room: room, - eventId: '123123412445', - content: { - 'm.calls': [ - { - 'm.call_id': '1674811256006mfqnmsAbzqxjYtWZ', - 'm.devices': [ - { - 'device_id': 'ZEEGCGPTGI', - 'session_id': 'fhovqxwcasdfr', - 'expires_ts': DateTime.now() - .subtract(Duration(minutes: 1)) - .millisecondsSinceEpoch, - 'feeds': [ - {'purpose': 'm.usermedia'} - ] - }, - ] - } - ], - }, - originServerTs: DateTime.now(), - stateKey: '@test3:example.com'), - ); - expect( - room.groupCallParticipantCount('1674811256006mfqnmsAbzqxjYtWZ'), 2); - }); - test('EventTooLarge on exceeding max PDU size', () async { try { await room.sendTextEvent(''' diff --git a/test/webrtc_stub.dart b/test/webrtc_stub.dart index 9c61a072..18c3d0b9 100644 --- a/test/webrtc_stub.dart +++ b/test/webrtc_stub.dart @@ -6,7 +6,6 @@ import 'package:matrix/matrix.dart'; class MockWebRTCDelegate implements WebRTCDelegate { @override - // TODO: implement canHandleNewCall bool get canHandleNewCall => true; @override @@ -16,18 +15,13 @@ class MockWebRTCDelegate implements WebRTCDelegate { ]) async => MockRTCPeerConnection(); - @override - VideoRenderer createRenderer() { - return MockVideoRenderer(); - } - @override Future handleCallEnded(CallSession session) async { Logs().i('handleCallEnded called in MockWebRTCDelegate'); } @override - Future handleGroupCallEnded(GroupCall groupCall) async { + Future handleGroupCallEnded(GroupCallSession groupCall) async { Logs().i('handleGroupCallEnded called in MockWebRTCDelegate'); } @@ -42,7 +36,7 @@ class MockWebRTCDelegate implements WebRTCDelegate { } @override - Future handleNewGroupCall(GroupCall groupCall) async { + Future handleNewGroupCall(GroupCallSession groupCall) async { Logs().i('handleNewGroupCall called in MockWebRTCDelegate'); } @@ -61,6 +55,27 @@ class MockWebRTCDelegate implements WebRTCDelegate { Future stopRingtone() async { Logs().i('stopRingtone called in MockWebRTCDelegate'); } + + @override + EncryptionKeyProvider? get keyProvider => MockEncryptionKeyProvider(); +} + +class MockEncryptionKeyProvider implements EncryptionKeyProvider { + @override + Future onSetEncryptionKey( + CallParticipant participant, Uint8List key, int index) async { + Logs().i('Mock onSetEncryptionKey called for ${participant.id} '); + } + + @override + Future onExportKey(CallParticipant participant, int index) { + throw UnimplementedError(); + } + + @override + Future onRatchetKey(CallParticipant participant, int index) { + throw UnimplementedError(); + } } class MockMediaDevices implements MediaDevices { @@ -69,25 +84,21 @@ class MockMediaDevices implements MediaDevices { @override Future> enumerateDevices() { - // TODO: implement enumerateDevices throw UnimplementedError(); } @override Future getDisplayMedia(Map mediaConstraints) { - // TODO: implement getDisplayMedia throw UnimplementedError(); } @override Future getSources() { - // TODO: implement getSources throw UnimplementedError(); } @override MediaTrackSupportedConstraints getSupportedConstraints() { - // TODO: implement getSupportedConstraints throw UnimplementedError(); } @@ -99,7 +110,6 @@ class MockMediaDevices implements MediaDevices { @override Future selectAudioOutput([AudioOutputOptions? options]) { - // TODO: implement selectAudioOutput throw UnimplementedError(); } } @@ -347,15 +357,12 @@ class MockRTCPeerConnection implements RTCPeerConnection { } @override - // TODO: implement receivers Future> get receivers => throw UnimplementedError(); @override - // TODO: implement senders Future> get senders => throw UnimplementedError(); @override - // TODO: implement transceivers Future> get transceivers => throw UnimplementedError(); } @@ -415,65 +422,53 @@ class MockRTCRtpTransceiver implements RTCRtpTransceiver { } @override - // TODO: implement stoped bool get stoped => throw UnimplementedError(); } class MockRTCRtpSender implements RTCRtpSender { @override Future dispose() { - // TODO: implement dispose throw UnimplementedError(); } @override - // TODO: implement dtmfSender RTCDTMFSender get dtmfSender => throw UnimplementedError(); @override Future> getStats() { - // TODO: implement getStats throw UnimplementedError(); } @override - // TODO: implement ownsTrack bool get ownsTrack => throw UnimplementedError(); @override - // TODO: implement parameters RTCRtpParameters get parameters => throw UnimplementedError(); @override Future replaceTrack(MediaStreamTrack? track) { - // TODO: implement replaceTrack throw UnimplementedError(); } @override - // TODO: implement senderId String get senderId => throw UnimplementedError(); @override Future setParameters(RTCRtpParameters parameters) { - // TODO: implement setParameters throw UnimplementedError(); } @override Future setStreams(List streams) { - // TODO: implement setStreams throw UnimplementedError(); } @override Future setTrack(MediaStreamTrack? track, {bool takeOwnership = true}) { - // TODO: implement setTrack throw UnimplementedError(); } @override - // TODO: implement track MediaStreamTrack? get track => throw UnimplementedError(); // Mock implementation for RTCRtpSender } @@ -485,20 +480,16 @@ class MockRTCRtpReceiver implements RTCRtpReceiver { @override Future> getStats() { - // TODO: implement getStats throw UnimplementedError(); } @override - // TODO: implement parameters RTCRtpParameters get parameters => throw UnimplementedError(); @override - // TODO: implement receiverId String get receiverId => throw UnimplementedError(); @override - // TODO: implement track MediaStreamTrack? get track => throw UnimplementedError(); // Mock implementation for RTCRtpReceiver }