diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c41a26d..9db07699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +## [0.7.0-nullsafety.10] - 26nd Nov 2021 +- feat: Migrate olm sessions on database migration +- chore: Enable E2EE recovery by default + +## [0.7.0-nullsafety.9] - 25nd Nov 2021 +- fix: Limited timeline clean up on web +- fix: Remove account avatar + +## [0.7.0-nullsafety.8] - 24nd Nov 2021 +- chore: Update FluffyBox + +## [0.7.0-nullsafety.7] - 23nd Nov 2021 +- feat: Add commands to create chats +- feat: Add clear cache command +- feat: Implement new FluffyBox database API implementation +- fix: Workaround for a null exception for a non nullable boolean while user device key updating +- fix: Limited timeline clears too many events +- fix: Ability to remove avatar from room and account +- fix: Request history in archived rooms +- fix: Decrypt last event of a room +- refactor: Remove Sembast database implementation + +## [0.7.0-nullsafety.6] - 16nd Nov 2021 +- feat: Implement sembast store +- fix: HtmlToText crashes with an empty code block +- fix: use originServerTs to check if state event is old +- fix: Dont enable e2ee in new direct chats without encryption support +- fix: Change eventstatus of edits in prevEvent +- chore: Trim formatted username fallback + +## [0.7.0-nullsafety.5] - 10nd Nov 2021 +- fix: Edits as lastEvent do not update +- fix: JSON parsing in decryptRoomEvent method +- fix: Wrong null check in hive database +- fix: crash on invalid displaynames +- chore: Update matrix_api_lite + ## [0.7.0-nullsafety.4] - 09nd Nov 2021 - feat: More advanced create chat methods (encryption is now enabled by default) - feat: Make waiting on init db optional diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index b40a4d76..f49a175c 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -33,7 +33,6 @@ import 'utils/bootstrap.dart'; class Encryption { final Client client; final bool debug; - final bool enableE2eeRecovery; bool get enabled => olmManager.enabled; @@ -53,7 +52,6 @@ class Encryption { Encryption({ required this.client, this.debug = false, - required this.enableE2eeRecovery, }) { ssss = SSSS(this); keyManager = KeyManager(this); @@ -232,8 +230,7 @@ class Encryption { decryptedPayload = json.decode(decryptResult.plaintext); } catch (exception) { // alright, if this was actually by our own outbound group session, we might as well clear it - if (client.enableE2eeRecovery && - exception.toString() != DecryptException.unknownSession && + if (exception.toString() != DecryptException.unknownSession && (keyManager .getOutboundGroupSession(roomId) ?.outboundGroupSession diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index e2229f45..8e182ab0 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -182,7 +182,7 @@ class KeyManager { // attempt to decrypt the last event final event = room.getState(EventTypes.Encrypted); if (event != null && event.content['session_id'] == sessionId) { - encryption.decryptRoomEvent(roomId, event, store: true); + room.setState(encryption.decryptRoomEventSync(roomId, event)); } // and finally broadcast the new session room.onSessionKeyReceived.add(sessionId); @@ -219,8 +219,7 @@ class KeyManager { void maybeAutoRequest(String roomId, String sessionId, String senderKey) { final room = client.getRoomById(roomId); final requestIdent = '$roomId|$sessionId|$senderKey'; - if (client.enableE2eeRecovery && - room != null && + if (room != null && !_requestedSessionIds.contains(requestIdent) && !client.isUnknownSession) { // do e2ee recovery diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index 155be967..05478d18 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -514,10 +514,10 @@ class OlmManager { return _decryptToDeviceEvent(event); } catch (_) { // okay, the thing errored while decrypting. It is safe to assume that the olm session is corrupt and we should generate a new one - if (client.enableE2eeRecovery) { - // ignore: unawaited_futures - runInRoot(() => restoreOlmSession(event.senderId, senderKey)); - } + + // ignore: unawaited_futures + runInRoot(() => restoreOlmSession(event.senderId, senderKey)); + rethrow; } } diff --git a/lib/matrix.dart b/lib/matrix.dart index 71c2671d..24a0c9ef 100644 --- a/lib/matrix.dart +++ b/lib/matrix.dart @@ -24,6 +24,7 @@ export 'package:matrix_api_lite/matrix_api_lite.dart'; export 'src/client.dart'; export 'src/database/database_api.dart'; export 'src/database/hive_database.dart'; +export 'src/database/fluffybox_database.dart'; export 'src/event.dart'; export 'src/event_status.dart'; export 'src/voip.dart'; diff --git a/lib/src/client.dart b/lib/src/client.dart index cc46ce78..0aa483f5 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -74,8 +74,6 @@ class Client extends MatrixApi { DatabaseApi? get database => _database; - bool enableE2eeRecovery; - @deprecated MatrixApi get api => this; @@ -120,7 +118,6 @@ class Client extends MatrixApi { /// [databaseBuilder]: A function that creates the database instance, that will be used. /// [legacyDatabaseBuilder]: Use this for your old database implementation to perform an automatic migration /// [databaseDestroyer]: A function that can be used to destroy a database instance, for example by deleting files from disk. - /// [enableE2eeRecovery]: Enable additional logic to try to recover from bad e2ee sessions /// [verificationMethods]: A set of all the verification methods this client can handle. Includes: /// KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported /// KeyVerificationMethod.emoji: Compare emojis @@ -157,7 +154,8 @@ class Client extends MatrixApi { this.databaseDestroyer, this.legacyDatabaseBuilder, this.legacyDatabaseDestroyer, - this.enableE2eeRecovery = false, + @Deprecated('This is now always enabled by default.') + bool? enableE2eeRecovery, Set? verificationMethods, http.Client? httpClient, Set? importantStateEvents, @@ -568,7 +566,8 @@ class Client extends MatrixApi { final directChatRoomId = getDirectChatFromUserId(mxid); if (directChatRoomId != null) return directChatRoomId; - enableEncryption ??= await userOwnsEncryptionKeys(mxid); + enableEncryption ??= + encryptionEnabled && await userOwnsEncryptionKeys(mxid); if (enableEncryption) { initialState ??= []; if (!initialState.any((s) => s.type == EventTypes.Encryption)) { @@ -609,6 +608,7 @@ class Client extends MatrixApi { List? invite, CreateRoomPreset preset = CreateRoomPreset.privateChat, List? initialState, + Visibility? visibility, bool waitForSync = true, }) async { enableEncryption ??= @@ -629,6 +629,7 @@ class Client extends MatrixApi { preset: preset, name: groupName, initialState: initialState, + visibility: visibility, ); if (waitForSync) { @@ -649,7 +650,7 @@ class Client extends MatrixApi { return true; } final keys = await queryKeys({userId: []}); - return keys.deviceKeys?.isNotEmpty ?? false; + return keys.deviceKeys?[userId]?.isNotEmpty ?? false; } /// Creates a new space and returns the Room ID. The parameters are mostly @@ -772,6 +773,7 @@ class Client extends MatrixApi { leftRoom, )); }); + leftRoom.prev_batch = room.timeline?.prevBatch; room.state?.forEach((event) { leftRoom.setState(Event.fromMatrixEvent( event, @@ -817,8 +819,15 @@ class Client extends MatrixApi { } } - /// Uploads a new user avatar for this user. - Future setAvatar(MatrixFile file) async { + /// Uploads a new user avatar for this user. Leave file null to remove the + /// current avatar. + Future setAvatar(MatrixFile? file) async { + if (file == null) { + // We send an empty String to remove the avatar. Sending Null **should** + // work but it doesn't with Synapse. See: + // https://gitlab.com/famedly/company/frontend/famedlysdk/-/issues/254 + return setAvatarUrl(userID!, Uri.parse('')); + } final uploadResp = await uploadContent( file.bytes, filename: file.name, @@ -1063,8 +1072,7 @@ class Client extends MatrixApi { // make sure to throw an exception if libolm doesn't exist await olm.init(); olm.get_library_version(); - encryption = - Encryption(client: this, enableE2eeRecovery: enableE2eeRecovery); + encryption = Encryption(client: this); } catch (_) { encryption?.dispose(); encryption = null; @@ -1806,7 +1814,13 @@ class Client extends MatrixApi { final deviceKeysList = _userDeviceKeys[userId] ??= DeviceKeysList(userId, this); final failure = _keyQueryFailures[userId.domain]; - if (deviceKeysList.outdated && + + // deviceKeysList.outdated is not nullable but we have seen this error + // in production: `Failed assertion: boolean expression must not be null` + // So this could either be a null safety bug in Dart or a result of + // using unsound null safety. The extra equal check `!= false` should + // save us here. + if (deviceKeysList.outdated != false && (failure == null || DateTime.now() .subtract(Duration(minutes: 5)) @@ -2381,6 +2395,24 @@ class Client extends MatrixApi { ); } } + Logs().d('Migrate OLM sessions...'); + try { + final olmSessions = await legacyDatabase.getAllOlmSessions(); + for (final identityKey in olmSessions.keys) { + final sessions = olmSessions[identityKey]!; + for (final sessionId in sessions.keys) { + final session = sessions[sessionId]!; + await database.storeOlmSession( + identityKey, + session['session_id'] as String, + session['pickle'] as String, + session['last_received'] as int, + ); + } + } + } catch (e, s) { + Logs().e('Unable to migrate OLM sessions!', e, s); + } Logs().d('Migrate Device Keys...'); final userDeviceKeys = await legacyDatabase.getUserDeviceKeys(this); for (final userId in userDeviceKeys.keys) { diff --git a/lib/src/database/database_api.dart b/lib/src/database/database_api.dart index 368f1ce8..2e3b754e 100644 --- a/lib/src/database/database_api.dart +++ b/lib/src/database/database_api.dart @@ -67,8 +67,6 @@ abstract class DatabaseApi { Future getEventById(String eventId, Room room); - bool eventIsKnown(String eventId, String roomId); - Future forgetRoom(String roomId); Future clearCache(); @@ -272,6 +270,8 @@ abstract class DatabaseApi { String userId, ); + Future> getAllOlmSessions(); + Future> getOlmSessionsForDevices( List identityKeys, String userId, diff --git a/lib/src/database/fluffybox_database.dart b/lib/src/database/fluffybox_database.dart new file mode 100644 index 00000000..05c4eb5f --- /dev/null +++ b/lib/src/database/fluffybox_database.dart @@ -0,0 +1,1479 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2019, 2020, 2021 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import 'dart:async'; +import 'dart:convert'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:fluffybox/fluffybox.dart'; +import 'package:hive/hive.dart' show HiveCipher; +import 'package:matrix/encryption/utils/olm_session.dart'; +import 'package:matrix/encryption/utils/outbound_group_session.dart'; +import 'package:matrix/encryption/utils/ssss_cache.dart'; +import 'package:matrix/encryption/utils/stored_inbound_group_session.dart'; +import 'package:matrix/matrix.dart'; +import 'package:matrix/src/event_status.dart'; +import 'package:matrix/src/utils/queued_to_device_event.dart'; +import 'package:matrix/src/utils/run_benchmarked.dart'; + +/// This database does not support file caching! +class FluffyBoxDatabase extends DatabaseApi { + static const int version = 6; + final String name; + final String path; + final HiveCipher? key; + late final BoxCollection _collection; + late Box _clientBox; + late Box _accountDataBox; + late Box _roomsBox; + late Box _toDeviceQueueBox; + + /// Key is a tuple as TupleKey(roomId, type) where stateKey can be + /// an empty string. + late Box _roomStateBox; + + /// Key is a tuple as TupleKey(roomId, userId) + late Box _roomMembersBox; + + /// Key is a tuple as TupleKey(roomId, type) + late Box _roomAccountDataBox; + late Box _inboundGroupSessionsBox; + late Box _outboundGroupSessionsBox; + late Box _olmSessionsBox; + + /// Key is a tuple as TupleKey(userId, deviceId) + late Box _userDeviceKeysBox; + + /// Key is the user ID as a String + late Box _userDeviceKeysOutdatedBox; + + /// Key is a tuple as TupleKey(userId, publicKey) + late Box _userCrossSigningKeysBox; + late Box _ssssCacheBox; + late Box _presencesBox; + + /// Key is a tuple as Multikey(roomId, fragmentId) while the default + /// fragmentId is an empty String + late Box _timelineFragmentsBox; + + /// Key is a tuple as TupleKey(roomId, eventId) + late Box _eventsBox; + + /// Key is a tuple as TupleKey(userId, deviceId) + late Box _seenDeviceIdsBox; + + late Box _seenDeviceKeysBox; + + String get _clientBoxName => 'box_client'; + + String get _accountDataBoxName => 'box_account_data'; + + String get _roomsBoxName => 'box_rooms'; + + String get _toDeviceQueueBoxName => 'box_to_device_queue'; + + String get _roomStateBoxName => 'box_room_states'; + + String get _roomMembersBoxName => 'box_room_members'; + + String get _roomAccountDataBoxName => 'box_room_account_data'; + + String get _inboundGroupSessionsBoxName => 'box_inbound_group_session'; + + String get _outboundGroupSessionsBoxName => 'box_outbound_group_session'; + + String get _olmSessionsBoxName => 'box_olm_session'; + + String get _userDeviceKeysBoxName => 'box_user_device_keys'; + + String get _userDeviceKeysOutdatedBoxName => 'box_user_device_keys_outdated'; + + String get _userCrossSigningKeysBoxName => 'box_cross_signing_keys'; + + String get _ssssCacheBoxName => 'box_ssss_cache'; + + String get _presencesBoxName => 'box_presences'; + + String get _timelineFragmentsBoxName => 'box_timeline_fragments'; + + String get _eventsBoxName => 'box_events'; + + String get _seenDeviceIdsBoxName => 'box_seen_device_ids'; + + String get _seenDeviceKeysBoxName => 'box_seen_device_keys'; + + FluffyBoxDatabase(this.name, this.path, {this.key}); + + @override + int get maxFileSize => 0; + + Future open() async { + _collection = await BoxCollection.open( + name, + { + _clientBoxName, + _accountDataBoxName, + _roomsBoxName, + _toDeviceQueueBoxName, + _roomStateBoxName, + _roomMembersBoxName, + _roomAccountDataBoxName, + _inboundGroupSessionsBoxName, + _outboundGroupSessionsBoxName, + _olmSessionsBoxName, + _userDeviceKeysBoxName, + _userDeviceKeysOutdatedBoxName, + _userCrossSigningKeysBoxName, + _ssssCacheBoxName, + _presencesBoxName, + _timelineFragmentsBoxName, + _eventsBoxName, + _seenDeviceIdsBoxName, + _seenDeviceKeysBoxName, + }, + key: key, + path: path, + ); + _clientBox = await _collection.openBox( + _clientBoxName, + preload: true, + ); + _accountDataBox = await _collection.openBox( + _accountDataBoxName, + preload: true, + ); + _roomsBox = await _collection.openBox( + _roomsBoxName, + preload: true, + ); + _roomStateBox = await _collection.openBox( + _roomStateBoxName, + ); + _roomMembersBox = await _collection.openBox( + _roomMembersBoxName, + ); + _toDeviceQueueBox = await _collection.openBox( + _toDeviceQueueBoxName, + preload: true, + ); + _roomAccountDataBox = await _collection.openBox( + _roomAccountDataBoxName, + preload: true, + ); + _inboundGroupSessionsBox = await _collection.openBox( + _inboundGroupSessionsBoxName, + ); + _outboundGroupSessionsBox = await _collection.openBox( + _outboundGroupSessionsBoxName, + ); + _olmSessionsBox = await _collection.openBox( + _olmSessionsBoxName, + ); + _userDeviceKeysBox = await _collection.openBox( + _userDeviceKeysBoxName, + ); + _userDeviceKeysOutdatedBox = await _collection.openBox( + _userDeviceKeysOutdatedBoxName, + ); + _userCrossSigningKeysBox = await _collection.openBox( + _userCrossSigningKeysBoxName, + ); + _ssssCacheBox = await _collection.openBox( + _ssssCacheBoxName, + ); + _presencesBox = await _collection.openBox( + _presencesBoxName, + ); + _timelineFragmentsBox = await _collection.openBox( + _timelineFragmentsBoxName, + ); + _eventsBox = await _collection.openBox( + _eventsBoxName, + ); + _seenDeviceIdsBox = await _collection.openBox( + _seenDeviceIdsBoxName, + ); + _seenDeviceKeysBox = await _collection.openBox( + _seenDeviceKeysBoxName, + ); + + // Check version and check if we need a migration + final currentVersion = int.tryParse(await _clientBox.get('version') ?? ''); + if (currentVersion == null) { + await _clientBox.put('version', version.toString()); + } else if (currentVersion != version) { + await _migrateFromVersion(currentVersion); + } + + return; + } + + Future _migrateFromVersion(int currentVersion) async { + Logs().i('Migrate store database from version $currentVersion to $version'); + await clearCache(); + await _clientBox.put('version', version.toString()); + } + + @override + Future clear() => _collection.deleteFromDisk(); + + @override + Future clearCache() => transaction(() async { + await _roomsBox.clear(); + await _accountDataBox.clear(); + await _roomStateBox.clear(); + await _roomMembersBox.clear(); + await _eventsBox.clear(); + await _timelineFragmentsBox.clear(); + await _outboundGroupSessionsBox.clear(); + await _presencesBox.clear(); + await _clientBox.delete('prev_batch'); + }); + + @override + Future clearSSSSCache() => _ssssCacheBox.clear(); + + @override + Future close() async => _collection.close(); + + @override + Future deleteFromToDeviceQueue(int id) async { + await _toDeviceQueueBox.delete(id.toString()); + return; + } + + @override + Future deleteOldFiles(int savedAt) async { + return; + } + + @override + Future forgetRoom(String roomId) => transaction(() async { + await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString()); + final eventsBoxKeys = await _eventsBox.getAllKeys(); + for (final key in eventsBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _eventsBox.delete(key); + } + final roomStateBoxKeys = await _roomStateBox.getAllKeys(); + for (final key in roomStateBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomStateBox.delete(key); + } + final roomMembersBoxKeys = await _roomMembersBox.getAllKeys(); + for (final key in roomMembersBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomMembersBox.delete(key); + } + final roomAccountDataBoxKeys = await _roomAccountDataBox.getAllKeys(); + for (final key in roomAccountDataBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomAccountDataBox.delete(key); + } + await _roomsBox.delete(roomId); + }); + + @override + Future> getAccountData() => + runBenchmarked>('Get all account data from store', + () async { + final accountData = {}; + final raws = await _accountDataBox.getAllValues(); + for (final entry in raws.entries) { + accountData[entry.key] = BasicEvent( + type: entry.key, + content: copyMap(entry.value), + ); + } + return accountData; + }); + + @override + Future?> getClient(String name) => + runBenchmarked('Get Client from store', () async { + final map = {}; + final keys = await _clientBox.getAllKeys(); + for (final key in keys) { + if (key == 'version') continue; + final value = await _clientBox.get(key); + if (value != null) map[key] = value; + } + if (map.isEmpty) return null; + return map; + }); + + @override + Future getEventById(String eventId, Room room) async { + final raw = await _eventsBox.get(TupleKey(room.id, eventId).toString()); + if (raw == null) return null; + return Event.fromJson(copyMap(raw), room); + } + + /// Loads a whole list of events at once from the store for a specific room + Future> _getEventsByIds(List eventIds, Room room) async { + final keys = eventIds + .map( + (eventId) => TupleKey(room.id, eventId).toString(), + ) + .toList(); + final rawEvents = await _eventsBox.getAll(keys); + return rawEvents + .map((rawEvent) => Event.fromJson(copyMap(rawEvent!), room)) + .toList(); + } + + @override + Future> getEventList( + Room room, { + int start = 0, + int? limit, + }) => + runBenchmarked>('Get event list', () async { + // Get the synced event IDs from the store + final timelineKey = TupleKey(room.id, '').toString(); + final timelineEventIds = + (await _timelineFragmentsBox.get(timelineKey) ?? []); + + // Get the local stored SENDING events from the store + late final List sendingEventIds; + if (start != 0) { + sendingEventIds = []; + } else { + final sendingTimelineKey = TupleKey(room.id, 'SENDING').toString(); + sendingEventIds = + (await _timelineFragmentsBox.get(sendingTimelineKey) ?? []); + } + + // Combine those two lists while respecting the start and limit parameters. + final end = min(timelineEventIds.length, + start + (limit ?? timelineEventIds.length)); + final eventIds = sendingEventIds + + (start < timelineEventIds.length + ? timelineEventIds.getRange(start, end).toList() + : []); + + return await _getEventsByIds(eventIds.cast(), room); + }); + + @override + Future getFile(Uri mxcUri) async { + return null; + } + + @override + Future getInboundGroupSession( + String roomId, + String sessionId, + ) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) return null; + return StoredInboundGroupSession.fromJson(copyMap(raw)); + } + + @override + Future> + getInboundGroupSessionsToUpload() async { + final sessions = (await _inboundGroupSessionsBox.getAllValues()) + .values + .where((rawSession) => rawSession['uploaded'] == false) + .take(50) + .map( + (json) => StoredInboundGroupSession.fromJson( + copyMap(json), + ), + ) + .toList(); + return sessions; + } + + @override + Future> getLastSentMessageUserDeviceKey( + String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()); + if (raw == null) return []; + return [raw['last_sent_message']]; + } + + @override + Future storeOlmSession(String identityKey, String sessionId, + String pickle, int lastReceived) async { + final rawSessions = (await _olmSessionsBox.get(identityKey)) ?? {}; + rawSessions[sessionId] = { + 'identity_key': identityKey, + 'pickle': pickle, + 'session_id': sessionId, + 'last_received': lastReceived, + }; + await _olmSessionsBox.put(identityKey, rawSessions); + return; + } + + @override + Future> getOlmSessions( + String identityKey, String userId) async { + final rawSessions = await _olmSessionsBox.get(identityKey); + if (rawSessions == null || rawSessions.isEmpty) return []; + return rawSessions.values + .map((json) => OlmSession.fromJson(copyMap(json), userId)) + .toList(); + } + + @override + Future> getAllOlmSessions() => + _olmSessionsBox.getAllValues(); + + @override + Future> getOlmSessionsForDevices( + List identityKey, String userId) async { + final sessions = await Future.wait( + identityKey.map((identityKey) => getOlmSessions(identityKey, userId))); + return [for (final sublist in sessions) ...sublist]; + } + + @override + Future getOutboundGroupSession( + String roomId, String userId) async { + final raw = await _outboundGroupSessionsBox.get(roomId); + if (raw == null) return null; + return OutboundGroupSession.fromJson(copyMap(raw), userId); + } + + @override + Future> getRoomList(Client client) => + runBenchmarked>('Get room list from store', () async { + final rooms = {}; + final userID = client.userID; + + final rawRooms = await _roomsBox.getAllValues(); + + final getRoomStateRequests = >{}; + final getRoomMembersRequests = >{}; + for (final raw in rawRooms.values) { + // Get the room + final room = Room.fromJson(copyMap(raw), client); + + final membersToPostload = {if (userID != null) userID}; + + // If the room is a direct chat, those IDs should be there too + if (room.isDirectChat) { + membersToPostload + .add(TupleKey(room.id, room.directChatMatrixID!).toString()); + } + // the lastEvent message preview might have an author we need to fetch, if it is a group chat + final lastEvent = room.getState(EventTypes.Message); + if (lastEvent != null && !room.isDirectChat) { + membersToPostload + .add(TupleKey(room.id, lastEvent.senderId).toString()); + } + // if the room has no name and no canonical alias, its name is calculated + // based on the heroes of the room + if (room.getState(EventTypes.RoomName) == null && + room.getState(EventTypes.RoomCanonicalAlias) == null) { + // we don't have a name and no canonical alias, so we'll need to + // post-load the heroes + final heroes = room.summary.mHeroes; + if (heroes != null) { + heroes.forEach((hero) => membersToPostload.add(hero)); + } + } + + // Get the "important" room states. All other states will be loaded once + // `getUnimportantRoomStates()` is called. + final dbKeys = client.importantStateEvents + .map((state) => TupleKey(room.id, state).toString()) + .toList(); + getRoomStateRequests[room.id] = _roomStateBox.getAll( + dbKeys, + ); + + // Load members + final membersDbKeys = membersToPostload + .map((member) => TupleKey(room.id, member).toString()) + .toList(); + getRoomMembersRequests[room.id] = _roomMembersBox.getAll( + membersDbKeys, + ); + + // Add to the list and continue. + rooms[room.id] = room; + } + + for (final room in rooms.values) { + // Add states to the room + final statesList = await getRoomStateRequests[room.id]; + if (statesList != null) { + for (final states in statesList) { + if (states == null) continue; + final stateEvents = states.values + .map((raw) => Event.fromJson(copyMap(raw), room)) + .toList(); + for (final state in stateEvents) { + room.setState(state); + } + } + } + + // Add members to the room + final members = await getRoomMembersRequests[room.id]; + if (members != null) { + for (final member in members) { + if (member == null) continue; + room.setState(Event.fromJson(copyMap(member), room)); + } + } + } + + // Get the room account data + final roomAccountDataRaws = await _roomAccountDataBox.getAllValues(); + for (final entry in roomAccountDataRaws.entries) { + final keys = TupleKey.fromString(entry.key); + final basicRoomEvent = BasicRoomEvent.fromJson( + copyMap(entry.value), + ); + final roomId = keys.parts.first; + if (rooms.containsKey(roomId)) { + rooms[roomId]!.roomAccountData[basicRoomEvent.type] = + basicRoomEvent; + } else { + Logs().w( + 'Found account data for unknown room $roomId. Delete now...'); + await _roomAccountDataBox + .delete(TupleKey(roomId, basicRoomEvent.type).toString()); + } + } + + return rooms.values.toList(); + }); + + @override + Future getSSSSCache(String type) async { + final raw = await _ssssCacheBox.get(type); + if (raw == null) return null; + return SSSSCache.fromJson(copyMap(raw)); + } + + @override + Future> getToDeviceEventQueue() async { + final raws = await _toDeviceQueueBox.getAllValues(); + final copiedRaws = raws.entries.map((entry) { + final copiedRaw = copyMap(entry.value); + copiedRaw['id'] = int.parse(entry.key); + copiedRaw['content'] = jsonDecode(copiedRaw['content']); + return copiedRaw; + }).toList(); + return copiedRaws.map((raw) => QueuedToDeviceEvent.fromJson(raw)).toList(); + } + + @override + Future> getUnimportantRoomEventStatesForRoom( + List events, Room room) async { + final keys = (await _roomStateBox.getAllKeys()).where((key) { + final tuple = TupleKey.fromString(key); + return tuple.parts.first == room.id && !events.contains(tuple.parts[1]); + }); + + final unimportantEvents = []; + for (final key in keys) { + final states = await _roomStateBox.get(key); + if (states == null) continue; + unimportantEvents.addAll( + states.values.map((raw) => Event.fromJson(copyMap(raw), room))); + } + return unimportantEvents; + } + + @override + Future getUser(String userId, Room room) async { + final state = + await _roomMembersBox.get(TupleKey(room.id, userId).toString()); + if (state == null) return null; + return Event.fromJson(copyMap(state), room).asUser; + } + + @override + Future> getUserDeviceKeys(Client client) => + runBenchmarked>( + 'Get all user device keys from store', () async { + final deviceKeysOutdated = + await _userDeviceKeysOutdatedBox.getAllKeys(); + if (deviceKeysOutdated.isEmpty) { + return {}; + } + final res = {}; + final userDeviceKeysBoxKeys = await _userDeviceKeysBox.getAllKeys(); + final userCrossSigningKeysBoxKeys = + await _userCrossSigningKeysBox.getAllKeys(); + for (final userId in deviceKeysOutdated) { + final deviceKeysBoxKeys = userDeviceKeysBoxKeys.where((tuple) { + final tupleKey = TupleKey.fromString(tuple); + return tupleKey.parts.first == userId; + }); + final crossSigningKeysBoxKeys = + userCrossSigningKeysBoxKeys.where((tuple) { + final tupleKey = TupleKey.fromString(tuple); + return tupleKey.parts.first == userId; + }); + final childEntries = await Future.wait( + deviceKeysBoxKeys.map( + (key) async { + final userDeviceKey = await _userDeviceKeysBox.get(key); + if (userDeviceKey == null) return null; + return copyMap(userDeviceKey); + }, + ), + ); + final crossSigningEntries = await Future.wait( + crossSigningKeysBoxKeys.map( + (key) async { + final crossSigningKey = await _userCrossSigningKeysBox.get(key); + if (crossSigningKey == null) return null; + return copyMap(crossSigningKey); + }, + ), + ); + res[userId] = DeviceKeysList.fromDbJson( + { + 'client_id': client.id, + 'user_id': userId, + 'outdated': await _userDeviceKeysOutdatedBox.get(userId), + }, + childEntries + .where((c) => c != null) + .toList() + .cast>(), + crossSigningEntries + .where((c) => c != null) + .toList() + .cast>(), + client); + } + return res; + }); + + @override + Future> getUsers(Room room) async { + final users = []; + final keys = (await _roomMembersBox.getAllKeys()) + .where((key) => TupleKey.fromString(key).parts.first == room.id) + .toList(); + final states = await _roomMembersBox.getAll(keys); + states.removeWhere((state) => state == null); + states.forEach( + (state) => users.add(Event.fromJson(copyMap(state!), room).asUser), + ); + + return users; + } + + @override + Future insertClient( + String name, + String homeserverUrl, + String token, + String userId, + String? deviceId, + String? deviceName, + String? prevBatch, + String? olmAccount) async { + await transaction(() async { + await _clientBox.put('homeserver_url', homeserverUrl); + await _clientBox.put('token', token); + await _clientBox.put('user_id', userId); + if (deviceId == null) { + await _clientBox.delete('device_id'); + } else { + await _clientBox.put('device_id', deviceId); + } + if (deviceName == null) { + await _clientBox.delete('device_name'); + } else { + await _clientBox.put('device_name', deviceName); + } + if (prevBatch == null) { + await _clientBox.delete('prev_batch'); + } else { + await _clientBox.put('prev_batch', prevBatch); + } + if (olmAccount == null) { + await _clientBox.delete('olm_account'); + } else { + await _clientBox.put('olm_account', olmAccount); + } + await _clientBox.delete('sync_filter_id'); + }); + return 0; + } + + @override + Future insertIntoToDeviceQueue( + String type, String txnId, String content) async { + final id = DateTime.now().millisecondsSinceEpoch; + await _toDeviceQueueBox.put(id.toString(), { + 'type': type, + 'txn_id': txnId, + 'content': content, + }); + return id; + } + + @override + Future markInboundGroupSessionAsUploaded( + String roomId, String sessionId) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) { + Logs().w( + 'Tried to mark inbound group session as uploaded which was not found in the database!'); + return; + } + raw['uploaded'] = true; + await _inboundGroupSessionsBox.put(sessionId, raw); + return; + } + + @override + Future markInboundGroupSessionsAsNeedingUpload() async { + final keys = await _inboundGroupSessionsBox.getAllKeys(); + for (final sessionId in keys) { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) continue; + raw['uploaded'] = false; + await _inboundGroupSessionsBox.put(sessionId, raw); + } + return; + } + + @override + Future removeEvent(String eventId, String roomId) async { + await _eventsBox.delete(TupleKey(roomId, eventId).toString()); + final keys = await _timelineFragmentsBox.getAllKeys(); + for (final key in keys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + final eventIds = await _timelineFragmentsBox.get(key) ?? []; + final prevLength = eventIds.length; + eventIds.removeWhere((id) => id == eventId); + if (eventIds.length < prevLength) { + await _timelineFragmentsBox.put(key, eventIds); + } + } + return; + } + + @override + Future removeOutboundGroupSession(String roomId) async { + await _outboundGroupSessionsBox.delete(roomId); + return; + } + + @override + Future removeUserCrossSigningKey( + String userId, String publicKey) async { + await _userCrossSigningKeysBox + .delete(TupleKey(userId, publicKey).toString()); + return; + } + + @override + Future removeUserDeviceKey(String userId, String deviceId) async { + await _userDeviceKeysBox.delete(TupleKey(userId, deviceId).toString()); + return; + } + + @override + Future resetNotificationCount(String roomId) async { + final raw = await _roomsBox.get(roomId); + if (raw == null) return; + raw['notification_count'] = raw['highlight_count'] = 0; + await _roomsBox.put(roomId, raw); + return; + } + + @override + Future setBlockedUserCrossSigningKey( + bool blocked, String userId, String publicKey) async { + final raw = await _userCrossSigningKeysBox + .get(TupleKey(userId, publicKey).toString()); + raw!['blocked'] = blocked; + await _userCrossSigningKeysBox.put( + TupleKey(userId, publicKey).toString(), + raw, + ); + return; + } + + @override + Future setBlockedUserDeviceKey( + bool blocked, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()); + raw!['blocked'] = blocked; + await _userDeviceKeysBox.put( + TupleKey(userId, deviceId).toString(), + raw, + ); + return; + } + + @override + Future setLastActiveUserDeviceKey( + int lastActive, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()); + raw!['last_active'] = lastActive; + await _userDeviceKeysBox.put( + TupleKey(userId, deviceId).toString(), + raw, + ); + } + + @override + Future setLastSentMessageUserDeviceKey( + String lastSentMessage, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()); + raw!['last_sent_message'] = lastSentMessage; + await _userDeviceKeysBox.put( + TupleKey(userId, deviceId).toString(), + raw, + ); + } + + @override + Future setRoomPrevBatch( + String prevBatch, String roomId, Client client) async { + final raw = await _roomsBox.get(roomId); + if (raw == null) return; + final room = Room.fromJson(copyMap(raw), client); + room.prev_batch = prevBatch; + await _roomsBox.put(roomId, room.toJson()); + return; + } + + @override + Future setVerifiedUserCrossSigningKey( + bool verified, String userId, String publicKey) async { + final raw = (await _userCrossSigningKeysBox + .get(TupleKey(userId, publicKey).toString())) ?? + {}; + raw['verified'] = verified; + await _userCrossSigningKeysBox.put( + TupleKey(userId, publicKey).toString(), + raw, + ); + return; + } + + @override + Future setVerifiedUserDeviceKey( + bool verified, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()); + raw!['verified'] = verified; + await _userDeviceKeysBox.put( + TupleKey(userId, deviceId).toString(), + raw, + ); + return; + } + + @override + Future storeAccountData(String type, String content) async { + await _accountDataBox.put(type, copyMap(jsonDecode(content))); + return; + } + + @override + Future storeEventUpdate(EventUpdate eventUpdate, Client client) async { + // Ephemerals should not be stored + if (eventUpdate.type == EventUpdateType.ephemeral) return; + final tmpRoom = Room(id: eventUpdate.roomID, client: client); + + // In case of this is a redaction event + if (eventUpdate.content['type'] == EventTypes.Redaction) { + final event = await getEventById(eventUpdate.content['redacts'], tmpRoom); + if (event != null) { + event.setRedactionEvent(Event.fromJson(eventUpdate.content, tmpRoom)); + await _eventsBox.put( + TupleKey(eventUpdate.roomID, event.eventId).toString(), + event.toJson()); + } + } + + // Store a common message event + if ({EventUpdateType.timeline, EventUpdateType.history} + .contains(eventUpdate.type)) { + final eventId = eventUpdate.content['event_id']; + // Is this ID already in the store? + final prevEvent = await _eventsBox + .get(TupleKey(eventUpdate.roomID, eventId).toString()); + final prevStatus = prevEvent == null + ? null + : () { + final json = copyMap(prevEvent); + final statusInt = json.tryGet('status') ?? + json + .tryGetMap('unsigned') + ?.tryGet(messageSendingStatusKey); + return statusInt == null ? null : eventStatusFromInt(statusInt); + }(); + + // calculate the status + final newStatus = eventStatusFromInt( + eventUpdate.content.tryGet('status') ?? + eventUpdate.content + .tryGetMap('unsigned') + ?.tryGet(messageSendingStatusKey) ?? + EventStatus.synced.intValue, + ); + + // Is this the response to a sending event which is already synced? Then + // there is nothing to do here. + if (!newStatus.isSynced && prevStatus != null && prevStatus.isSynced) { + return; + } + + final status = newStatus.isError || prevStatus == null + ? newStatus + : latestEventStatus( + prevStatus, + newStatus, + ); + + // Add the status and the sort order to the content so it get stored + eventUpdate.content['unsigned'] ??= {}; + eventUpdate.content['unsigned'][messageSendingStatusKey] = + eventUpdate.content['status'] = status.intValue; + + // In case this event has sent from this account we have a transaction ID + final transactionId = eventUpdate.content + .tryGetMap('unsigned') + ?.tryGet('transaction_id'); + await _eventsBox.put(TupleKey(eventUpdate.roomID, eventId).toString(), + eventUpdate.content); + + // Update timeline fragments + final key = TupleKey(eventUpdate.roomID, status.isSent ? '' : 'SENDING') + .toString(); + + final eventIds = + List.from(await _timelineFragmentsBox.get(key) ?? []); + + if (!eventIds.contains(eventId)) { + if (eventUpdate.type == EventUpdateType.history) { + eventIds.add(eventId); + } else { + eventIds.insert(0, eventId); + } + await _timelineFragmentsBox.put(key, eventIds); + } else if (status.isSynced && + prevStatus != null && + prevStatus.isSent && + eventUpdate.type != EventUpdateType.history) { + // Status changes from 1 -> 2? Make sure event is correctly sorted. + eventIds.remove(eventId); + eventIds.insert(0, eventId); + } + + // If event comes from server timeline, remove sending events with this ID + if (status.isSent) { + final key = TupleKey(eventUpdate.roomID, 'SENDING').toString(); + final eventIds = + List.from(await _timelineFragmentsBox.get(key) ?? []); + final i = eventIds.indexWhere((id) => id == eventId); + if (i != -1) { + await _timelineFragmentsBox.put(key, eventIds..removeAt(i)); + } + } + + // Is there a transaction id? Then delete the event with this id. + if (!status.isError && !status.isSending && transactionId != null) { + await removeEvent(transactionId, eventUpdate.roomID); + } + } + + // Store a common state event + if ({ + EventUpdateType.timeline, + EventUpdateType.state, + EventUpdateType.inviteState + }.contains(eventUpdate.type)) { + if (eventUpdate.content['type'] == EventTypes.RoomMember) { + await _roomMembersBox.put( + TupleKey( + eventUpdate.roomID, + eventUpdate.content['state_key'], + ).toString(), + eventUpdate.content); + } else { + final key = TupleKey( + eventUpdate.roomID, + eventUpdate.content['type'], + ).toString(); + final stateMap = copyMap(await _roomStateBox.get(key) ?? {}); + // store state events and new messages, that either are not an edit or an edit of the lastest message + // An edit is an event, that has an edit relation to the latest event. In some cases for the second edit, we need to compare if both have an edit relation to the same event instead. + if (eventUpdate.content + .tryGetMap('content') + ?.tryGetMap('m.relates_to') == + null) { + stateMap[eventUpdate.content['state_key'] ?? ''] = + eventUpdate.content; + await _roomStateBox.put(key, stateMap); + } else { + final editedEventRelationshipEventId = eventUpdate.content + .tryGetMap('content') + ?.tryGetMap('m.relates_to') + ?.tryGet('event_id'); + final state = stateMap[''] == null + ? null + : Event.fromJson(stateMap[''] as Map, tmpRoom); + if (eventUpdate.content['type'] != EventTypes.Message || + eventUpdate.content + .tryGetMap('content') + ?.tryGetMap('m.relates_to') + ?.tryGet('rel_type') != + RelationshipTypes.edit || + editedEventRelationshipEventId == state?.eventId || + ((state?.relationshipType == RelationshipTypes.edit && + editedEventRelationshipEventId == + state?.relationshipEventId))) { + stateMap[eventUpdate.content['state_key'] ?? ''] = + eventUpdate.content; + await _roomStateBox.put(key, stateMap); + } + } + } + } + + // Store a room account data event + if (eventUpdate.type == EventUpdateType.accountData) { + await _roomAccountDataBox.put( + TupleKey( + eventUpdate.roomID, + eventUpdate.content['type'], + ).toString(), + eventUpdate.content, + ); + } + } + + @override + Future storeFile(Uri mxcUri, Uint8List bytes, int time) async { + return; + } + + @override + Future storeInboundGroupSession( + String roomId, + String sessionId, + String pickle, + String content, + String indexes, + String allowedAtIndex, + String senderKey, + String senderClaimedKey) async { + await _inboundGroupSessionsBox.put( + sessionId, + StoredInboundGroupSession( + roomId: roomId, + sessionId: sessionId, + pickle: pickle, + content: content, + indexes: indexes, + allowedAtIndex: allowedAtIndex, + senderKey: senderKey, + senderClaimedKeys: senderClaimedKey, + uploaded: false, + ).toJson()); + return; + } + + @override + Future storeOutboundGroupSession( + String roomId, String pickle, String deviceIds, int creationTime) async { + await _outboundGroupSessionsBox.put(roomId, { + 'room_id': roomId, + 'pickle': pickle, + 'device_ids': deviceIds, + 'creation_time': creationTime, + }); + return; + } + + @override + Future storePrevBatch( + String prevBatch, + ) async { + if ((await _clientBox.getAllKeys()).isEmpty) return; + await _clientBox.put('prev_batch', prevBatch); + return; + } + + @override + Future storeRoomUpdate( + String roomId, SyncRoomUpdate roomUpdate, Client client) async { + // Leave room if membership is leave + if (roomUpdate is LeftRoomUpdate) { + await forgetRoom(roomId); + return; + } + final membership = roomUpdate is LeftRoomUpdate + ? Membership.leave + : roomUpdate is InvitedRoomUpdate + ? Membership.invite + : Membership.join; + // Make sure room exists + final currentRawRoom = await _roomsBox.get(roomId); + if (currentRawRoom == null) { + await _roomsBox.put( + roomId, + roomUpdate is JoinedRoomUpdate + ? Room( + client: client, + id: roomId, + membership: membership, + highlightCount: + roomUpdate.unreadNotifications?.highlightCount?.toInt() ?? + 0, + notificationCount: roomUpdate + .unreadNotifications?.notificationCount + ?.toInt() ?? + 0, + prev_batch: roomUpdate.timeline?.prevBatch, + summary: roomUpdate.summary, + ).toJson() + : Room( + client: client, + id: roomId, + membership: membership, + ).toJson()); + } else if (roomUpdate is JoinedRoomUpdate) { + final currentRoom = Room.fromJson(copyMap(currentRawRoom), client); + await _roomsBox.put( + roomId, + Room( + client: client, + id: roomId, + membership: membership, + highlightCount: + roomUpdate.unreadNotifications?.highlightCount?.toInt() ?? + currentRoom.highlightCount, + notificationCount: + roomUpdate.unreadNotifications?.notificationCount?.toInt() ?? + currentRoom.notificationCount, + prev_batch: + roomUpdate.timeline?.prevBatch ?? currentRoom.prev_batch, + summary: RoomSummary.fromJson(currentRoom.summary.toJson() + ..addAll(roomUpdate.summary?.toJson() ?? {})), + ).toJson()); + } + + // Is the timeline limited? Then all previous messages should be + // removed from the database! + if (roomUpdate is JoinedRoomUpdate && + roomUpdate.timeline?.limited == true) { + await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString()); + } + } + + @override + Future storeSSSSCache( + String type, String keyId, String ciphertext, String content) async { + await _ssssCacheBox.put( + type, + SSSSCache( + type: type, + keyId: keyId, + ciphertext: ciphertext, + content: content, + ).toJson()); + } + + @override + Future storeSyncFilterId( + String syncFilterId, + ) async { + await _clientBox.put('sync_filter_id', syncFilterId); + } + + @override + Future storeUserCrossSigningKey(String userId, String publicKey, + String content, bool verified, bool blocked) async { + await _userCrossSigningKeysBox.put( + TupleKey(userId, publicKey).toString(), + { + 'user_id': userId, + 'public_key': publicKey, + 'content': content, + 'verified': verified, + 'blocked': blocked, + }, + ); + } + + @override + Future storeUserDeviceKey(String userId, String deviceId, + String content, bool verified, bool blocked, int lastActive) async { + await _userDeviceKeysBox.put(TupleKey(userId, deviceId).toString(), { + 'user_id': userId, + 'device_id': deviceId, + 'content': content, + 'verified': verified, + 'blocked': blocked, + 'last_active': lastActive, + 'last_sent_message': '', + }); + return; + } + + @override + Future storeUserDeviceKeysInfo(String userId, bool outdated) async { + await _userDeviceKeysOutdatedBox.put(userId, outdated); + return; + } + + Completer? _transactionLock; + final _transactionZones = {}; + + @override + Future transaction(Future Function() action) async { + // we want transactions to lock, however NOT if transactoins are run inside of each other. + // to be able to do this, we use dart zones (https://dart.dev/articles/archive/zones). + // _transactionZones holds a set of all zones which are currently running a transaction. + // _transactionLock holds the lock. + + // first we try to determine if we are inside of a transaction currently + var isInTransaction = false; + Zone? zone = Zone.current; + // for that we keep on iterating to the parent zone until there is either no zone anymore + // or we have found a zone inside of _transactionZones. + while (zone != null) { + if (_transactionZones.contains(zone)) { + isInTransaction = true; + break; + } + zone = zone.parent; + } + // if we are inside a transaction....just run the action + if (isInTransaction) { + return await action(); + } + // if we are *not* in a transaction, time to wait for the lock! + while (_transactionLock != null) { + await _transactionLock!.future; + } + // claim the lock + final lock = Completer(); + _transactionLock = lock; + try { + // run the action inside of a new zone + return await runZoned(() async { + try { + // don't forget to add the new zone to _transactionZones! + _transactionZones.add(Zone.current); + late final T result; + await _collection.transaction(() async { + result = await action(); + }); + return result; + } finally { + // aaaand remove the zone from _transactionZones again + _transactionZones.remove(Zone.current); + } + }); + } finally { + // aaaand finally release the lock + _transactionLock = null; + lock.complete(); + } + } + + @override + Future updateClient( + String homeserverUrl, + String token, + String userId, + String? deviceId, + String? deviceName, + String? prevBatch, + String? olmAccount, + ) async { + await transaction(() async { + await _clientBox.put('homeserver_url', homeserverUrl); + await _clientBox.put('token', token); + await _clientBox.put('user_id', userId); + if (deviceId == null) { + await _clientBox.delete('device_id'); + } else { + await _clientBox.put('device_id', deviceId); + } + if (deviceName == null) { + await _clientBox.delete('device_name'); + } else { + await _clientBox.put('device_name', deviceName); + } + if (prevBatch == null) { + await _clientBox.delete('prev_batch'); + } else { + await _clientBox.put('prev_batch', prevBatch); + } + if (olmAccount == null) { + await _clientBox.delete('olm_account'); + } else { + await _clientBox.put('olm_account', olmAccount); + } + }); + return; + } + + @override + Future updateClientKeys( + String olmAccount, + ) async { + await _clientBox.put('olm_account', olmAccount); + return; + } + + @override + Future updateInboundGroupSessionAllowedAtIndex( + String allowedAtIndex, String roomId, String sessionId) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) { + Logs().w( + 'Tried to update inbound group session as uploaded which wasnt found in the database!'); + return; + } + raw['allowed_at_index'] = allowedAtIndex; + await _inboundGroupSessionsBox.put(sessionId, raw); + return; + } + + @override + Future updateInboundGroupSessionIndexes( + String indexes, String roomId, String sessionId) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) { + Logs().w( + 'Tried to update inbound group session indexes of a session which was not found in the database!'); + return; + } + final json = copyMap(raw); + json['indexes'] = indexes; + await _inboundGroupSessionsBox.put(sessionId, json); + return; + } + + @override + Future updateRoomSortOrder( + double oldestSortOrder, double newestSortOrder, String roomId) async { + final raw = await _roomsBox.get(roomId); + if (raw == null) throw ('Room not found'); + raw['oldest_sort_order'] = oldestSortOrder; + raw['newest_sort_order'] = newestSortOrder; + await _roomsBox.put(roomId, raw); + return; + } + + @override + Future> getAllInboundGroupSessions() async { + final rawSessions = await _inboundGroupSessionsBox.getAllValues(); + return rawSessions.values + .map((raw) => StoredInboundGroupSession.fromJson(copyMap(raw))) + .toList(); + } + + @override + Future addSeenDeviceId( + String userId, + String deviceId, + String publicKeysHash, + ) => + _seenDeviceIdsBox.put( + TupleKey(userId, deviceId).toString(), publicKeysHash); + + @override + Future addSeenPublicKey( + String publicKey, + String deviceId, + ) => + _seenDeviceKeysBox.put(publicKey, deviceId); + + @override + Future deviceIdSeen(userId, deviceId) async { + final raw = + await _seenDeviceIdsBox.get(TupleKey(userId, deviceId).toString()); + if (raw == null) return null; + return raw; + } + + @override + Future publicKeySeen(String publicKey) async { + final raw = await _seenDeviceKeysBox.get(publicKey); + if (raw == null) return null; + return raw; + } +} + +class TupleKey { + final List parts; + + TupleKey(String key1, [String? key2, String? key3]) + : parts = [ + key1, + if (key2 != null) key2, + if (key3 != null) key3, + ]; + + const TupleKey.byParts(this.parts); + + TupleKey.fromString(String multiKeyString) + : parts = multiKeyString.split('|').toList(); + + @override + String toString() => parts.join('|'); + + @override + bool operator ==(other) => parts.toString() == other.toString(); +} + +dynamic _castValue(dynamic value) { + if (value is Map) { + return copyMap(value); + } + if (value is List) { + return value.map(_castValue).toList(); + } + return value; +} + +/// The store always gives back an `_InternalLinkedHasMap`. This +/// creates a deep copy of the json and makes sure that the format is always +/// `Map`. +Map copyMap(Map map) { + final copy = Map.from(map); + for (final entry in copy.entries) { + copy[entry.key] = _castValue(entry.value); + } + return copy; +} diff --git a/lib/src/database/hive_database.dart b/lib/src/database/hive_database.dart index 90891efb..8cdb0acf 100644 --- a/lib/src/database/hive_database.dart +++ b/lib/src/database/hive_database.dart @@ -371,10 +371,6 @@ class FamedlySdkHiveDatabase extends DatabaseApi { return Event.fromJson(convertToJson(raw), room); } - @override - bool eventIsKnown(String eventId, String roomId) => - _eventsBox.keys.contains(MultiKey(roomId, eventId).toString()); - /// Loads a whole list of events at once from the store for a specific room Future> _getEventsByIds(List eventIds, Room room) => Future.wait(eventIds @@ -489,6 +485,21 @@ class FamedlySdkHiveDatabase extends DatabaseApi { .toList(); } + @override + Future> getAllOlmSessions() async { + final backup = Map.fromEntries( + await Future.wait( + _olmSessionsBox.keys.map( + (key) async => MapEntry( + key, + await _olmSessionsBox.get(key), + ), + ), + ), + ); + return backup.cast(); + } + @override Future> getOlmSessionsForDevices( List identityKey, String userId) async { diff --git a/lib/src/room.dart b/lib/src/room.dart index b8271112..20e8abc0 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -187,7 +187,8 @@ class Room { state.relationshipEventId != null && state.relationshipType == RelationshipTypes.edit && lastEvent != null && - !lastEvent.matchesEventOrTransactionId(state.relationshipEventId)) { + !state.matchesEventOrTransactionId(lastEvent.eventId) && + lastEvent.eventId != state.relationshipEventId) { return; } @@ -202,7 +203,8 @@ class Room { final prevEvent = getState(state.type, stateKey); if (prevEvent != null && prevEvent.eventId != state.eventId && - client.database?.eventIsKnown(state.eventId, roomId) == true) { + prevEvent.originServerTs.millisecondsSinceEpoch > + state.originServerTs.millisecondsSinceEpoch) { return; } @@ -1354,15 +1356,18 @@ class Room { } /// Uploads a new user avatar for this room. Returns the event ID of the new - /// m.room.avatar event. - Future setAvatar(MatrixFile file) async { - final uploadResp = - await client.uploadContent(file.bytes, filename: file.name); + /// m.room.avatar event. Leave empty to remove the current avatar. + Future setAvatar(MatrixFile? file) async { + final uploadResp = file == null + ? null + : await client.uploadContent(file.bytes, filename: file.name); return await client.setRoomStateWithKey( id, EventTypes.RoomAvatar, '', - {'url': uploadResp.toString()}, + { + if (uploadResp != null) 'url': uploadResp.toString(), + }, ); } diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index 5969d5e2..1e1e7b5e 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -120,13 +120,12 @@ class Timeline { this.onRemove, }) : events = events ?? [] { sub = room.client.onEvent.stream.listen(_handleEventUpdate); + // If the timeline is limited we want to clear our events cache roomSub = room.client.onSync.stream .where((sync) => sync.rooms?.join?[room.id]?.timeline?.limited == true) - .listen((_) { - this.events.clear(); - aggregatedEvents.clear(); - }); + .listen(_removeEventsNotInThisSync); + sessionIdReceivedSub = room.onSessionKeyReceived.stream.listen(_sessionKeyReceived); @@ -136,6 +135,13 @@ class Timeline { } } + /// Removes all entries from [events] which are not in this SyncUpdate. + void _removeEventsNotInThisSync(SyncUpdate sync) { + final newSyncEvents = sync.rooms?.join?[room.id]?.timeline?.events ?? []; + final keepEventIds = newSyncEvents.map((e) => e.eventId); + events.removeWhere((e) => !keepEventIds.contains(e.eventId)); + } + /// Don't forget to call this before you dismiss this object! void cancelSubscriptions() { sub?.cancel(); diff --git a/lib/src/user.dart b/lib/src/user.dart index 8147a0d7..2f9c53cc 100644 --- a/lib/src/user.dart +++ b/lib/src/user.dart @@ -129,7 +129,7 @@ class User extends Event { words[i] = words[i][0].toUpperCase() + words[i].substring(1); } } - return words.join(' '); + return words.join(' ').trim(); } return 'Unknown user'; } diff --git a/lib/src/utils/commands_extension.dart b/lib/src/utils/commands_extension.dart index 4167ae9f..038ab4e4 100644 --- a/lib/src/utils/commands_extension.dart +++ b/lib/src/utils/commands_extension.dart @@ -98,6 +98,19 @@ extension CommandsClientExtension on Client { txid: args.txid, ); }); + addCommand('dm', (CommandArgs args) async { + final parts = args.msg.split(' '); + return await args.room.client.startDirectChat( + parts.first, + enableEncryption: !parts.any((part) => part == '--no-encryption'), + ); + }); + addCommand('create', (CommandArgs args) async { + final parts = args.msg.split(' '); + return await args.room.client.createGroupChat( + enableEncryption: !parts.any((part) => part == '--no-encryption'), + ); + }); addCommand('plain', (CommandArgs args) async { return await args.room.sendTextEvent( args.msg, @@ -202,6 +215,10 @@ extension CommandsClientExtension on Client { .clearOrUseOutboundGroupSession(args.room.id, wipe: true); return ''; }); + addCommand('clearcache', (CommandArgs args) async { + await clearCache(); + return ''; + }); } } diff --git a/lib/src/utils/html_to_text.dart b/lib/src/utils/html_to_text.dart index 2abbf0a4..aa4d0bf2 100644 --- a/lib/src/utils/html_to_text.dart +++ b/lib/src/utils/html_to_text.dart @@ -50,11 +50,13 @@ class HtmlToText { .firstMatch(text); if (match == null) { text = HtmlUnescape().convert(text); - if (text[0] != '\n') { - text = '\n$text'; - } - if (text[text.length - 1] != '\n') { - text += '\n'; + if (text.isNotEmpty) { + if (text[0] != '\n') { + text = '\n$text'; + } + if (text[text.length - 1] != '\n') { + text += '\n'; + } } return text; } @@ -64,11 +66,13 @@ class HtmlToText { text = text.replaceAll( RegExp(r'$', multiLine: false, caseSensitive: false), ''); text = HtmlUnescape().convert(text); - if (text[0] != '\n') { - text = '\n$text'; - } - if (text[text.length - 1] != '\n') { - text += '\n'; + if (text.isNotEmpty) { + if (text[0] != '\n') { + text = '\n$text'; + } + if (text[text.length - 1] != '\n') { + text += '\n'; + } } final language = RegExp(r'language-(\w+)', multiLine: false, caseSensitive: false) diff --git a/pubspec.yaml b/pubspec.yaml index 846db442..c6dbadfe 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: matrix description: Matrix Dart SDK -version: 0.7.0-nullsafety.4 +version: 0.7.0-nullsafety.10 homepage: https://famedly.com environment: @@ -25,6 +25,7 @@ dependencies: collection: ^1.15.0 webrtc_interface: ^1.0.1 sdp_transform: ^0.3.2 + fluffybox: ^0.3.5 dev_dependencies: dart_code_metrics: ^4.4.0 diff --git a/test/commands_test.dart b/test/commands_test.dart index 1084db88..30ae47f2 100644 --- a/test/commands_test.dart +++ b/test/commands_test.dart @@ -274,6 +274,28 @@ void main() { }); }); + test('dm', () async { + FakeMatrixApi.calledEndpoints.clear(); + await room.sendTextEvent('/dm @alice:example.com --no-encryption'); + expect( + json.decode( + FakeMatrixApi.calledEndpoints['/client/r0/createRoom']?.first), + { + 'invite': ['@alice:example.com'], + 'is_direct': true, + 'preset': 'trusted_private_chat' + }); + }); + + test('create', () async { + FakeMatrixApi.calledEndpoints.clear(); + await room.sendTextEvent('/create @alice:example.com --no-encryption'); + expect( + json.decode( + FakeMatrixApi.calledEndpoints['/client/r0/createRoom']?.first), + {'preset': 'private_chat'}); + }); + test('discardsession', () async { if (olmEnabled) { await client.encryption?.keyManager.createOutboundGroupSession(room.id); @@ -289,6 +311,11 @@ void main() { } }); + test('create', () async { + await room.sendTextEvent('/clearcache'); + expect(room.client.prevBatch, null); + }); + test('dispose client', () async { await client.dispose(closeDatabase: true); }); diff --git a/test/database_api_test.dart b/test/database_api_test.dart index ca02d873..0d809a65 100644 --- a/test/database_api_test.dart +++ b/test/database_api_test.dart @@ -26,7 +26,11 @@ import 'package:olm/olm.dart' as olm; import 'fake_database.dart'; void main() { - /// All Tests related to the ChatTime + group('FluffyBox Database Test', () { + testDatabase( + getFluffyBoxDatabase(null), + ); + }); group('Hive Database Test', () { testDatabase( getHiveDatabase(null), @@ -309,6 +313,42 @@ void testDatabase( ); expect(olm.isEmpty, true); }); + test('getAllOlmSessions', () async { + var sessions = await database.getAllOlmSessions(); + expect(sessions.isEmpty, true); + await database.storeOlmSession( + 'identityKey', + 'sessionId', + 'pickle', + 0, + ); + await database.storeOlmSession( + 'identityKey', + 'sessionId2', + 'pickle', + 0, + ); + sessions = await database.getAllOlmSessions(); + expect( + sessions, + { + 'identityKey': { + 'sessionId': { + 'identity_key': 'identityKey', + 'pickle': 'pickle', + 'session_id': 'sessionId', + 'last_received': 0 + }, + 'sessionId2': { + 'identity_key': 'identityKey', + 'pickle': 'pickle', + 'session_id': 'sessionId2', + 'last_received': 0 + } + } + }, + ); + }); test('getOlmSessionsForDevices', () async { final olm = await database.getOlmSessionsForDevices( ['identityKeys'], diff --git a/test/fake_database.dart b/test/fake_database.dart index a5dad91b..64ee745b 100644 --- a/test/fake_database.dart +++ b/test/fake_database.dart @@ -23,11 +23,25 @@ import 'package:matrix/matrix.dart'; import 'package:matrix/src/database/hive_database.dart'; import 'package:file/memory.dart'; import 'package:hive/hive.dart'; +import 'package:matrix/src/database/fluffybox_database.dart'; Future getDatabase(Client? _) => getHiveDatabase(_); bool hiveInitialized = false; +Future getFluffyBoxDatabase(Client? c) async { + final fileSystem = MemoryFileSystem(); + final testHivePath = + '${fileSystem.path}/build/.test_store/${Random().nextDouble()}'; + Directory(testHivePath).createSync(recursive: true); + final db = FluffyBoxDatabase( + 'unit_test.${c?.hashCode}', + testHivePath, + ); + await db.open(); + return db; +} + Future getHiveDatabase(Client? c) async { if (!hiveInitialized) { final fileSystem = MemoryFileSystem(); diff --git a/test/html_to_text_test.dart b/test/html_to_text_test.dart index fc29a6a3..9dde56e7 100644 --- a/test/html_to_text_test.dart +++ b/test/html_to_text_test.dart @@ -21,80 +21,81 @@ import 'package:test/test.dart'; void main() { group('htmlToText', () { - test('stuff', () async { - final testMap = { - '': '', - 'hello world\nthis is a test': 'hello world\nthis is a test', - 'That\'s not a test, this is a test': - '*That\'s* not a test, **this** is a test', - 'Visit our website (outdated)': - 'Visit ~~πŸ”—our website~~ (outdated)', - '(cw spiders) spiders are pretty cool': - '(cw spiders) β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ', - 'spiders are pretty cool': - '(cw spiders) β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ', - 'a test case': 'a test case', - 'List of cute animals:\n
    \n
  • Kittens
  • \n
  • Puppies
  • \n
  • Snakes
    (I think they\'re cute!)
  • \n
\n(This list is incomplete, you can help by adding to it!)': - 'List of cute animals:\n● Kittens\n● Puppies\n● Snakes\n (I think they\'re cute!)\n(This list is incomplete, you can help by adding to it!)', - 'fox': '*fox*', - 'fox': '*fox*', - 'fox': '**fox**', - 'fox': '**fox**', - 'fox': '__fox__', - 'fox': '__fox__', - 'fox': '~~fox~~', - 'fox': '~~fox~~', - 'fox': '~~fox~~', - '>fox': '`>fox`', - '
meep
': '```\nmeep\n```', - '
meep\n
': '```\nmeep\n```', - '
meep
': - '```floof\nmeep\n```', - 'before
code
after': 'before\n```\ncode\n```\nafter', - '

before

code

after

': - 'before\n```\ncode\n```\nafter', - '

fox

': 'fox', - '

fox

floof

': 'fox\n\nfloof', - 'website': 'πŸ”—website', - 'fox': 'fox', - 'fox': 'fox', - ':wave:': ':wave:', - 'fox
floof': 'fox\nfloof', - '
fox
floof': '> fox\nfloof', - '

fox

floof': '> fox\nfloof', - '

fox

floof

': '> fox\nfloof', - 'a
fox
floof': 'a\n> fox\nfloof', - '
fox
floof
fluff': - '> > fox\n> floof\nfluff', - '
  • hey
    • a
    • b
  • foxies
': - '● hey\n β—‹ a\n β—‹ b\n● foxies', - '
  1. a
  2. b
': '1. a\n2. b', - '
  1. a
  2. b
': '42. a\n43. b', - '
  1. a
    1. aa
    2. bb
  2. b
': - '1. a\n 1. aa\n 2. bb\n2. b', - '
  1. a
    • aa
    • bb
  2. b
': - '1. a\n β—‹ aa\n β—‹ bb\n2. b', - '
  • a
    1. aa
    2. bb
  • b
': - '● a\n 1. aa\n 2. bb\n● b', - 'bunnyfox': 'fox', - 'fox
floof': 'fox\n----------\nfloof', - '

fox


floof

': 'fox\n----------\nfloof', - '

fox

floof': '# fox\nfloof', - '

fox

floof

': '# fox\nfloof', - 'floof

fox

': 'floof\n# fox', - '

floof

fox

': 'floof\n# fox', - '

fox

': '## fox', - '

fox

': '### fox', - '

fox

': '#### fox', - '
fox
': '##### fox', - '
fox
': '###### fox', - 'fox': 'fox', - '

fox

\n

floof

': 'fox\n\nfloof', - 'beep

fox

\n

floof

': 'fox\n\nfloof', - }; - for (final entry in testMap.entries) { + final testMap = { + '': '', + 'hello world\nthis is a test': 'hello world\nthis is a test', + 'That\'s not a test, this is a test': + '*That\'s* not a test, **this** is a test', + 'Visit our website (outdated)': + 'Visit ~~πŸ”—our website~~ (outdated)', + '(cw spiders) spiders are pretty cool': + '(cw spiders) β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ', + 'spiders are pretty cool': + '(cw spiders) β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ', + 'a test case': 'a test case', + 'List of cute animals:\n
    \n
  • Kittens
  • \n
  • Puppies
  • \n
  • Snakes
    (I think they\'re cute!)
  • \n
\n(This list is incomplete, you can help by adding to it!)': + 'List of cute animals:\n● Kittens\n● Puppies\n● Snakes\n (I think they\'re cute!)\n(This list is incomplete, you can help by adding to it!)', + 'fox': '*fox*', + 'fox': '*fox*', + 'fox': '**fox**', + 'fox': '**fox**', + 'fox': '__fox__', + 'fox': '__fox__', + 'fox': '~~fox~~', + 'fox': '~~fox~~', + 'fox': '~~fox~~', + '>fox': '`>fox`', + '
meep
': '```\nmeep\n```', + '
meep\n
': '```\nmeep\n```', + '
meep
': + '```floof\nmeep\n```', + 'before
code
after': 'before\n```\ncode\n```\nafter', + '

before

code

after

': + 'before\n```\ncode\n```\nafter', + '

fox

': 'fox', + '

fox

floof

': 'fox\n\nfloof', + 'website': 'πŸ”—website', + 'fox': 'fox', + 'fox': 'fox', + ':wave:': ':wave:', + 'fox
floof': 'fox\nfloof', + '
fox
floof': '> fox\nfloof', + '

fox

floof': '> fox\nfloof', + '

fox

floof

': '> fox\nfloof', + 'a
fox
floof': 'a\n> fox\nfloof', + '
fox
floof
fluff': + '> > fox\n> floof\nfluff', + '
  • hey
    • a
    • b
  • foxies
': + '● hey\n β—‹ a\n β—‹ b\n● foxies', + '
  1. a
  2. b
': '1. a\n2. b', + '
  1. a
  2. b
': '42. a\n43. b', + '
  1. a
    1. aa
    2. bb
  2. b
': + '1. a\n 1. aa\n 2. bb\n2. b', + '
  1. a
    • aa
    • bb
  2. b
': + '1. a\n β—‹ aa\n β—‹ bb\n2. b', + '
  • a
    1. aa
    2. bb
  • b
': + '● a\n 1. aa\n 2. bb\n● b', + 'bunnyfox': 'fox', + 'fox
floof': 'fox\n----------\nfloof', + '

fox


floof

': 'fox\n----------\nfloof', + '

fox

floof': '# fox\nfloof', + '

fox

floof

': '# fox\nfloof', + 'floof

fox

': 'floof\n# fox', + '

floof

fox

': 'floof\n# fox', + '

fox

': '## fox', + '

fox

': '### fox', + '

fox

': '#### fox', + '
fox
': '##### fox', + '
fox
': '###### fox', + 'fox': 'fox', + '

fox

\n

floof

': 'fox\n\nfloof', + 'beep

fox

\n

floof

': 'fox\n\nfloof', + '
': '``````', + }; + for (final entry in testMap.entries) { + test(entry.key, () async { expect(HtmlToText.convert(entry.key), entry.value); - } - }); + }); + } }); } diff --git a/test/room_test.dart b/test/room_test.dart index b68f08de..651aa83f 100644 --- a/test/room_test.dart +++ b/test/room_test.dart @@ -234,9 +234,35 @@ void main() { 'm.relates_to': {'rel_type': 'm.replace', 'event_id': '2'}, }, stateKey: '', + status: EventStatus.sending, ), ); expect(room.lastEvent?.body, 'edited cdc'); + expect(room.lastEvent?.status, EventStatus.sending); + expect(room.lastEvent?.eventId, '4'); + + // Status update on edits working? + room.setState( + Event( + senderId: '@test:example.com', + type: 'm.room.encrypted', + room: room, + eventId: '5', + unsigned: {'transaction_id': '4'}, + originServerTs: DateTime.now(), + content: { + 'msgtype': 'm.text', + 'body': 'edited cdc', + 'm.new_content': {'msgtype': 'm.text', 'body': 'edited cdc'}, + 'm.relates_to': {'rel_type': 'm.replace', 'event_id': '2'}, + }, + stateKey: '', + status: EventStatus.sent, + ), + ); + expect(room.lastEvent?.eventId, '5'); + expect(room.lastEvent?.body, 'edited cdc'); + expect(room.lastEvent?.status, EventStatus.sent); }); test('lastEvent when reply parent edited', () async { room.setState(