From 14ee16fe1615f1ddd67568c02b3daf551c99d66c Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Tue, 15 Jun 2021 09:54:27 +0200 Subject: [PATCH] feat: Implement new Hive Database The hive database now implements the whole API except for storing files which should be better done by the flutter_cache_manager package inside of the flutter app. All tests already run with Hive now but the Moor database is still tested too. We needed to change some wait jobs in the tests because the Hive database is not 100% in memory for the tests like Moor. For now both database implementations are equal and the developer can pick which one to use but we plan to get rid of Moor in the future. --- lib/famedlysdk.dart | 1 + lib/src/database/hive_database.dart | 1061 +++++++++++++++++++ pubspec.yaml | 3 +- test/client_test.dart | 4 +- test/database_api_test.dart | 5 +- test/encryption/key_request_test.dart | 3 + test/encryption/key_verification_test.dart | 2 +- test/encryption/online_key_backup_test.dart | 1 + test/fake_database_native.dart | 26 +- 9 files changed, 1100 insertions(+), 6 deletions(-) create mode 100644 lib/src/database/hive_database.dart diff --git a/lib/famedlysdk.dart b/lib/famedlysdk.dart index e3f23141..c19deeea 100644 --- a/lib/famedlysdk.dart +++ b/lib/famedlysdk.dart @@ -39,3 +39,4 @@ export 'src/timeline.dart'; export 'src/user.dart'; export 'src/database/database.dart' show Database; export 'src/database/database_api.dart'; +export 'src/database/hive_database.dart'; diff --git a/lib/src/database/hive_database.dart b/lib/src/database/hive_database.dart new file mode 100644 index 00000000..70c70f6e --- /dev/null +++ b/lib/src/database/hive_database.dart @@ -0,0 +1,1061 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:math'; + +import 'package:famedlysdk/encryption/utils/stored_inbound_group_session.dart'; +import 'package:famedlysdk/encryption/utils/ssss_cache.dart'; +import 'package:famedlysdk/encryption/utils/outbound_group_session.dart'; +import 'package:famedlysdk/encryption/utils/olm_session.dart'; +import 'dart:typed_data'; + +import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/QueuedToDeviceEvent.dart'; +import 'package:hive/hive.dart'; + +/// This is a basic database for the Matrix SDK using the hive store. You need +/// to make sure that you perform `Hive.init()` or `Hive.flutterInit()` before +/// you use this. +/// +/// This database does not support file caching! +class FamedlySdkHiveDatabase extends DatabaseApi { + final String name; + Box _clientBox; + Box _accountDataBox; + Box _roomsBox; + Box _toDeviceQueueBox; + + /// Key is a tuple as MultiKey(roomId, type, stateKey) where stateKey can be + /// an empty string. + LazyBox _roomStateBox; + + /// Key is a tuple as MultiKey(roomId, type) + LazyBox _roomAccountDataBox; + LazyBox _inboundGroupSessionsBox; + LazyBox _outboundGroupSessionsBox; + LazyBox _olmSessionsBox; + + /// Key is a tuple as MultiKey(userId, deviceId) + LazyBox _userDeviceKeysBox; + + /// Key is the user ID as a String + LazyBox _userDeviceKeysOutdatedBox; + + /// Key is a tuple as MultiKey(userId, publicKey) + LazyBox _userCrossSigningKeysBox; + LazyBox _ssssCacheBox; + LazyBox _presencesBox; + + /// Key is a tuple as Multikey(roomId, fragmentId) while the default + /// fragmentId is an empty String + LazyBox _timelineFragmentsBox; + + /// Key is a tuple as MultiKey(roomId, eventId) + LazyBox _eventsBox; + + String get _clientBoxName => '$name.box.client'; + String get _accountDataBoxName => '$name.box.account_data'; + String get _roomsBoxName => '$name.box.rooms'; + String get _toDeviceQueueBoxName => '$name.box.to_device_queue'; + String get _roomStateBoxName => '$name.box.room_states'; + String get _roomAccountDataBoxName => '$name.box.room_account_data'; + String get _inboundGroupSessionsBoxName => '$name.box.inbound_group_session'; + String get _outboundGroupSessionsBoxName => + '$name.box.outbound_group_session'; + String get _olmSessionsBoxName => '$name.box.olm_session'; + String get _userDeviceKeysBoxName => '$name.box.user_device_keys'; + String get _userDeviceKeysOutdatedBoxName => + '$name.box.user_device_keys_outdated'; + String get _userCrossSigningKeysBoxName => '$name.box.cross_signing_keys'; + String get _ssssCacheBoxName => '$name.box.ssss_cache'; + String get _presencesBoxName => '$name.box.presences'; + String get _timelineFragmentsBoxName => '$name.box.timeline_fragments'; + String get _eventsBoxName => '$name.box.events'; + + final HiveCipher encryptionCipher; + + FamedlySdkHiveDatabase(this.name, {this.encryptionCipher}); + + @override + int get maxFileSize => 0; + + Future _actionOnAllBoxes(Future Function(BoxBase box) action) => + Future.wait([ + action(_clientBox), + action(_accountDataBox), + action(_roomsBox), + action(_roomStateBox), + action(_toDeviceQueueBox), + action(_roomAccountDataBox), + action(_inboundGroupSessionsBox), + action(_outboundGroupSessionsBox), + action(_olmSessionsBox), + action(_userDeviceKeysBox), + action(_userDeviceKeysOutdatedBox), + action(_userCrossSigningKeysBox), + action(_ssssCacheBox), + action(_presencesBox), + action(_timelineFragmentsBox), + action(_eventsBox), + ]); + + Future open() async { + _clientBox = await Hive.openBox( + _clientBoxName, + encryptionCipher: encryptionCipher, + ); + _accountDataBox = await Hive.openBox( + _accountDataBoxName, + encryptionCipher: encryptionCipher, + ); + _roomsBox = await Hive.openBox( + _roomsBoxName, + encryptionCipher: encryptionCipher, + ); + _roomStateBox = await Hive.openLazyBox( + _roomStateBoxName, + encryptionCipher: encryptionCipher, + ); + _toDeviceQueueBox = await Hive.openBox( + _toDeviceQueueBoxName, + encryptionCipher: encryptionCipher, + ); + _roomAccountDataBox = await Hive.openLazyBox( + _roomAccountDataBoxName, + encryptionCipher: encryptionCipher, + ); + _inboundGroupSessionsBox = await Hive.openLazyBox( + _inboundGroupSessionsBoxName, + encryptionCipher: encryptionCipher, + ); + _outboundGroupSessionsBox = await Hive.openLazyBox( + _outboundGroupSessionsBoxName, + encryptionCipher: encryptionCipher, + ); + _olmSessionsBox = await Hive.openLazyBox( + _olmSessionsBoxName, + encryptionCipher: encryptionCipher, + ); + _userDeviceKeysBox = await Hive.openLazyBox( + _userDeviceKeysBoxName, + encryptionCipher: encryptionCipher, + ); + _userDeviceKeysOutdatedBox = await Hive.openLazyBox( + _userDeviceKeysOutdatedBoxName, + encryptionCipher: encryptionCipher, + ); + _userCrossSigningKeysBox = await Hive.openLazyBox( + _userCrossSigningKeysBoxName, + encryptionCipher: encryptionCipher, + ); + _ssssCacheBox = await Hive.openLazyBox( + _ssssCacheBoxName, + encryptionCipher: encryptionCipher, + ); + _presencesBox = await Hive.openLazyBox( + _presencesBoxName, + encryptionCipher: encryptionCipher, + ); + _timelineFragmentsBox = await Hive.openLazyBox( + _timelineFragmentsBoxName, + encryptionCipher: encryptionCipher, + ); + _eventsBox = await Hive.openLazyBox( + _eventsBoxName, + encryptionCipher: encryptionCipher, + ); + return; + } + + @override + Future clear(int clientId) async { + Logs().i('Clear and close hive database...'); + await _actionOnAllBoxes((box) async { + await box.deleteAll(box.keys); + await box.close(); + }); + return; + } + + @override + Future clearCache(int clientId) async { + await _roomsBox.deleteAll(_roomsBox.keys); + await _accountDataBox.deleteAll(_accountDataBox.keys); + await _roomStateBox.deleteAll(_roomStateBox.keys); + await _eventsBox.deleteAll(_eventsBox.keys); + await _timelineFragmentsBox.deleteAll(_timelineFragmentsBox.keys); + await _outboundGroupSessionsBox.deleteAll(_outboundGroupSessionsBox.keys); + await _presencesBox.deleteAll(_presencesBox.keys); + } + + @override + Future clearSSSSCache(int clientId) async { + await _ssssCacheBox.deleteAll(_ssssCacheBox.keys); + } + + @override + Future close() => _actionOnAllBoxes((box) => box.close()); + + @override + Future deleteFromToDeviceQueue(int clientId, int id) async { + await _toDeviceQueueBox.deleteAt(id); + return; + } + + @override + Future deleteOldFiles(int savedAt) async { + return; + } + + @override + Future forgetRoom(int clientId, String roomId) async { + await _timelineFragmentsBox.delete(MultiKey(roomId, '').toString()); + for (final key in _eventsBox.keys) { + final multiKey = MultiKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _eventsBox.delete(key); + } + for (final key in _roomStateBox.keys) { + final multiKey = MultiKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomStateBox.delete(key); + } + for (final key in _roomAccountDataBox.keys) { + final multiKey = MultiKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomAccountDataBox.delete(key); + } + await _roomsBox.delete(roomId); + } + + @override + Future> getAccountData(int clientId) async { + final accountData = {}; + for (final key in _accountDataBox.keys) { + final raw = await _accountDataBox.get(key); + accountData[key] = BasicEvent( + type: key, + content: convertToJson(raw), + ); + } + return accountData; + } + + @override + Future> getClient(String name) async { + final map = {}; + for (final key in _clientBox.keys) { + map[key] = await _clientBox.get(key); + } + if (map.isEmpty) return null; + return map; + } + + @override + Future getEventById(int clientId, String eventId, Room room) async { + final raw = await _eventsBox.get(MultiKey(room.id, eventId).toString()); + if (raw == null) return null; + return Event.fromJson(convertToJson(raw), room); + } + + @override + Future> getEventList(int clientId, Room room) async { + final List eventIds = + (await _timelineFragmentsBox.get(MultiKey(room.id, '').toString()) ?? + []); + final events = await Future.wait(eventIds + .map( + (eventId) async => Event.fromJson( + convertToJson( + await _eventsBox.get(MultiKey(room.id, eventId).toString()), + ), + room, + ), + ) + .toList()); + events.sort((a, b) => b.sortOrder.compareTo(a.sortOrder)); + return events; + } + + @override + Future getFile(String mxcUri) async { + return null; + } + + @override + Future getInboundGroupSession( + int clientId, + String roomId, + String sessionId, + ) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) return null; + return StoredInboundGroupSession.fromJson(convertToJson(raw)); + } + + @override + Future> + getInboundGroupSessionsToUpload() async { + final sessions = (await Future.wait(_inboundGroupSessionsBox.keys.map( + (sessionId) async => + await _inboundGroupSessionsBox.get(sessionId)))) + .where((rawSession) => rawSession['uploaded'] == false) + .take(500) + .map( + (json) => StoredInboundGroupSession.fromJson( + convertToJson(json), + ), + ) + .toList(); + return sessions; + } + + @override + Future> getLastSentMessageUserDeviceKey( + int clientId, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(MultiKey(userId, deviceId).toString()); + if (raw == null) return []; + return [raw['last_sent_message']]; + } + + @override + Future storeOlmSession(int clientId, String identityKey, + String sessionId, String pickle, int lastReceived) async { + final rawSessions = (await _olmSessionsBox.get(identityKey) as Map) ?? {}; + rawSessions[sessionId] = { + 'identity_key': identityKey, + 'pickle': pickle, + 'session_id': sessionId, + 'last_received': lastReceived, + }; + await _olmSessionsBox.put(identityKey, rawSessions); + return; + } + + @override + Future> getOlmSessions( + int clientId, String identityKey, String userId) async { + final rawSessions = await _olmSessionsBox.get(identityKey) as Map; + if (rawSessions?.isEmpty ?? true) return []; + return rawSessions.values + .map((json) => OlmSession.fromJson(convertToJson(json), userId)) + .toList(); + } + + @override + Future> getOlmSessionsForDevices( + int clientId, List identityKey, String userId) async { + final sessions = await Future.wait(identityKey + .map((identityKey) => getOlmSessions(clientId, identityKey, userId))); + return [for (final sublist in sessions) ...sublist]; + } + + @override + Future getOutboundGroupSession( + int clientId, String roomId, String userId) async { + final raw = await _outboundGroupSessionsBox.get(roomId); + if (raw == null) return null; + return OutboundGroupSession.fromJson(convertToJson(raw), userId); + } + + @override + Future> getRoomList(Client client) async { + final rooms = {}; + final importantRoomStates = client.importantStateEvents; + for (final key in _roomsBox.keys) { + // Get the room + final raw = await _roomsBox.get(key); + final room = Room.fromJson(convertToJson(raw), client); + + // let's see if we need any m.room.member events + // We always need the member event for ourself + final membersToPostload = {client.userID}; + // If the room is a direct chat, those IDs should be there too + if (room.isDirectChat) membersToPostload.add(room.directChatMatrixID); + // the lastEvent message preview might have an author we need to fetch, if it is a group chat + if (room.getState(EventTypes.Message) != null && !room.isDirectChat) { + membersToPostload.add(room.getState(EventTypes.Message).senderId); + } + // if the room has no name and no canonical alias, its name is calculated + // based on the heroes of the room + if (room.getState(EventTypes.RoomName) == null && + room.getState(EventTypes.RoomCanonicalAlias) == null) { + // we don't have a name and no canonical alias, so we'll need to + // post-load the heroes + membersToPostload.addAll(room.summary?.mHeroes ?? []); + } + // Load members + for (final userId in membersToPostload) { + final state = await _roomStateBox + .get(MultiKey(room.id, EventTypes.RoomMember, userId).toString()); + if (state == null) { + Logs().w('Unable to post load member $userId'); + continue; + } + room.setState(Event.fromJson(convertToJson(state), room)); + } + + // Get the "important" room states. All other states will be loaded once + // `getUnimportantRoomStates()` is called. + for (final type in importantRoomStates) { + final state = + await _roomStateBox.get(MultiKey(room.id, type, '').toString()); + if (state == null) continue; + room.setState(Event.fromJson(convertToJson(state), room)); + } + + // Add to the list and continue. + rooms[room.id] = room; + } + + // Get the room account data + for (final key in _roomAccountDataBox.keys) { + final roomId = MultiKey.fromString(key).parts.first; + if (rooms.containsKey(roomId)) { + final raw = await _roomAccountDataBox.get(key); + final basicRoomEvent = BasicRoomEvent.fromJson( + convertToJson(raw), + ); + rooms[roomId].roomAccountData[basicRoomEvent.type] = basicRoomEvent; + } else { + Logs().w('Found account data for unknown room $roomId. Delete now...'); + await _roomAccountDataBox.delete(key); + } + } + + return rooms.values.toList(); + } + + @override + Future getSSSSCache(int clientId, String type) async { + final raw = await _ssssCacheBox.get(type); + if (raw == null) return null; + return SSSSCache.fromJson(convertToJson(raw)); + } + + @override + Future> getToDeviceEventQueue(int clientId) async => + await Future.wait(_toDeviceQueueBox.keys.map((i) async { + final raw = await _toDeviceQueueBox.get(i); + raw['id'] = i; + return QueuedToDeviceEvent.fromJson(convertToJson(raw)); + }).toList()); + + @override + Future> getUnimportantRoomEventStatesForRoom( + int clientId, List events, Room room) async { + final keys = _roomStateBox.keys.where((key) { + final tuple = MultiKey.fromString(key); + return tuple.parts.first == room.id && !events.contains(tuple.parts[1]); + }); + return await Future.wait( + keys.map( + (key) async => + Event.fromJson(convertToJson(await _roomStateBox.get(key)), room), + ), + ); + } + + @override + Future getUser(int clientId, String userId, Room room) async { + final state = await _roomStateBox + .get(MultiKey(room.id, EventTypes.RoomMember, userId).toString()); + if (state == null) return null; + return Event.fromJson(convertToJson(state), room).asUser; + } + + @override + Future> getUserDeviceKeys(Client client) async { + final deviceKeysOutdated = _userDeviceKeysOutdatedBox.keys; + if (deviceKeysOutdated.isEmpty) { + return {}; + } + final res = {}; + for (final userId in deviceKeysOutdated) { + final deviceKeysBoxKeys = _userDeviceKeysBox.keys.where((tuple) { + final tupleKey = MultiKey.fromString(tuple); + return tupleKey.parts.first == userId; + }); + final crossSigningKeysBoxKeys = + _userCrossSigningKeysBox.keys.where((tuple) { + final tupleKey = MultiKey.fromString(tuple); + return tupleKey.parts.first == userId; + }); + res[userId] = DeviceKeysList.fromDbJson( + { + 'client_id': client.id, + 'user_id': userId, + 'outdated': await _userDeviceKeysOutdatedBox.get(userId), + }, + await Future.wait(deviceKeysBoxKeys.map( + (key) async => convertToJson(await _userDeviceKeysBox.get(key)))), + await Future.wait(crossSigningKeysBoxKeys.map((key) async => + convertToJson(await _userCrossSigningKeysBox.get(key)))), + client); + } + return res; + } + + @override + Future> getUsers(int clientId, Room room) async { + final users = []; + for (final key in _roomStateBox.keys) { + final statesKey = MultiKey.fromString(key); + if (statesKey.parts[0] != room.id || + statesKey.parts[1] != EventTypes.RoomMember) continue; + final state = await _roomStateBox.get(key); + users.add(Event.fromJson(convertToJson(state), room).asUser); + } + return users; + } + + @override + Future insertClient( + String name, + String homeserverUrl, + String token, + String userId, + String deviceId, + String deviceName, + String prevBatch, + String olmAccount) async { + await _clientBox.put('homeserver_url', homeserverUrl); + await _clientBox.put('token', token); + await _clientBox.put('user_id', userId); + await _clientBox.put('device_id', deviceId); + await _clientBox.put('device_name', deviceName); + await _clientBox.put('prev_batch', prevBatch); + await _clientBox.put('olm_account', olmAccount); + await _clientBox.put('sync_filter_id', null); + return; + } + + @override + Future insertIntoToDeviceQueue( + int clientId, String type, String txnId, String content) async { + return await _toDeviceQueueBox.add({ + 'type': type, + 'txn_id': txnId, + 'content': content, + }); + } + + @override + Future markInboundGroupSessionAsUploaded( + int clientId, String roomId, String sessionId) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) { + Logs().w( + 'Tried to mark inbound group session as uploaded which was not found in the database!'); + return; + } + raw['uploaded'] = true; + await _inboundGroupSessionsBox.put(sessionId, raw); + return; + } + + @override + Future markInboundGroupSessionsAsNeedingUpload(int clientId) async { + for (final sessionId in _inboundGroupSessionsBox.keys) { + final raw = await _inboundGroupSessionsBox.get(sessionId); + raw['uploaded'] = false; + await _inboundGroupSessionsBox.put(sessionId, raw); + } + return; + } + + @override + Future removeEvent(int clientId, String eventId, String roomId) async { + await _eventsBox.delete(MultiKey(roomId, eventId).toString()); + for (final key in _timelineFragmentsBox.keys) { + final multiKey = MultiKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + final List eventIds = await _timelineFragmentsBox.get(key) ?? []; + final prevLength = eventIds.length; + eventIds.removeWhere((id) => id == eventId); + if (eventIds.length < prevLength) { + await _timelineFragmentsBox.put(key, eventIds); + } + } + return; + } + + @override + Future removeOutboundGroupSession(int clientId, String roomId) async { + await _outboundGroupSessionsBox.delete(roomId); + return; + } + + @override + Future removeUserCrossSigningKey( + int clientId, String userId, String publicKey) async { + await _userCrossSigningKeysBox + .delete(MultiKey(userId, publicKey).toString()); + return; + } + + @override + Future removeUserDeviceKey( + int clientId, String userId, String deviceId) async { + await _userDeviceKeysBox.delete(MultiKey(userId, deviceId).toString()); + return; + } + + @override + Future resetNotificationCount(int clientId, String roomId) async { + final raw = await _roomsBox.get(roomId); + if (raw == null) return; + raw['notification_count'] = raw['highlight_count'] = 0; + await _roomsBox.put(roomId, raw); + return; + } + + @override + Future setBlockedUserCrossSigningKey( + bool blocked, int clientId, String userId, String publicKey) async { + final raw = await _userCrossSigningKeysBox + .get(MultiKey(userId, publicKey).toString()); + raw['blocked'] = blocked; + await _userCrossSigningKeysBox.put( + MultiKey(userId, publicKey).toString(), + raw, + ); + return; + } + + @override + Future setBlockedUserDeviceKey( + bool blocked, int clientId, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(MultiKey(userId, deviceId).toString()); + raw['blocked'] = blocked; + await _userDeviceKeysBox.put( + MultiKey(userId, deviceId).toString(), + raw, + ); + return; + } + + @override + Future setLastActiveUserDeviceKey( + int lastActive, int clientId, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(MultiKey(userId, deviceId).toString()); + raw['last_active'] = lastActive; + await _userDeviceKeysBox.put( + MultiKey(userId, deviceId).toString(), + raw, + ); + } + + @override + Future setLastSentMessageUserDeviceKey(String lastSentMessage, + int clientId, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(MultiKey(userId, deviceId).toString()); + raw['last_sent_message'] = lastSentMessage; + await _userDeviceKeysBox.put( + MultiKey(userId, deviceId).toString(), + raw, + ); + } + + @override + Future setRoomPrevBatch( + String prevBatch, int clientId, String roomId) async { + final raw = await _roomsBox.get(roomId); + if (raw == null) return; + final room = Room.fromJson(convertToJson(raw)); + room.prev_batch = prevBatch; + await _roomsBox.put(roomId, room.toJson()); + return; + } + + @override + Future setVerifiedUserCrossSigningKey( + bool verified, int clientId, String userId, String publicKey) async { + final raw = (await _userCrossSigningKeysBox + .get(MultiKey(userId, publicKey).toString()) as Map) ?? + {}; + raw['verified'] = verified; + await _userCrossSigningKeysBox.put( + MultiKey(userId, publicKey).toString(), + raw, + ); + return; + } + + @override + Future setVerifiedUserDeviceKey( + bool verified, int clientId, String userId, String deviceId) async { + final raw = + await _userDeviceKeysBox.get(MultiKey(userId, deviceId).toString()); + raw['verified'] = verified; + await _userDeviceKeysBox.put( + MultiKey(userId, deviceId).toString(), + raw, + ); + return; + } + + @override + Future storeAccountData( + int clientId, String type, String content) async { + await _accountDataBox.put(type, convertToJson(jsonDecode(content))); + return; + } + + @override + Future storeEventUpdate(int clientId, EventUpdate eventUpdate) async { + // Ephemerals should not be stored + if (eventUpdate.type == EventUpdateType.ephemeral) return; + + // In case of this is a redaction event + if (eventUpdate.content['type'] == EventTypes.Redaction) { + final tmpRoom = Room(id: eventUpdate.roomID); + final event = + await getEventById(clientId, eventUpdate.content['redacts'], tmpRoom); + if (event != null) { + event.setRedactionEvent(Event.fromJson(eventUpdate.content, tmpRoom)); + await _eventsBox.put( + MultiKey(eventUpdate.roomID, event.eventId).toString(), + event.toJson()); + } + } + + // Store a common message event + if ({EventUpdateType.timeline, EventUpdateType.history} + .contains(eventUpdate.type)) { + final eventId = eventUpdate.content['event_id']; + // Is this ID already in the store? + final prevEvent = _eventsBox + .containsKey(MultiKey(eventUpdate.roomID, eventId).toString()) + ? Event.fromJson( + convertToJson(await _eventsBox + .get(MultiKey(eventUpdate.roomID, eventId).toString())), + null) + : null; + + // calculate the status + final newStatus = eventUpdate.content.tryGet('status') ?? + eventUpdate.content + .tryGetMap('unsigned') + ?.tryGet(messageSendingStatusKey) ?? + 2; + + final status = newStatus == -1 || prevEvent?.status == null + ? newStatus + : max(prevEvent.status, newStatus); + + // Add the status and the sort order to the content so it get stored + eventUpdate.content['unsigned'] ??= {}; + eventUpdate.content['unsigned'][messageSendingStatusKey] = + eventUpdate.content['status'] = status; + eventUpdate.content['unsigned'][sortOrderKey] = eventUpdate.sortOrder; + + // In case this event has sent from this account we have a transaction ID + final transactionId = eventUpdate.content + .tryGetMap('unsigned') + ?.tryGet('transaction_id'); + + await _eventsBox.put(MultiKey(eventUpdate.roomID, eventId).toString(), + eventUpdate.content); + + // Update timeline fragments + final key = MultiKey(eventUpdate.roomID, '').toString(); + final List eventIds = (await _timelineFragmentsBox.get(key) ?? []); + if (!eventIds.any((id) => id == eventId)) { + eventIds.add(eventId); + await _timelineFragmentsBox.put(key, eventIds); + } + + // Is there a transaction id? Then delete the event with this id. + if (status != -1 && status != 0 && transactionId != null) { + await removeEvent(clientId, transactionId, eventUpdate.roomID); + } + } + + // Store a common state event + if ({EventUpdateType.timeline, EventUpdateType.state} + .contains(eventUpdate.type)) { + await _roomStateBox.put( + MultiKey( + eventUpdate.roomID, + eventUpdate.content['type'], + eventUpdate.content['state_key'] ?? '', + ).toString(), + eventUpdate.content); + } + + // Store a room account data event + if (eventUpdate.type == EventUpdateType.accountData) { + await _roomAccountDataBox.put( + MultiKey( + eventUpdate.roomID, + eventUpdate.content['type'], + ).toString(), + eventUpdate.content, + ); + } + } + + @override + Future storeFile(String mxcUri, Uint8List bytes, int time) async { + return; + } + + @override + Future storeInboundGroupSession( + int clientId, + String roomId, + String sessionId, + String pickle, + String content, + String indexes, + String allowedAtIndex, + String senderKey, + String senderClaimedKey) async { + await _inboundGroupSessionsBox.put( + sessionId, + StoredInboundGroupSession( + clientId: clientId, + roomId: roomId, + sessionId: sessionId, + pickle: pickle, + content: content, + indexes: indexes, + allowedAtIndex: allowedAtIndex, + senderKey: senderKey, + senderClaimedKeys: senderClaimedKey, + uploaded: false, + ).toJson()); + return; + } + + @override + Future storeOutboundGroupSession( + int clientId, + String roomId, + String pickle, + String deviceIds, + int creationTime, + int sentMessages) async { + await _outboundGroupSessionsBox.put(roomId, { + 'room_id': roomId, + 'pickle': pickle, + 'device_ids': deviceIds, + 'creation_time': creationTime, + 'sent_messages': sentMessages ?? 0, + }); + return; + } + + @override + Future storePrevBatch(String prevBatch, int clientId) async { + if (_clientBox.keys.isEmpty) return; + await _clientBox.put('prev_batch', prevBatch); + return; + } + + @override + Future storeRoomUpdate(int clientId, RoomUpdate roomUpdate, [_]) async { + // Leave room if membership is leave + if ({Membership.leave, Membership.ban}.contains(roomUpdate.membership)) { + await forgetRoom(clientId, roomUpdate.id); + return; + } + // Make sure room exists + if (!_roomsBox.containsKey(roomUpdate.id)) { + await _roomsBox.put( + roomUpdate.id, + Room( + id: roomUpdate.id, + membership: roomUpdate.membership, + highlightCount: roomUpdate.highlight_count, + notificationCount: roomUpdate.notification_count, + prev_batch: roomUpdate.prev_batch, + summary: roomUpdate.summary, + ).toJson()); + } else { + final currentRawRoom = await _roomsBox.get(roomUpdate.id); + final currentRoom = Room.fromJson(convertToJson(currentRawRoom)); + await _roomsBox.put( + roomUpdate.id, + Room( + id: roomUpdate.id, + membership: roomUpdate.membership ?? currentRoom.membership, + highlightCount: + roomUpdate.highlight_count ?? currentRoom.highlightCount, + notificationCount: + roomUpdate.notification_count ?? currentRoom.notificationCount, + prev_batch: roomUpdate.prev_batch ?? currentRoom.prev_batch, + summary: RoomSummary.fromJson(currentRoom.summary.toJson() + ..addAll(roomUpdate.summary?.toJson() ?? {})), + newestSortOrder: + roomUpdate.limitedTimeline ? 0.0 : currentRoom.newSortOrder, + oldestSortOrder: + roomUpdate.limitedTimeline ? 0.0 : currentRoom.oldSortOrder, + ).toJson()); + } + + // Is the timeline limited? Then all previous messages should be + // removed from the database! + if (roomUpdate.limitedTimeline) { + await _timelineFragmentsBox + .delete(MultiKey(roomUpdate.id, '').toString()); + } + } + + @override + Future storeSSSSCache(int clientId, String type, String keyId, + String ciphertext, String content) async { + await _ssssCacheBox.put( + type, + SSSSCache( + type: type, + keyId: keyId, + ciphertext: ciphertext, + content: content, + ).toJson()); + } + + @override + Future storeSyncFilterId(String syncFilterId, int clientId) async { + await _clientBox.put('sync_filter_id', syncFilterId); + } + + @override + Future storeUserCrossSigningKey(int clientId, String userId, + String publicKey, String content, bool verified, bool blocked) async { + await _userCrossSigningKeysBox.put( + MultiKey(userId, publicKey).toString(), + { + 'user_id': userId, + 'public_key': publicKey, + 'content': content, + 'verified': verified ?? false, + 'blocked': blocked ?? false, + }, + ); + } + + @override + Future storeUserDeviceKey(int clientId, String userId, String deviceId, + String content, bool verified, bool blocked, int lastActive) async { + await _userDeviceKeysBox.put(MultiKey(userId, deviceId).toString(), { + 'user_id': userId, + 'device_id': deviceId, + 'content': content, + 'verified': verified ?? false, + 'blocked': blocked ?? false, + 'last_active': lastActive, + 'last_sent_message': '', + }); + return; + } + + @override + Future storeUserDeviceKeysInfo( + int clientId, String userId, bool outdated) async { + await _userDeviceKeysOutdatedBox.put(userId, outdated); + return; + } + + @override + Future transaction(Future Function() action) => action(); + + @override + Future updateClient( + String homeserverUrl, + String token, + String userId, + String deviceId, + String deviceName, + String prevBatch, + String olmAccount, + int clientId) async { + await _clientBox.put('homeserver_url', homeserverUrl); + await _clientBox.put('token', token); + await _clientBox.put('user_id', userId); + await _clientBox.put('device_id', deviceId); + await _clientBox.put('device_name', deviceName); + await _clientBox.put('prev_batch', prevBatch); + await _clientBox.put('olm_account', olmAccount); + return; + } + + @override + Future updateClientKeys(String olmAccount, int clientId) async { + await _clientBox.put('olm_account', olmAccount); + return; + } + + @override + Future updateInboundGroupSessionAllowedAtIndex(String allowedAtIndex, + int clientId, String roomId, String sessionId) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) { + Logs().w( + 'Tried to update inbound group session as uploaded which wasnt found in the database!'); + return; + } + raw['allowed_at_index'] = allowedAtIndex; + await _inboundGroupSessionsBox.put(sessionId, raw); + return; + } + + @override + Future updateInboundGroupSessionIndexes( + String indexes, int clientId, String roomId, String sessionId) async { + final raw = await _inboundGroupSessionsBox.get(sessionId); + if (raw == null) { + Logs().w( + 'Tried to update inbound group session indexes of a session which was not found in the database!'); + return; + } + raw['indexes'] = indexes; + await _inboundGroupSessionsBox.put(sessionId, raw); + return; + } + + @override + Future updateRoomSortOrder(double oldestSortOrder, + double newestSortOrder, int clientId, String roomId) async { + final raw = await _roomsBox.get(roomId); + raw['oldest_sort_order'] = oldestSortOrder; + raw['newest_sort_order'] = newestSortOrder; + await _roomsBox.put(roomId, raw); + return; + } +} + +Map convertToJson(Map map) { + final jsonMap = { + for (final entry in map.entries) ...{ + if (entry.value is Map) + '${entry.key.toString()}': convertToJson(entry.value), + if (!(entry.value is Map)) '${entry.key.toString()}': entry.value, + } + }; + return jsonMap; +} + +class MultiKey { + final List parts; + MultiKey(String key1, [String key2, String key3]) + : parts = [ + key1, + if (key2 != null) key2, + if (key3 != null) key3, + ]; + const MultiKey.byParts(this.parts); + + MultiKey.fromString(String multiKeyString) + : parts = multiKeyString.split('|'); + + @override + String toString() => parts.join('|'); + + @override + bool operator ==(other) => parts.toString() == other.toString(); +} diff --git a/pubspec.yaml b/pubspec.yaml index ca7ac8ca..1c861fcd 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -20,6 +20,7 @@ dependencies: olm: ^2.0.0 isolate: ^2.0.3 matrix_api_lite: ^0.3.3 + hive: ^2.0.4 dev_dependencies: test: ^1.15.7 @@ -27,5 +28,5 @@ dev_dependencies: moor_generator: ^4.0.0 build_runner: ^1.11.1 pedantic: ^1.11.0 - dapackages: ^1.4.0 + file: ^6.1.1 #flutter_test: {sdk: flutter} diff --git a/test/client_test.dart b/test/client_test.dart index 38d06cee..14b3f35d 100644 --- a/test/client_test.dart +++ b/test/client_test.dart @@ -540,7 +540,7 @@ void main() { newOlmAccount: pickledOlmAccount, ); - await Future.delayed(Duration(milliseconds: 50)); + await Future.delayed(Duration(milliseconds: 500)); expect(client1.isLogged(), true); expect(client1.rooms.length, 2); @@ -552,7 +552,7 @@ void main() { ); await client2.init(); - await Future.delayed(Duration(milliseconds: 100)); + await Future.delayed(Duration(milliseconds: 500)); expect(client2.isLogged(), true); expect(client2.accessToken, client1.accessToken); diff --git a/test/database_api_test.dart b/test/database_api_test.dart index f0bce0e8..1657144b 100644 --- a/test/database_api_test.dart +++ b/test/database_api_test.dart @@ -27,7 +27,10 @@ import 'fake_database_native.dart'; void main() { /// All Tests related to the ChatTime group('Moor Database Test', () { - testDatabase(getDatabase(null), 0); + testDatabase(getMoorDatabase(null), 0); + }); + group('Hive Database Test', () { + testDatabase(getHiveDatabase(null), 0); }); } diff --git a/test/encryption/key_request_test.dart b/test/encryption/key_request_test.dart index 59fa4408..fd111742 100644 --- a/test/encryption/key_request_test.dart +++ b/test/encryption/key_request_test.dart @@ -386,6 +386,9 @@ void main() { null, false); + // There is a non awaiting setInboundGroupSession call on the database + await Future.delayed(Duration(seconds: 1)); + await matrix.dispose(closeDatabase: true); }); }); diff --git a/test/encryption/key_verification_test.dart b/test/encryption/key_verification_test.dart index a253a9cf..1f0ad196 100644 --- a/test/encryption/key_verification_test.dart +++ b/test/encryption/key_verification_test.dart @@ -216,7 +216,7 @@ void main() { await client1.userDeviceKeys[client2.userID].startVerification(); expect(req1.state, KeyVerificationState.askSSSS); await req1.openSSSS(recoveryKey: ssssKey); - await Future.delayed(Duration(milliseconds: 10)); + await Future.delayed(Duration(seconds: 1)); expect(req1.state, KeyVerificationState.waitingAccept); await req1.cancel(); diff --git a/test/encryption/online_key_backup_test.dart b/test/encryption/online_key_backup_test.dart index 63420aaf..3b53c7e1 100644 --- a/test/encryption/online_key_backup_test.dart +++ b/test/encryption/online_key_backup_test.dart @@ -96,6 +96,7 @@ void main() { client.encryption.keyManager.setInboundGroupSession( roomId, sessionId, senderKey, sessionPayload, forwarded: true); + await Future.delayed(Duration(milliseconds: 500)); var dbSessions = await client.database.getInboundGroupSessionsToUpload(); expect(dbSessions.isNotEmpty, true); await client.encryption.keyManager.backgroundTasks(); diff --git a/test/fake_database_native.dart b/test/fake_database_native.dart index b0fc07bd..0bfc07e2 100644 --- a/test/fake_database_native.dart +++ b/test/fake_database_native.dart @@ -16,11 +16,35 @@ * along with this program. If not, see . */ +import 'dart:io'; +import 'dart:math'; + import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/database/hive_database.dart'; +import 'package:file/memory.dart'; +import 'package:hive/hive.dart'; import 'package:moor/moor.dart'; import 'package:moor/ffi.dart' as moor; -Future getDatabase(Client _) async { +Future getDatabase(Client _) => getHiveDatabase(_); + +Future getMoorDatabase(Client _) async { moorRuntimeOptions.dontWarnAboutMultipleDatabases = true; return Database(moor.VmDatabase.memory()); } + +bool hiveInitialized = false; + +Future getHiveDatabase(Client c) async { + if (!hiveInitialized) { + final fileSystem = MemoryFileSystem(); + final testHivePath = + '${fileSystem.path}/build/.test_store/${Random().nextDouble()}'; + Directory(testHivePath).createSync(recursive: true); + Hive.init(testHivePath); + hiveInitialized = true; + } + final db = FamedlySdkHiveDatabase('unit_test.${c.hashCode}'); + await db.open(); + return db; +}