diff --git a/lib/src/client.dart b/lib/src/client.dart index e1832512..9b820141 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -58,6 +58,8 @@ class Client { Set verificationMethods; + Set importantStateEvents; + /// Create a client /// clientName = unique identifier of this client /// debug: Print debug output? @@ -66,13 +68,37 @@ class Client { /// verificationMethods: A set of all the verification methods this client can handle. Includes: /// KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported /// KeyVerificationMethod.emoji: Compare emojis + /// importantStateEvents: A set of all the important state events to load when the client connects. + /// To speed up performance only a set of state events is loaded on startup, those that are + /// needed to display a room list. All the remaining state events are automatically post-loaded + /// when opening the timeline of a room or manually by calling `room.postLoad()`. + /// This set will always include the following state events: + /// - m.room.name + /// - m.room.avatar + /// - m.room.message + /// - m.room.encrypted + /// - m.room.encryption + /// - m.room.canonical_alias + /// - m.room.tombstone + /// - *some* m.room.member events, where needed Client(this.clientName, {this.debug = false, this.database, this.enableE2eeRecovery = false, this.verificationMethods, - http.Client httpClient}) { + http.Client httpClient, + this.importantStateEvents}) { verificationMethods ??= {}; + importantStateEvents ??= {}; + importantStateEvents.addAll([ + EventTypes.RoomName, + EventTypes.RoomAvatar, + EventTypes.Message, + EventTypes.Encrypted, + EventTypes.Encryption, + EventTypes.RoomCanonicalAlias, + EventTypes.RoomTombstone, + ]); api = MatrixApi(debug: debug, httpClient: httpClient); onLoginStateChanged.stream.listen((loginState) { if (debug) { @@ -644,7 +670,7 @@ class Client { _rooms = await database.getRoomList(this, onlyLeft: false); _sortRooms(); accountData = await database.getAccountData(id); - presences = await database.getPresences(id); + presences.clear(); } onLoginStateChanged.add(LoginState.logged); @@ -744,14 +770,6 @@ class Client { } if (sync.presence != null) { for (final newPresence in sync.presence) { - if (database != null) { - await database.storePresence( - id, - newPresence.type, - newPresence.senderId, - jsonEncode(newPresence.toJson()), - ); - } presences[newPresence.senderId] = newPresence; onPresence.add(newPresence); } @@ -969,6 +987,16 @@ class Client { if (event['type'] == EventTypes.Encrypted && encryptionEnabled) { update = await update.decrypt(room); } + if (event['type'] == EventTypes.Message && + !room.isDirectChat && + database != null && + room.getState(EventTypes.RoomMember, event['sender']) == null) { + // In order to correctly render room list previews we need to fetch the member from the database + final user = await database.getUser(id, event['sender'], room); + if (user != null) { + room.setState(user); + } + } if (type != 'ephemeral' && database != null) { await database.storeEventUpdate(id, update); } diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 39a37839..c329d0b0 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -157,9 +157,12 @@ class Database extends _$Database { ? t.membership.equals('leave') : t.membership.equals('leave').not())) .get(); - final resStates = await getAllRoomStates(client.id).get(); + final resStates = await getImportantRoomStates( + client.id, client.importantStateEvents.toList()) + .get(); final resAccountData = await getAllRoomAccountData(client.id).get(); final roomList = []; + final allMembersToPostload = >{}; for (final r in res) { final room = await sdk.Room.getRoomFromTableRow( r, @@ -168,6 +171,81 @@ class Database extends _$Database { roomAccountData: resAccountData.where((rs) => rs.roomId == r.roomId), ); roomList.add(room); + // let's see if we need any m.room.member events + final membersToPostload = {}; + // the lastEvent message preview might have an author we need to fetch, if it is a group chat + if (room.getState(EventTypes.Message) != null && !room.isDirectChat) { + membersToPostload.add(room.getState(EventTypes.Message).senderId); + } + // if the room has no name and no canonical alias, its name is calculated + // based on the heroes of the room + if (room.getState(EventTypes.RoomName) == null && + room.getState(EventTypes.RoomCanonicalAlias) == null && + room.mHeroes != null) { + // we don't have a name and no canonical alias, so we'll need to + // post-load the heroes + membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty)); + } + // okay, only load from the database if we actually have stuff to load + if (membersToPostload.isNotEmpty) { + // save it for loading later + allMembersToPostload[room.id] = membersToPostload; + } + } + // now we postload all members, if thre are any + if (allMembersToPostload.isNotEmpty) { + // we will generate a query to fetch as many events as possible at once, as that + // significantly improves performance. However, to prevent too large queries from being constructed, + // we limit to only fetching 500 rooms at once. + // This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts), + // however this very conservative value should be on the safe side. + const MAX_ROOMS_PER_QUERY = 500; + // as we iterate over our entries in separate chunks one-by-one we use an iterator + // which persists accross the chunks, and thus we just re-sume iteration at the place + // we prreviously left off. + final entriesIterator = allMembersToPostload.entries.iterator; + // now we iterate over all our 500-room-chunks... + for (var i = 0; + i < allMembersToPostload.keys.length; + i += MAX_ROOMS_PER_QUERY) { + // query the current chunk and build the query + final membersRes = await (select(roomStates) + ..where((s) { + // all chunks have to have the reight client id and must be of type `m.room.member` + final basequery = s.clientId.equals(client.id) & + s.type.equals('m.room.member'); + // this is where the magic happens. Here we build a query with the form + // OR room_id = '!roomId1' AND state_key IN ('@member') OR room_id = '!roomId2' AND state_key IN ('@member') + // subqueries holds our query fragment + Expression subqueries; + // here we iterate over our chunk....we musn't forget to progress our iterator! + // we must check for if our chunk is done *before* progressing the + // iterator, else we might progress it twice around chunk edges, missing on rooms + for (var j = 0; + j < MAX_ROOMS_PER_QUERY && entriesIterator.moveNext(); + j++) { + final entry = entriesIterator.current; + // builds room_id = '!roomId1' AND state_key IN ('@member') + final q = + s.roomId.equals(entry.key) & s.stateKey.isIn(entry.value); + // adds it either as the start of subqueries or as a new OR condition to it + if (subqueries == null) { + subqueries = q; + } else { + subqueries = subqueries | q; + } + } + // combinde the basequery with the subquery together, giving our final query + return basequery & subqueries; + })) + .get(); + // now that we got all the entries from the database, set them as room states + for (final dbMember in membersRes) { + final room = roomList.firstWhere((r) => r.id == dbMember.roomId); + final event = sdk.Event.fromDb(dbMember, room); + room.setState(event); + } + } } return roomList; } @@ -192,22 +270,6 @@ class Database extends _$Database { return newAccountData; } - Future> getPresences(int clientId) async { - final newPresences = {}; - final rawPresences = await getAllPresences(clientId).get(); - for (final d in rawPresences) { - // TODO: Why is this not working? - try { - final content = sdk.Event.getMapFromPayload(d.content); - var presence = api.Presence.fromJson(content); - presence.senderId = d.sender; - presence.type = d.type; - newPresences[d.sender] = api.Presence.fromJson(content); - } catch (_) {} - } - return newPresences; - } - /// Stores a RoomUpdate object in the database. Must be called inside of /// [transaction]. final Set _ensuredRooms = {}; diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index d51eec59..e17101d1 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -5999,35 +5999,6 @@ abstract class _$Database extends GeneratedDatabase { ); } - DbPresence _rowToDbPresence(QueryRow row) { - return DbPresence( - clientId: row.readInt('client_id'), - type: row.readString('type'), - sender: row.readString('sender'), - content: row.readString('content'), - ); - } - - Selectable getAllPresences(int client_id) { - return customSelect('SELECT * FROM presences WHERE client_id = :client_id', - variables: [Variable.withInt(client_id)], - readsFrom: {presences}).map(_rowToDbPresence); - } - - Future storePresence( - int client_id, String type, String sender, String content) { - return customInsert( - 'INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content)', - variables: [ - Variable.withInt(client_id), - Variable.withString(type), - Variable.withString(sender), - Variable.withString(content) - ], - updates: {presences}, - ); - } - Future updateEvent(String unsigned, String content, String prev_content, int client_id, String event_id, String room_id) { return customUpdate( @@ -6077,6 +6048,22 @@ abstract class _$Database extends GeneratedDatabase { ); } + Selectable getImportantRoomStates( + int client_id, List events) { + var $arrayStartIndex = 2; + final expandedevents = $expandVar($arrayStartIndex, events.length); + $arrayStartIndex += events.length; + return customSelect( + 'SELECT * FROM room_states WHERE client_id = :client_id AND type IN ($expandedevents)', + variables: [ + Variable.withInt(client_id), + for (var $ in events) Variable.withString($) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); + } + Selectable getAllRoomStates(int client_id) { return customSelect( 'SELECT * FROM room_states WHERE client_id = :client_id', @@ -6084,6 +6071,23 @@ abstract class _$Database extends GeneratedDatabase { readsFrom: {roomStates}).map(_rowToDbRoomState); } + Selectable getUnimportantRoomStatesForRoom( + int client_id, String room_id, List events) { + var $arrayStartIndex = 3; + final expandedevents = $expandVar($arrayStartIndex, events.length); + $arrayStartIndex += events.length; + return customSelect( + 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN ($expandedevents)', + variables: [ + Variable.withInt(client_id), + Variable.withString(room_id), + for (var $ in events) Variable.withString($) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); + } + Future storeEvent( int client_id, String event_id, @@ -6192,6 +6196,23 @@ abstract class _$Database extends GeneratedDatabase { }).map(_rowToDbRoomState); } + Selectable dbGetUsers( + int client_id, List mxids, String room_id) { + var $arrayStartIndex = 2; + final expandedmxids = $expandVar($arrayStartIndex, mxids.length); + $arrayStartIndex += mxids.length; + return customSelect( + 'SELECT * FROM room_states WHERE client_id = :client_id AND type = \'m.room.member\' AND state_key IN ($expandedmxids) AND room_id = :room_id', + variables: [ + Variable.withInt(client_id), + for (var $ in mxids) Variable.withString($), + Variable.withString(room_id) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); + } + DbEvent _rowToDbEvent(QueryRow row) { return DbEvent( clientId: row.readInt('client_id'), diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index c2e5ff0c..716bdc64 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -206,11 +206,11 @@ setRoomPrevBatch: UPDATE rooms SET prev_batch = :prev_batch WHERE client_id = :c updateRoomSortOrder: UPDATE rooms SET oldest_sort_order = :oldest_sort_order, newest_sort_order = :newest_sort_order WHERE client_id = :client_id AND room_id = :room_id; getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id; storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content); -getAllPresences: SELECT * FROM presences WHERE client_id = :client_id; -storePresence: INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content); updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id; +getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events; getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id; +getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events; storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status); storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key); getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id; diff --git a/lib/src/room.dart b/lib/src/room.dart index 6f93ea34..40aa4312 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -101,6 +101,25 @@ class Room { _oldestSortOrder, _newestSortOrder, client.id, id); } + /// Flag if the room is partial, meaning not all state events have been loaded yet + bool partial = true; + + /// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing. + Future postLoad() async { + if (!partial || client.database == null) { + return; + } + final allStates = await client.database + .getUnimportantRoomStatesForRoom( + client.id, id, client.importantStateEvents.toList()) + .get(); + for (final state in allStates) { + final newState = Event.fromDb(state, this); + setState(newState); + } + partial = false; + } + /// Returns the [Event] for the given [typeKey] and optional [stateKey]. /// If no [stateKey] is provided, it defaults to an empty string. Event getState(String typeKey, [String stateKey = '']) => @@ -934,6 +953,7 @@ class Room { Future getTimeline( {onTimelineUpdateCallback onUpdate, onTimelineInsertCallback onInsert}) async { + await postLoad(); var events; if (client.database != null) { events = await client.database.getEventList(client.id, this); @@ -1033,6 +1053,15 @@ class Room { if (getState(EventTypes.RoomMember, mxID) != null) { return getState(EventTypes.RoomMember, mxID).asUser; } + if (client.database != null) { + // it may be in the database + final user = await client.database.getUser(client.id, mxID, this); + if (user != null) { + setState(user); + if (onUpdate != null) onUpdate.add(id); + return user; + } + } if (mxID == null || !_requestingMatrixIds.add(mxID)) return null; Map resp; try {