feat: Implement upload sync filters

lib/src/client.dart

@@ -86,6 +86,9 @@ class Client extends MatrixApi {
 
   // For CommandsClientExtension
   final Map<String, FutureOr<String> Function(CommandArgs)> commands = {};
+  final Filter syncFilter;
+
+  String syncFilterId;
 
   /// Create a client
   /// [clientName] = unique identifier of this client
@@ -117,7 +120,8 @@ class Client extends MatrixApi {
   /// be formatted in the way, that all "_" characters are becomming white spaces and
   /// the first character of each word becomes uppercase.
   /// If your client supports more login types like login with token or SSO, then add this to
-  /// [supportedLoginTypes].
+  /// [supportedLoginTypes]. Set a custom [syncFilter] if you like. By default the app
+  /// will use lazy_load_members.
   Client(
     this.clientName, {
     this.databaseBuilder,
@@ -130,8 +134,14 @@ class Client extends MatrixApi {
     this.sendMessageTimeoutSeconds = 60,
     this.requestHistoryOnLimitedTimeline = false,
     this.supportedLoginTypes,
+    Filter syncFilter,
     @deprecated bool debug,
-  }) {
+  }) : syncFilter = syncFilter ??
+            Filter(
+              room: RoomFilter(
+                state: StateFilter(lazyLoadMembers: true),
+              ),
+            ) {
     supportedLoginTypes ??= {AuthenticationTypes.password};
     verificationMethods ??= <KeyVerificationMethod>{};
     importantStateEvents ??= {};
@@ -647,9 +657,6 @@ class Client extends MatrixApi {
       : null;
 
   static const Set<String> supportedVersions = {'r0.5.0', 'r0.6.0'};
-  static const String syncFilters =
-      '{"room":{"state":{"lazy_load_members":true}}}';
-  static const String messagesFilters = '{"lazy_load_members":true}';
   static const List<String> supportedDirectEncryptionAlgorithms = [
     AlgorithmTypes.olmV1Curve25519AesSha2
   ];
@@ -805,6 +812,7 @@ class Client extends MatrixApi {
           _userID = account.userId;
           _deviceID = account.deviceId;
           _deviceName = account.deviceName;
+          syncFilterId = account.syncFilterId;
           prevBatch = account.prevBatch;
           olmAccount = account.olmAccount;
         }
@@ -903,7 +911,7 @@ class Client extends MatrixApi {
   void clear() {
     Logs().outputEvents.clear();
     database?.clear(id);
-    _id = accessToken =
+    _id = accessToken = syncFilterId =
         homeserver = _userID = _deviceID = _deviceName = prevBatch = null;
     _rooms = [];
     encryption?.dispose();
@@ -946,14 +954,23 @@ class Client extends MatrixApi {
   /// Presence that is set on sync.
   PresenceType syncPresence;
 
+  Future<void> _checkSyncFilter() async {
+    if (syncFilterId == null) {
+      syncFilterId = await uploadFilter(userID, syncFilter);
+      await database?.storeSyncFilterId(syncFilterId, id);
+    }
+    return;
+  }
+
   Future<void> _innerSync() async {
     await _retryDelay;
     _retryDelay = Future.delayed(Duration(seconds: syncErrorTimeoutSec));
     if (!isLogged() || _disposed || _aborted) return null;
     try {
       var syncError;
+      await _checkSyncFilter();
       final syncRequest = sync(
-        filter: syncFilters,
+        filter: syncFilterId,
         since: prevBatch,
         timeout: prevBatch != null ? 30000 : null,
         setPresence: syncPresence,

lib/src/database/database.dart

@@ -70,7 +70,7 @@ class Database extends _$Database {
   Database.connect(DatabaseConnection connection) : super.connect(connection);
 
   @override
-  int get schemaVersion => 11;
+  int get schemaVersion => 12;
 
   int get maxFileSize => 1 * 1024 * 1024;
 
@@ -175,6 +175,10 @@ class Database extends _$Database {
               await m.createIndexIfNotExists(toDeviceQueueIndex);
               from++;
             }
+            if (from == 11) {
+              await m.addColumnIfNotExists(clients, clients.syncFilterId);
+              from++;
+            }
           } catch (e, s) {
             api.Logs().e('Database migration failed', e, s);
             onError.add(SdkError(exception: e, stackTrace: s));

lib/src/database/database.g.dart

@@ -16,6 +16,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
   final String deviceId;
   final String deviceName;
   final String prevBatch;
+  final String syncFilterId;
   final String olmAccount;
   DbClient(
       {@required this.clientId,
@@ -26,6 +27,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
       this.deviceId,
       this.deviceName,
       this.prevBatch,
+      this.syncFilterId,
       this.olmAccount});
   factory DbClient.fromData(Map<String, dynamic> data, GeneratedDatabase db,
       {String prefix}) {
@@ -48,6 +50,8 @@ class DbClient extends DataClass implements Insertable<DbClient> {
           .mapFromDatabaseResponse(data['${effectivePrefix}device_name']),
       prevBatch: stringType
           .mapFromDatabaseResponse(data['${effectivePrefix}prev_batch']),
+      syncFilterId: stringType
+          .mapFromDatabaseResponse(data['${effectivePrefix}sync_filter_id']),
       olmAccount: stringType
           .mapFromDatabaseResponse(data['${effectivePrefix}olm_account']),
     );
@@ -79,6 +83,9 @@ class DbClient extends DataClass implements Insertable<DbClient> {
     if (!nullToAbsent || prevBatch != null) {
       map['prev_batch'] = Variable<String>(prevBatch);
     }
+    if (!nullToAbsent || syncFilterId != null) {
+      map['sync_filter_id'] = Variable<String>(syncFilterId);
+    }
     if (!nullToAbsent || olmAccount != null) {
       map['olm_account'] = Variable<String>(olmAccount);
     }
@@ -107,6 +114,9 @@ class DbClient extends DataClass implements Insertable<DbClient> {
       prevBatch: prevBatch == null && nullToAbsent
           ? const Value.absent()
           : Value(prevBatch),
+      syncFilterId: syncFilterId == null && nullToAbsent
+          ? const Value.absent()
+          : Value(syncFilterId),
       olmAccount: olmAccount == null && nullToAbsent
           ? const Value.absent()
           : Value(olmAccount),
@@ -125,6 +135,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
       deviceId: serializer.fromJson<String>(json['device_id']),
       deviceName: serializer.fromJson<String>(json['device_name']),
       prevBatch: serializer.fromJson<String>(json['prev_batch']),
+      syncFilterId: serializer.fromJson<String>(json['sync_filter_id']),
       olmAccount: serializer.fromJson<String>(json['olm_account']),
     );
   }
@@ -140,6 +151,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
       'device_id': serializer.toJson<String>(deviceId),
       'device_name': serializer.toJson<String>(deviceName),
       'prev_batch': serializer.toJson<String>(prevBatch),
+      'sync_filter_id': serializer.toJson<String>(syncFilterId),
       'olm_account': serializer.toJson<String>(olmAccount),
     };
   }
@@ -153,6 +165,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
           String deviceId,
           String deviceName,
           String prevBatch,
+          String syncFilterId,
           String olmAccount}) =>
       DbClient(
         clientId: clientId ?? this.clientId,
@@ -163,6 +176,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
         deviceId: deviceId ?? this.deviceId,
         deviceName: deviceName ?? this.deviceName,
         prevBatch: prevBatch ?? this.prevBatch,
+        syncFilterId: syncFilterId ?? this.syncFilterId,
         olmAccount: olmAccount ?? this.olmAccount,
       );
   @override
@@ -176,6 +190,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
           ..write('deviceId: $deviceId, ')
           ..write('deviceName: $deviceName, ')
           ..write('prevBatch: $prevBatch, ')
+          ..write('syncFilterId: $syncFilterId, ')
           ..write('olmAccount: $olmAccount')
           ..write(')'))
         .toString();
@@ -196,8 +211,10 @@ class DbClient extends DataClass implements Insertable<DbClient> {
                           deviceId.hashCode,
                           $mrjc(
                               deviceName.hashCode,
-                              $mrjc(prevBatch.hashCode,
+                              $mrjc(
-                                  olmAccount.hashCode)))))))));
+                                  prevBatch.hashCode,
+                                  $mrjc(syncFilterId.hashCode,
+                                      olmAccount.hashCode))))))))));
   @override
   bool operator ==(dynamic other) =>
       identical(this, other) ||
@@ -210,6 +227,7 @@ class DbClient extends DataClass implements Insertable<DbClient> {
           other.deviceId == this.deviceId &&
           other.deviceName == this.deviceName &&
           other.prevBatch == this.prevBatch &&
+          other.syncFilterId == this.syncFilterId &&
           other.olmAccount == this.olmAccount);
 }
 
@@ -222,6 +240,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
   final Value<String> deviceId;
   final Value<String> deviceName;
   final Value<String> prevBatch;
+  final Value<String> syncFilterId;
   final Value<String> olmAccount;
   const ClientsCompanion({
     this.clientId = const Value.absent(),
@@ -232,6 +251,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
     this.deviceId = const Value.absent(),
     this.deviceName = const Value.absent(),
     this.prevBatch = const Value.absent(),
+    this.syncFilterId = const Value.absent(),
     this.olmAccount = const Value.absent(),
   });
   ClientsCompanion.insert({
@@ -243,6 +263,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
     this.deviceId = const Value.absent(),
     this.deviceName = const Value.absent(),
     this.prevBatch = const Value.absent(),
+    this.syncFilterId = const Value.absent(),
     this.olmAccount = const Value.absent(),
   })  : name = Value(name),
         homeserverUrl = Value(homeserverUrl),
@@ -257,6 +278,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
     Expression<String> deviceId,
     Expression<String> deviceName,
     Expression<String> prevBatch,
+    Expression<String> syncFilterId,
     Expression<String> olmAccount,
   }) {
     return RawValuesInsertable({
@@ -268,6 +290,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
       if (deviceId != null) 'device_id': deviceId,
       if (deviceName != null) 'device_name': deviceName,
       if (prevBatch != null) 'prev_batch': prevBatch,
+      if (syncFilterId != null) 'sync_filter_id': syncFilterId,
       if (olmAccount != null) 'olm_account': olmAccount,
     });
   }
@@ -281,6 +304,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
       Value<String> deviceId,
       Value<String> deviceName,
       Value<String> prevBatch,
+      Value<String> syncFilterId,
       Value<String> olmAccount}) {
     return ClientsCompanion(
       clientId: clientId ?? this.clientId,
@@ -291,6 +315,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
       deviceId: deviceId ?? this.deviceId,
       deviceName: deviceName ?? this.deviceName,
       prevBatch: prevBatch ?? this.prevBatch,
+      syncFilterId: syncFilterId ?? this.syncFilterId,
       olmAccount: olmAccount ?? this.olmAccount,
     );
   }
@@ -322,6 +347,9 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
     if (prevBatch.present) {
       map['prev_batch'] = Variable<String>(prevBatch.value);
     }
+    if (syncFilterId.present) {
+      map['sync_filter_id'] = Variable<String>(syncFilterId.value);
+    }
     if (olmAccount.present) {
       map['olm_account'] = Variable<String>(olmAccount.value);
     }
@@ -339,6 +367,7 @@ class ClientsCompanion extends UpdateCompanion<DbClient> {
           ..write('deviceId: $deviceId, ')
           ..write('deviceName: $deviceName, ')
           ..write('prevBatch: $prevBatch, ')
+          ..write('syncFilterId: $syncFilterId, ')
           ..write('olmAccount: $olmAccount')
           ..write(')'))
         .toString();
@@ -417,6 +446,16 @@ class Clients extends Table with TableInfo<Clients, DbClient> {
         $customConstraints: '');
   }
 
+  final VerificationMeta _syncFilterIdMeta =
+      const VerificationMeta('syncFilterId');
+  GeneratedTextColumn _syncFilterId;
+  GeneratedTextColumn get syncFilterId =>
+      _syncFilterId ??= _constructSyncFilterId();
+  GeneratedTextColumn _constructSyncFilterId() {
+    return GeneratedTextColumn('sync_filter_id', $tableName, true,
+        $customConstraints: '');
+  }
+
   final VerificationMeta _olmAccountMeta = const VerificationMeta('olmAccount');
   GeneratedTextColumn _olmAccount;
   GeneratedTextColumn get olmAccount => _olmAccount ??= _constructOlmAccount();
@@ -435,6 +474,7 @@ class Clients extends Table with TableInfo<Clients, DbClient> {
         deviceId,
         deviceName,
         prevBatch,
+        syncFilterId,
         olmAccount
       ];
   @override
@@ -492,6 +532,12 @@ class Clients extends Table with TableInfo<Clients, DbClient> {
       context.handle(_prevBatchMeta,
           prevBatch.isAcceptableOrUnknown(data['prev_batch'], _prevBatchMeta));
     }
+    if (data.containsKey('sync_filter_id')) {
+      context.handle(
+          _syncFilterIdMeta,
+          syncFilterId.isAcceptableOrUnknown(
+              data['sync_filter_id'], _syncFilterIdMeta));
+    }
     if (data.containsKey('olm_account')) {
       context.handle(
           _olmAccountMeta,
@@ -6625,6 +6671,18 @@ abstract class _$Database extends GeneratedDatabase {
     );
   }
 
+  Future<int> storeSyncFilterId(String sync_filter_id, int client_id) {
+    return customUpdate(
+      'UPDATE clients SET sync_filter_id = :sync_filter_id WHERE client_id = :client_id',
+      variables: [
+        Variable.withString(sync_filter_id),
+        Variable.withInt(client_id)
+      ],
+      updates: {clients},
+      updateKind: UpdateKind.update,
+    );
+  }
+
   Selectable<DbUserDeviceKey> getAllUserDeviceKeys(int client_id) {
     return customSelect(
         'SELECT * FROM user_device_keys WHERE client_id = :client_id',

lib/src/database/database.moor

@@ -9,6 +9,7 @@ CREATE TABLE clients (
   device_id TEXT,
   device_name TEXT,
   prev_batch TEXT,
+  sync_filter_id TEXT,
   olm_account TEXT,
   UNIQUE(name)
 ) AS DbClient;
@@ -189,6 +190,7 @@ dbGetClient: SELECT * FROM clients WHERE name = :name;
 updateClient: UPDATE clients SET homeserver_url = :homeserver_url, token = :token, user_id = :user_id, device_id = :device_id, device_name = :device_name, prev_batch = :prev_batch, olm_account = :olm_account WHERE client_id = :client_id;
 updateClientKeys: UPDATE clients SET olm_account = :olm_account WHERE client_id = :client_id;
 storePrevBatch: UPDATE clients SET prev_batch = :prev_batch WHERE client_id = :client_id;
+storeSyncFilterId: UPDATE clients SET sync_filter_id = :sync_filter_id WHERE client_id = :client_id;
 getAllUserDeviceKeys: SELECT * FROM user_device_keys WHERE client_id = :client_id;
 getAllUserDeviceKeysKeys: SELECT * FROM user_device_keys_key WHERE client_id = :client_id;
 getAllUserCrossSigningKeys: SELECT * FROM user_cross_signing_keys WHERE client_id = :client_id;

lib/src/room.dart

@@ -949,7 +949,7 @@ class Room {
       prev_batch,
       Direction.b,
       limit: historyCount,
-      filter: Client.messagesFilters,
+      filter: jsonEncode(StateFilter(lazyLoadMembers: true).toJson()),
     );
 
     if (onHistoryReceived != null) onHistoryReceived();

pubspec.yaml

@@ -23,7 +23,7 @@ dependencies:
   matrix_file_e2ee: ^1.0.5
   isolate: ^2.0.3
   logger: ^0.9.4
-  matrix_api_lite: ^0.1.7
+  matrix_api_lite: ^0.1.8
 
 dev_dependencies:
   test: ^1.15.7

test/encryption/bootstrap_test.dart

@@ -48,6 +48,7 @@ void main() {
 
     test('setupClient', () async {
       client = await getClient();
+      await client.abortSync();
     });
 
     test('setup', () async {

test/encryption/encrypt_decrypt_to_device_test.dart

@@ -52,6 +52,7 @@ void main() {
 
     test('setupClient', () async {
       client = await getClient();
+      await client.abortSync();
       await otherClient.checkHomeserver('https://fakeserver.notexisting',
           checkWellKnown: false);
       otherClient.init(
@@ -62,6 +63,7 @@ void main() {
         newDeviceID: 'FOXDEVICE',
         newOlmAccount: otherPickledOlmAccount,
       );
+      await otherClient.abortSync();
 
       await Future.delayed(Duration(milliseconds: 10));
       device = DeviceKeys.fromJson({

test/fake_matrix_api.dart

@@ -110,7 +110,7 @@ class FakeMatrixApi extends MockClient {
             res = {'event_id': '\$event${FakeMatrixApi.eventCounter++}'};
           } else if (action.contains('/client/r0/sync')) {
             res = {
-              'next_batch': DateTime.now().millisecondsSinceEpoch.toString
+              'next_batch': DateTime.now().millisecondsSinceEpoch.toString(),
             };
           } else if (method == 'PUT' &&
               client != null &&
@@ -1593,13 +1593,12 @@ class FakeMatrixApi extends MockClient {
           },
       '/client/r0/sync?filter=%7B%22room%22%3A%7B%22include_leave%22%3Atrue%2C%22timeline%22%3A%7B%22limit%22%3A10%7D%7D%7D&timeout=0':
           (var req) => archiveSyncResponse,
-      '/client/r0/sync?filter=%7B%22room%22%3A%7B%22state%22%3A%7B%22lazy_load_members%22%3Atrue%7D%7D%7D':
+      '/client/r0/sync?filter=1234': (var req) => syncResponse,
-          (var req) => syncResponse,
+      '/client/r0/sync?filter=1234&since=1234&full_state=false&set_presence=unavailable&timeout=15':
-      '/client/r0/sync?filter=%7B%7D&since=1234&full_state=false&set_presence=unavailable&timeout=15':
           (var req) => syncResponse,
       '/client/r0/register/available?username=testuser': (var req) =>
           {'available': true},
-      '/client/r0/user/${Uri.encodeComponent('alice@example.com')}/filter/1234':
+      '/client/r0/user/${Uri.encodeComponent('@test:fakeServer.notExisting')}/filter/1234':
           (var req) => {
                 'room': {
                   'state': {
@@ -1777,7 +1776,9 @@ class FakeMatrixApi extends MockClient {
       '/client/r0/rooms/!localpart%3Aexample.com/receipt/m.read/%241234%3Aexample.com':
           (var req) => {},
       '/client/r0/rooms/!localpart%3Aexample.com/read_markers': (var req) => {},
-      '/client/r0/user/${Uri.encodeComponent('alice@example.com')}/filter':
+      '/client/r0/user/${Uri.encodeComponent('@othertest:fakeServer.notExisting')}/filter':
+          (var req) => {'filter_id': '1234'},
+      '/client/r0/user/${Uri.encodeComponent('@test:fakeServer.notExisting')}/filter':
           (var req) => {'filter_id': '1234'},
       '/client/r0/publicRooms?server=example.com': (var req) => {
             'chunk': [

test/timeline_test.dart

@@ -28,7 +28,6 @@ import 'package:olm/olm.dart' as olm;
 import 'fake_client.dart';
 
 void main() {
-  /// All Tests related to the MxContent
   group('Timeline', () {
     Logs().level = Level.error;
     final roomID = '!1234:example.com';