feat: Implement new Matrix Dart SDK Database

fix: Edit last event breaks db

feat: Add native sqflite indexeddb database

feat: Split up preload and nonpreload room state boxes
This commit is contained in:
Christian Pauly 2022-08-28 08:40:05 +02:00 committed by Krille
parent 562363c263
commit 6db019ae23
No known key found for this signature in database
10 changed files with 2266 additions and 5 deletions

View File

@ -64,7 +64,7 @@ jobs:
- uses: actions/checkout@v3
- name: Run tests
run: |
apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl lcov python3 python3-distutils
apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl lcov python3 python3-distutils libsqlite3-dev
curl -o /bin/lcov_cobertura.py https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py && sed 's/env python/env python3/' -i /bin/lcov_cobertura.py && chmod +x /bin/lcov_cobertura.py
dart pub get
./scripts/test.sh

View File

@ -46,7 +46,7 @@ coverage_without_olm:
variables:
NO_OLM: 1
before_script:
- apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl lcov python3 python3-distutils
- apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl lcov python3 python3-distutils libsqlite3-dev
- curl -o /bin/lcov_cobertura.py https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py && sed 's/env python/env python3/' -i /bin/lcov_cobertura.py && chmod +x /bin/lcov_cobertura.py
script:
- dart pub get

View File

@ -24,6 +24,7 @@ export 'package:matrix_api_lite/matrix_api_lite.dart';
export 'src/client.dart';
export 'src/database/database_api.dart';
export 'src/database/hive_database.dart';
export 'src/database/matrix_sdk_database.dart';
export 'src/database/hive_collections_database.dart';
export 'src/event.dart';
export 'src/presence.dart';

View File

@ -0,0 +1,216 @@
import 'dart:async';
import 'dart:html';
import 'dart:indexed_db';
/// Key-Value store abstraction over IndexedDB so that the sdk database can use
/// a single interface for all platforms. API is inspired by Hive.
class BoxCollection {
final Database _db;
final Set<String> boxNames;
BoxCollection(this._db, this.boxNames);
static Future<BoxCollection> open(
String name,
Set<String> boxNames, {
Object? sqfliteDatabase,
IdbFactory? idbFactory,
}) async {
idbFactory ??= window.indexedDB!;
final db = await idbFactory.open(name, version: 1,
onUpgradeNeeded: (VersionChangeEvent event) {
final db = event.target.result;
for (final name in boxNames) {
db.createObjectStore(name, autoIncrement: true);
}
});
return BoxCollection(db, boxNames);
}
Box<V> openBox<V>(String name) {
if (!boxNames.contains(name)) {
throw ('Box with name $name is not in the known box names of this collection.');
}
return Box<V>(name, this);
}
List<Future<void> Function(Transaction txn)>? _txnCache;
Future<void> transaction(
Future<void> Function() action, {
List<String>? boxNames,
bool readOnly = false,
}) async {
boxNames ??= _db.objectStoreNames!.toList();
_txnCache = [];
await action();
final cache = List<Future<void> Function(Transaction txn)>.from(_txnCache!);
_txnCache = null;
if (cache.isEmpty) return;
final txn = _db.transaction(boxNames, readOnly ? 'readonly' : 'readwrite');
for (final fun in cache) {
// The IDB methods return a Future in Dart but must not be awaited in
// order to have an actual transaction. They must only be performed and
// then the transaction object must call `txn.completed;` which then
// returns the actual future.
// https://developer.mozilla.org/en-US/docs/Web/API/IDBTransaction
unawaited(fun(txn));
}
await txn.completed;
return;
}
Future<void> clear() async {
for (final name in boxNames) {
_db.deleteObjectStore(name);
}
}
Future<void> close() async {
assert(_txnCache == null, 'Database closed while in transaction!');
return _db.close();
}
}
class Box<V> {
final String name;
final BoxCollection boxCollection;
final Map<String, V?> _cache = {};
/// _cachedKeys is only used to make sure that if you fetch all keys from a
/// box, you do not need to have an expensive read operation twice. There is
/// no other usage for this at the moment. So the cache is never partial.
/// Once the keys are cached, they need to be updated when changed in put and
/// delete* so that the cache does not become outdated.
Set<String>? _cachedKeys;
bool get _keysCached => _cachedKeys != null;
Box(this.name, this.boxCollection);
Future<List<String>> getAllKeys([Transaction? txn]) async {
if (_keysCached) return _cachedKeys!.toList();
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
final request = store.getAllKeys(null);
await request.onSuccess.first;
final keys = request.result.cast<String>();
_cachedKeys = keys.toSet();
return keys;
}
Future<Map<String, V>> getAllValues([Transaction? txn]) async {
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
final map = <String, V>{};
final cursorStream = store.openCursor(autoAdvance: true);
await for (final cursor in cursorStream) {
map[cursor.key as String] = _fromValue(cursor.value) as V;
}
return map;
}
Future<V?> get(String key, [Transaction? txn]) async {
if (_cache.containsKey(key)) return _cache[key];
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
_cache[key] = await store.getObject(key).then(_fromValue);
return _cache[key];
}
Future<List<V?>> getAll(List<String> keys, [Transaction? txn]) async {
if (keys.every((key) => _cache.containsKey(key))) {
return keys.map((key) => _cache[key]).toList();
}
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
final list = await Future.wait(
keys.map((key) => store.getObject(key).then(_fromValue)));
for (var i = 0; i < keys.length; i++) {
_cache[keys[i]] = list[i];
}
return list;
}
Future<void> put(String key, V val, [Transaction? txn]) async {
if (boxCollection._txnCache != null) {
boxCollection._txnCache!.add((txn) => put(key, val, txn));
_cache[key] = val;
_cachedKeys?.add(key);
return;
}
txn ??= boxCollection._db.transaction(name, 'readwrite');
final store = txn.objectStore(name);
await store.put(val as Object, key);
_cache[key] = val;
_cachedKeys?.add(key);
return;
}
Future<void> delete(String key, [Transaction? txn]) async {
if (boxCollection._txnCache != null) {
boxCollection._txnCache!.add((txn) => delete(key, txn));
_cache.remove(key);
_cachedKeys?.remove(key);
return;
}
txn ??= boxCollection._db.transaction(name, 'readwrite');
final store = txn.objectStore(name);
await store.delete(key);
_cache.remove(key);
_cachedKeys?.remove(key);
return;
}
Future<void> deleteAll(List<String> keys, [Transaction? txn]) async {
if (boxCollection._txnCache != null) {
boxCollection._txnCache!.add((txn) => deleteAll(keys, txn));
keys.forEach(_cache.remove);
_cachedKeys?.removeAll(keys);
return;
}
txn ??= boxCollection._db.transaction(name, 'readwrite');
final store = txn.objectStore(name);
for (final key in keys) {
await store.delete(key);
_cache.remove(key);
_cachedKeys?.remove(key);
}
return;
}
Future<void> clear([Transaction? txn]) async {
if (boxCollection._txnCache != null) {
boxCollection._txnCache!.add((txn) => clear(txn));
_cache.clear();
_cachedKeys = null;
return;
}
txn ??= boxCollection._db.transaction(name, 'readwrite');
final store = txn.objectStore(name);
await store.clear();
_cache.clear();
_cachedKeys = null;
return;
}
V? _fromValue(Object? value) {
if (value == null) return null;
switch (V) {
case const (List<dynamic>):
return List.unmodifiable(value as List) as V;
case const (Map<dynamic, dynamic>):
return Map.unmodifiable(value as Map) as V;
case const (int):
case const (double):
case const (bool):
case const (String):
default:
return value as V;
}
}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,343 @@
import 'dart:async';
import 'dart:convert';
import 'package:sqflite_common/sqlite_api.dart';
/// Key-Value store abstraction over Sqflite so that the sdk database can use
/// a single interface for all platforms. API is inspired by Hive.
class BoxCollection {
final Database _db;
final Set<String> boxNames;
BoxCollection(this._db, this.boxNames);
static Future<BoxCollection> open(
String name,
Set<String> boxNames, {
Object? sqfliteDatabase,
dynamic idbFactory,
}) async {
if (sqfliteDatabase is! Database) {
throw ('You must provide a Database `sqfliteDatabase` for FluffyBox on native.');
}
final batch = sqfliteDatabase.batch();
for (final name in boxNames) {
batch.execute(
'CREATE TABLE IF NOT EXISTS $name (k TEXT PRIMARY KEY NOT NULL, v TEXT)',
);
batch.execute('CREATE INDEX IF NOT EXISTS k_index ON $name (k)');
}
await batch.commit(noResult: true);
return BoxCollection(sqfliteDatabase, boxNames);
}
Box<V> openBox<V>(String name) {
if (!boxNames.contains(name)) {
throw ('Box with name $name is not in the known box names of this collection.');
}
return Box<V>(name, this);
}
Batch? _activeBatch;
Completer<void>? _transactionLock;
final _transactionZones = <Zone>{};
Future<void> transaction(
Future<void> Function() action, {
List<String>? boxNames,
bool readOnly = false,
}) async {
// we want transactions to lock, however NOT if transactoins are run inside of each other.
// to be able to do this, we use dart zones (https://dart.dev/articles/archive/zones).
// _transactionZones holds a set of all zones which are currently running a transaction.
// _transactionLock holds the lock.
// first we try to determine if we are inside of a transaction currently
var isInTransaction = false;
Zone? zone = Zone.current;
// for that we keep on iterating to the parent zone until there is either no zone anymore
// or we have found a zone inside of _transactionZones.
while (zone != null) {
if (_transactionZones.contains(zone)) {
isInTransaction = true;
break;
}
zone = zone.parent;
}
// if we are inside a transaction....just run the action
if (isInTransaction) {
return await action();
}
// if we are *not* in a transaction, time to wait for the lock!
while (_transactionLock != null) {
await _transactionLock!.future;
}
// claim the lock
final lock = Completer<void>();
_transactionLock = lock;
try {
// run the action inside of a new zone
return await runZoned(() async {
try {
// don't forget to add the new zone to _transactionZones!
_transactionZones.add(Zone.current);
final batch = _db.batch();
_activeBatch = batch;
await action();
_activeBatch = null;
await batch.commit(noResult: true);
return;
} finally {
// aaaand remove the zone from _transactionZones again
_transactionZones.remove(Zone.current);
}
});
} finally {
// aaaand finally release the lock
_transactionLock = null;
lock.complete();
}
}
Future<void> clear() => transaction(
() async {
for (final name in boxNames) {
await _db.delete(name);
}
},
);
Future<void> close() => _db.close();
}
class Box<V> {
final String name;
final BoxCollection boxCollection;
final Map<String, V?> _cache = {};
/// _cachedKeys is only used to make sure that if you fetch all keys from a
/// box, you do not need to have an expensive read operation twice. There is
/// no other usage for this at the moment. So the cache is never partial.
/// Once the keys are cached, they need to be updated when changed in put and
/// delete* so that the cache does not become outdated.
Set<String>? _cachedKeys;
bool get _keysCached => _cachedKeys != null;
static const Set<Type> allowedValueTypes = {
List<dynamic>,
Map<dynamic, dynamic>,
String,
int,
double,
bool,
};
Box(this.name, this.boxCollection) {
if (!allowedValueTypes.any((type) => V == type)) {
throw Exception(
'Illegal value type for Box: "${V.toString()}". Must be one of $allowedValueTypes',
);
}
}
String? _toString(V? value) {
if (value == null) return null;
switch (V) {
case const (List<dynamic>):
case const (Map<dynamic, dynamic>):
return jsonEncode(value);
case const (String):
case const (int):
case const (double):
case const (bool):
default:
return value.toString();
}
}
V? _fromString(Object? value) {
if (value == null) return null;
if (value is! String) {
throw Exception(
'Wrong database type! Expected String but got one of type ${value.runtimeType}');
}
switch (V) {
case const (int):
return int.parse(value) as V;
case const (double):
return double.parse(value) as V;
case const (bool):
return (value == 'true') as V;
case const (List<dynamic>):
return List.unmodifiable(jsonDecode(value)) as V;
case const (Map<dynamic, dynamic>):
return Map.unmodifiable(jsonDecode(value)) as V;
case const (String):
default:
return value as V;
}
}
Future<List<String>> getAllKeys([Transaction? txn]) async {
if (_keysCached) return _cachedKeys!.toList();
final executor = txn ?? boxCollection._db;
final result = await executor.query(name, columns: ['k']);
final keys = result.map((row) => row['k'] as String).toList();
_cachedKeys = keys.toSet();
return keys;
}
Future<Map<String, V>> getAllValues([Transaction? txn]) async {
final executor = txn ?? boxCollection._db;
final result = await executor.query(name);
return Map.fromEntries(
result.map(
(row) => MapEntry(
row['k'] as String,
_fromString(row['v']) as V,
),
),
);
}
Future<V?> get(String key, [Transaction? txn]) async {
if (_cache.containsKey(key)) return _cache[key];
final executor = txn ?? boxCollection._db;
final result = await executor.query(
name,
columns: ['v'],
where: 'k = ?',
whereArgs: [key],
);
final value = result.isEmpty ? null : _fromString(result.single['v']);
_cache[key] = value;
return value;
}
Future<List<V?>> getAll(List<String> keys, [Transaction? txn]) async {
if (!keys.any((key) => !_cache.containsKey(key))) {
return keys.map((key) => _cache[key]).toList();
}
// The SQL operation might fail with more than 1000 keys. We define some
// buffer here and half the amount of keys recursively for this situation.
const getAllMax = 800;
if (keys.length > getAllMax) {
final half = keys.length ~/ 2;
return [
...(await getAll(keys.sublist(0, half))),
...(await getAll(keys.sublist(half))),
];
}
final executor = txn ?? boxCollection._db;
final list = <V?>[];
final result = await executor.query(
name,
where: 'k IN (${keys.map((_) => '?').join(',')})',
whereArgs: keys,
);
final resultMap = Map<String, V?>.fromEntries(
result.map((row) => MapEntry(row['k'] as String, _fromString(row['v']))),
);
// We want to make sure that they values are returnd in the exact same
// order than the given keys. That's why we do this instead of just return
// `resultMap.values`.
list.addAll(keys.map((key) => resultMap[key]));
_cache.addAll(resultMap);
return list;
}
Future<void> put(String key, V val) async {
final txn = boxCollection._activeBatch;
final params = {
'k': key,
'v': _toString(val),
};
if (txn == null) {
await boxCollection._db.insert(
name,
params,
conflictAlgorithm: ConflictAlgorithm.replace,
);
} else {
txn.insert(
name,
params,
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
_cache[key] = val;
_cachedKeys?.add(key);
return;
}
Future<void> delete(String key, [Batch? txn]) async {
txn ??= boxCollection._activeBatch;
if (txn == null) {
await boxCollection._db.delete(name, where: 'k = ?', whereArgs: [key]);
} else {
txn.delete(name, where: 'k = ?', whereArgs: [key]);
}
_cache.remove(key);
_cachedKeys?.remove(key);
return;
}
Future<void> deleteAll(List<String> keys, [Batch? txn]) async {
txn ??= boxCollection._activeBatch;
final placeholder = keys.map((_) => '?').join(',');
if (txn == null) {
await boxCollection._db.delete(
name,
where: 'k IN ($placeholder)',
whereArgs: keys,
);
} else {
txn.delete(
name,
where: 'k IN ($placeholder)',
whereArgs: keys,
);
}
for (final key in keys) {
_cache.remove(key);
_cachedKeys?.removeAll(keys);
}
return;
}
Future<void> clear([Batch? txn]) async {
txn ??= boxCollection._activeBatch;
if (txn == null) {
await boxCollection._db.delete(name);
} else {
txn.delete(name);
}
_cache.clear();
_cachedKeys = null;
return;
}
}

View File

@ -28,6 +28,7 @@ dependencies:
random_string: ^2.3.1
sdp_transform: ^0.3.2
slugify: ^2.0.0
sqflite_common: ^2.4.5
typed_data: ^1.3.2
webrtc_interface: ^1.0.13
@ -36,6 +37,7 @@ dev_dependencies:
file: ">=6.1.1 <8.0.0"
import_sorter: ^4.6.0
lints: ^3.0.0
sqflite_common_ffi: ^2.2.5
test: ^1.15.7
#flutter_test: {sdk: flutter}
#dependency_overrides:

80
test/box_test.dart Normal file
View File

@ -0,0 +1,80 @@
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
import 'package:test/test.dart';
import 'package:matrix/src/database/sqflite_box.dart';
void main() {
group('Box tests', () {
late BoxCollection collection;
const Set<String> boxNames = {'cats', 'dogs'};
const data = {'name': 'Fluffy', 'age': 2};
const data2 = {'name': 'Loki', 'age': 4};
setUp(() async {
final db = await databaseFactoryFfi.openDatabase(':memory:');
collection = await BoxCollection.open(
'testbox',
boxNames,
sqfliteDatabase: db,
);
});
test('Box.put and Box.get', () async {
final box = collection.openBox<Map>('cats');
await box.put('fluffy', data);
expect(await box.get('fluffy'), data);
await box.clear();
});
test('Box.getAll', () async {
final box = collection.openBox<Map>('cats');
await box.put('fluffy', data);
await box.put('loki', data2);
expect(await box.getAll(['fluffy', 'loki']), [data, data2]);
await box.clear();
});
test('Box.getAllKeys', () async {
final box = collection.openBox<Map>('cats');
await box.put('fluffy', data);
await box.put('loki', data2);
expect(await box.getAllKeys(), ['fluffy', 'loki']);
await box.clear();
});
test('Box.getAllValues', () async {
final box = collection.openBox<Map>('cats');
await box.put('fluffy', data);
await box.put('loki', data2);
expect(await box.getAllValues(), {'fluffy': data, 'loki': data2});
await box.clear();
});
test('Box.delete', () async {
final box = collection.openBox<Map>('cats');
await box.put('fluffy', data);
await box.put('loki', data2);
await box.delete('fluffy');
expect(await box.get('fluffy'), null);
await box.clear();
});
test('Box.deleteAll', () async {
final box = collection.openBox<Map>('cats');
await box.put('fluffy', data);
await box.put('loki', data2);
await box.deleteAll(['fluffy', 'loki']);
expect(await box.get('fluffy'), null);
expect(await box.get('loki'), null);
await box.clear();
});
test('Box.clear', () async {
final box = collection.openBox<Map>('cats');
await box.put('fluffy', data);
await box.put('loki', data2);
await box.clear();
expect(await box.get('fluffy'), null);
expect(await box.get('loki'), null);
});
});
}

View File

@ -27,11 +27,11 @@ import 'package:matrix/matrix.dart';
import 'fake_database.dart';
void main() {
group('HiveCollections Database Test', () {
group('Matrix SDK Database Test', () {
late DatabaseApi database;
late int toDeviceQueueIndex;
test('Open', () async {
database = await getHiveCollectionsDatabase(null);
test('Setup', () async {
database = await getMatrixSdkDatabase(null);
});
test('transaction', () async {
var counter = 0;

View File

@ -18,6 +18,7 @@
import 'package:file/local.dart';
import 'package:hive/hive.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
import 'package:matrix/matrix.dart';
@ -40,6 +41,14 @@ Future<HiveCollectionsDatabase> getHiveCollectionsDatabase(Client? c) async {
return db;
}
// ignore: deprecated_member_use_from_same_package
Future<MatrixSdkDatabase> getMatrixSdkDatabase(Client? c) async {
final database = await databaseFactoryFfi.openDatabase(':memory:');
final db = MatrixSdkDatabase('unit_test.${c?.hashCode}', database: database);
await db.open();
return db;
}
// ignore: deprecated_member_use_from_same_package
Future<FamedlySdkHiveDatabase> getHiveDatabase(Client? c) async {
if (!hiveInitialized) {