diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart
index bd3c7036..2127e320 100644
--- a/lib/encryption/key_manager.dart
+++ b/lib/encryption/key_manager.dart
@@ -159,6 +159,8 @@ class KeyManager {
       if (uploaded) {
         client.database
             .markInboundGroupSessionAsUploaded(client.id, roomId, sessionId);
+      } else {
+        _haveKeysToUpload = true;
       }
     });
     final room = client.getRoomById(roomId);
@@ -680,18 +682,20 @@ class KeyManager {
   }
 
   bool _isUploadingKeys = false;
+  bool _haveKeysToUpload = true;
   Future<void> backgroundTasks() async {
     if (_isUploadingKeys || client.database == null) {
       return;
     }
     _isUploadingKeys = true;
     try {
-      if (!(await isCached())) {
+      if (!_haveKeysToUpload || !(await isCached())) {
         return; // we can't backup anyways
       }
       final dbSessions =
           await client.database.getInboundGroupSessionsToUpload().get();
       if (dbSessions.isEmpty) {
+        _haveKeysToUpload = false;
         return; // nothing to do
       }
       final privateKey =
diff --git a/lib/src/client.dart b/lib/src/client.dart
index ccb559a1..bc02e1f6 100644
--- a/lib/src/client.dart
+++ b/lib/src/client.dart
@@ -1698,14 +1698,20 @@ sort order of ${prevState.sortOrder}. This should never happen...''');
     }
   }
 
+  bool _toDeviceQueueNeedsProcessing = true;
+
   /// Processes the to_device queue and tries to send every entry.
   /// This function MAY throw an error, which just means the to_device queue wasn't
   /// proccessed all the way.
   Future<void> processToDeviceQueue() async {
-    if (database == null) {
+    if (database == null || !_toDeviceQueueNeedsProcessing) {
       return;
     }
     final entries = await database.getToDeviceQueue(id).get();
+    if (entries.isEmpty) {
+      _toDeviceQueueNeedsProcessing = false;
+      return;
+    }
     for (final entry in entries) {
       // ohgod what is this...
       final data = (json.decode(entry.content) as Map).map((k, v) =>
@@ -1736,6 +1742,7 @@ sort order of ${prevState.sortOrder}. This should never happen...''');
           e,
           s);
       if (database != null) {
+        _toDeviceQueueNeedsProcessing = true;
         await database.insertIntoToDeviceQueue(
             id, eventType, txnId, json.encode(messages));
       }
diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart
index 21cde8a0..0d01d0b7 100644
--- a/lib/src/database/database.dart
+++ b/lib/src/database/database.dart
@@ -282,7 +282,8 @@ class Database extends _$Database {
       );
       roomList.add(room);
       // let's see if we need any m.room.member events
-      final membersToPostload = <String>{};
+      // We always need the member event for ourself
+      final membersToPostload = <String>{client.userID};
       // the lastEvent message preview might have an author we need to fetch, if it is a group chat
       if (room.getState(api.EventTypes.Message) != null && !room.isDirectChat) {
         membersToPostload.add(room.getState(api.EventTypes.Message).senderId);
@@ -296,11 +297,8 @@ class Database extends _$Database {
         // post-load the heroes
         membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty));
       }
-      // okay, only load from the database if we actually have stuff to load
-      if (membersToPostload.isNotEmpty) {
-        // save it for loading later
-        allMembersToPostload[room.id] = membersToPostload;
-      }
+      // save it for loading later
+      allMembersToPostload[room.id] = membersToPostload;
     }
     // now we postload all members, if thre are any
     if (allMembersToPostload.isNotEmpty) {