diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart
index 732553aa..63b4d681 100644
--- a/lib/encryption/olm_manager.dart
+++ b/lib/encryption/olm_manager.dart
@@ -262,19 +262,25 @@ class OlmManager {
     final haveFallbackKeys = encryption.isMinOlmVersion(3, 2, 0);
     // Check if there are at least half of max_number_of_one_time_keys left on the server
     // and generate and upload more if not.
-    if ((countJson != null &&
-            ((countJson.containsKey('signed_curve25519') &&
-                    countJson['signed_curve25519'] <
-                        (_olmAccount.max_number_of_one_time_keys() / 2)) ||
-                !countJson.containsKey('signed_curve25519'))) ||
-        (haveFallbackKeys &&
-            unusedFallbackKeyTypes?.contains('signed_curve25519') == false)) {
+
+    // If the server did not send us a count, assume it is 0
+    final keyCount = countJson?.tryGet<int>('signed_curve25519', 0) ?? 0;
+
+    // If the server does not support fallback keys, it will not tell us about them.
+    // If the server supports them but has no key, upload a new one.
+    var unusedFallbackKey = true;
+    if (unusedFallbackKeyTypes?.contains('signed_curve25519') == false) {
+      unusedFallbackKey = false;
+    }
+
+    // Only upload keys if they are less than half of the max or we have no unused fallback key
+    if (keyCount < (_olmAccount.max_number_of_one_time_keys() / 2) ||
+        !unusedFallbackKey) {
       uploadKeys(
-        oldKeyCount:
-            countJson != null ? (countJson['signed_curve25519'] ?? 0) : null,
-        unusedFallbackKey: haveFallbackKeys
-            ? unusedFallbackKeyTypes?.contains('signed_curve25519')
+        oldKeyCount: keyCount < (_olmAccount.max_number_of_one_time_keys() / 2)
+            ? keyCount
             : null,
+        unusedFallbackKey: haveFallbackKeys ? unusedFallbackKey : null,
       );
     }
   }
diff --git a/lib/src/client.dart b/lib/src/client.dart
index a063bf7e..3616a8ef 100644
--- a/lib/src/client.dart
+++ b/lib/src/client.dart
@@ -1184,9 +1184,7 @@ class Client extends MatrixApi {
     if (sync.deviceLists != null) {
       await _handleDeviceListsEvents(sync.deviceLists);
     }
-    if ((sync.deviceUnusedFallbackKeyTypes != null ||
-            sync.deviceOneTimeKeysCount != null) &&
-        encryptionEnabled) {
+    if (encryptionEnabled) {
       encryption.handleDeviceOneTimeKeysCount(
           sync.deviceOneTimeKeysCount, sync.deviceUnusedFallbackKeyTypes);
     }
diff --git a/test/client_test.dart b/test/client_test.dart
index b5fcaa0e..1c5ba715 100644
--- a/test/client_test.dart
+++ b/test/client_test.dart
@@ -614,8 +614,9 @@ void main() {
       expect(client2.deviceID, client1.deviceID);
       expect(client2.deviceName, client1.deviceName);
       if (client2.encryptionEnabled) {
-        expect(client2.encryption.pickledOlmAccount,
-            client1.encryption.pickledOlmAccount);
+        expect(client2.encryption.fingerprintKey,
+            client1.encryption.fingerprintKey);
+        expect(client2.encryption.identityKey, client1.encryption.identityKey);
         expect(client2.rooms[1].id, client1.rooms[1].id);
       }
 
diff --git a/test/encryption/olm_manager_test.dart b/test/encryption/olm_manager_test.dart
index 36ace7f2..9d4ad2a3 100644
--- a/test/encryption/olm_manager_test.dart
+++ b/test/encryption/olm_manager_test.dart
@@ -112,13 +112,14 @@ void main() {
           FakeMatrixApi.calledEndpoints.containsKey('/client/r0/keys/upload'),
           true);
 
+      // this will upload keys because we assume the key count is 0, if the server doesn't send one
       FakeMatrixApi.calledEndpoints.clear();
       client.encryption.olmManager
           .handleDeviceOneTimeKeysCount(null, ['signed_curve25519']);
       await Future.delayed(Duration(milliseconds: 50));
       expect(
           FakeMatrixApi.calledEndpoints.containsKey('/client/r0/keys/upload'),
-          false);
+          true);
     });
 
     test('restoreOlmSession', () async {