diff --git a/lib/src/models/retry_event_model.dart b/lib/src/models/retry_event_model.dart
deleted file mode 100644
index dcda3ec6..00000000
--- a/lib/src/models/retry_event_model.dart
+++ /dev/null
@@ -1,19 +0,0 @@
-import 'dart:async';
-
-/// Retries the `retryFunction` after a set `timeInterval` until `dispose` is called
-class RetryEventModel {
-  final Duration timeInterval;
-  void Function(Timer? timer) retryFunction;
-
-  final Timer? _timer;
-
-  RetryEventModel({
-    required this.timeInterval,
-    required this.retryFunction,
-  }) : _timer = Timer.periodic(timeInterval, retryFunction) {
-    // run it once because timer.periodic waits for duration before first run
-    retryFunction(null);
-  }
-
-  void dispose() => _timer?.cancel();
-}
diff --git a/lib/src/voip/backend/livekit_backend.dart b/lib/src/voip/backend/livekit_backend.dart
index acc91dd5..13ad6baf 100644
--- a/lib/src/voip/backend/livekit_backend.dart
+++ b/lib/src/voip/backend/livekit_backend.dart
@@ -3,7 +3,6 @@ import 'dart:convert';
 import 'dart:typed_data';
 
 import 'package:matrix/matrix.dart';
-import 'package:matrix/src/models/retry_event_model.dart';
 import 'package:matrix/src/utils/crypto/crypto.dart';
 import 'package:matrix/src/voip/models/call_membership.dart';
 
@@ -303,16 +302,11 @@ class LiveKitBackend extends CallBackend {
     };
   }
 
-  Map<CallParticipant, RetryEventModel> requestEncrytionKeyPending = {};
-
   @override
   Future<void> requestEncrytionKey(
     GroupCallSession groupCall,
     List<CallParticipant> remoteParticipants,
   ) async {
-    Logs().v(
-        '[VOIP E2EE] requesting stream encryption keys from ${remoteParticipants.map((e) => e.id)}');
-
     final Map<String, Object> data = {
       'conf_id': groupCall.groupCallId,
       'device_id': groupCall.client.deviceID!,
@@ -325,14 +319,6 @@ class LiveKitBackend extends CallBackend {
       data,
       EventTypes.GroupCallMemberEncryptionKeysRequest,
     );
-
-    for (final rp in remoteParticipants) {
-      requestEncrytionKeyPending.remove(rp)?.dispose();
-      requestEncrytionKeyPending[rp] = RetryEventModel(
-        timeInterval: Duration(seconds: 2),
-        retryFunction: (_) => requestEncrytionKey(groupCall, [rp]),
-      );
-    }
   }
 
   @override
@@ -350,34 +336,21 @@ class LiveKitBackend extends CallBackend {
 
     final callId = keyContent.callId;
 
-    final p =
-        CallParticipant(groupCall.voip, userId: userId, deviceId: deviceId);
-
     if (keyContent.keys.isEmpty) {
       Logs().w(
           '[VOIP E2EE] Received m.call.encryption_keys where keys is empty: callId=$callId');
       return;
     } else {
       Logs().i(
-          '[VOIP E2EE]: onCallEncryption, got keys from ${p.id} ${keyContent.toJson()}');
+          '[VOIP E2EE]: onCallEncryption, got keys from $userId:$deviceId ${keyContent.toJson()}');
     }
 
-    // TODO (td): this could potentially cancel a retry request for a different keyId
-    // something like
-    // 1. undecryptable frame
-    // 2. request keys
-    // 3. stream change
-    // 4. request keys
-    // 5. answer for 2.
-    // 6. cancel request for 4.
-    requestEncrytionKeyPending.remove(p)?.dispose();
-
     for (final key in keyContent.keys) {
       final encryptionKey = key.key;
       final encryptionKeyIndex = key.index;
       await _setEncryptionKey(
         groupCall,
-        p,
+        CallParticipant(groupCall.voip, userId: userId, deviceId: deviceId),
         encryptionKeyIndex,
         // base64Decode here because we receive base64Encoded version
         base64Decode(encryptionKey),