diff --git a/src/KeyRequestHandler.js b/src/KeyRequestHandler.js
index fbb869d468..4354fba269 100644
--- a/src/KeyRequestHandler.js
+++ b/src/KeyRequestHandler.js
@@ -88,6 +88,12 @@ export default class KeyRequestHandler {
         }
         console.log("Forgetting room key request");
         requests.splice(idx, 1);
+        if (requests.length === 0) {
+            delete this._pendingKeyRequests[userId][deviceId];
+            if (Object.keys(this._pendingKeyRequests[userId]).length === 0) {
+                delete this._pendingKeyRequests[userId];
+            }
+        }
     }
 
     _processNextRequest() {