diff --git a/matrix_alertbot/webhook.py b/matrix_alertbot/webhook.py
index c43035e..a2efe6c 100644
--- a/matrix_alertbot/webhook.py
+++ b/matrix_alertbot/webhook.py
@@ -128,7 +128,6 @@ async def create_alert(
     if alert.firing:
         cache.set(event.event_id, alert.fingerprint, expire=config.cache_expire_time)
     else:
-        cache.delete(event.event_id)
         cache.delete(alert.fingerprint)
 
 
diff --git a/tests/test_webhook.py b/tests/test_webhook.py
index bee3d4b..c58d5ff 100644
--- a/tests/test_webhook.py
+++ b/tests/test_webhook.py
@@ -133,13 +133,9 @@ class WebhookApplicationTestCase(aiohttp.test_utils.AioHTTPTestCase):
             "fingerprint1",
             expire=self.fake_config.cache_expire_time,
         )
-        self.assertEqual(3, self.fake_cache.delete.call_count)
+        self.assertEqual(2, self.fake_cache.delete.call_count)
         self.fake_cache.delete.assert_has_calls(
-            [
-                call("fingerprint1"),
-                call(fake_send_text_to_room.return_value.event_id),
-                call("fingerprint2"),
-            ]
+            [call("fingerprint1"), call("fingerprint2")]
         )
 
     @patch.object(matrix_alertbot.webhook, "send_text_to_room")
@@ -187,10 +183,7 @@ class WebhookApplicationTestCase(aiohttp.test_utils.AioHTTPTestCase):
             "fingerprint1",
             expire=self.fake_config.cache_expire_time,
         )
-        self.assertEqual(2, self.fake_cache.delete.call_count)
-        self.fake_cache.delete.assert_has_calls(
-            [call(fake_send_text_to_room.return_value.event_id), call("fingerprint2")]
-        )
+        self.fake_cache.delete.assert_called_once_with("fingerprint2")
 
     @patch.object(matrix_alertbot.webhook, "send_text_to_room")
     async def test_post_alerts_with_alertmanager_error(
@@ -238,13 +231,7 @@ class WebhookApplicationTestCase(aiohttp.test_utils.AioHTTPTestCase):
             notice=False,
         )
         self.fake_cache.set.assert_not_called()
-        self.assertEqual(2, self.fake_cache.delete.call_count)
-        self.fake_cache.delete.assert_has_calls(
-            [
-                call(fake_send_text_to_room.return_value.event_id),
-                call("fingerprint2"),
-            ]
-        )
+        self.fake_cache.delete.assert_called_once_with("fingerprint2")
 
     @patch.object(matrix_alertbot.webhook, "send_text_to_room")
     async def test_post_alerts_in_unauthorized_room(