diff --git a/matrix_alertbot/callback.py b/matrix_alertbot/callback.py
index 967002e..5adc0e6 100644
--- a/matrix_alertbot/callback.py
+++ b/matrix_alertbot/callback.py
@@ -175,9 +175,7 @@ class Callbacks:
         # Get the original event that was reacted to
         event_response = await self.client.room_get_event(room.room_id, reacted_to_id)
         if isinstance(event_response, RoomGetEventError):
-            logger.warning(
-                "Error getting event that was reacted to (%s)", reacted_to_id
-            )
+            logger.warning(f"Error getting event that was reacted to ({reacted_to_id})")
             return
         reacted_to_event = event_response.event
 
@@ -213,6 +211,10 @@ class Callbacks:
         if event.sender == self.config.user_id:
             return
 
+        if event.redacts not in self.cache:
+            logger.warning(f"Error removing silence from reaction {event.redacts}")
+            return
+
         reacted_to_id = self.cache[event.redacts]
         reacted_to_event = await self.client.room_get_event(room.room_id, reacted_to_id)
         if isinstance(reacted_to_event, RoomGetEventError):
diff --git a/matrix_alertbot/command.py b/matrix_alertbot/command.py
index e75a34b..67e77b3 100644
--- a/matrix_alertbot/command.py
+++ b/matrix_alertbot/command.py
@@ -7,7 +7,7 @@ from nio import AsyncClient, MatrixRoom
 from matrix_alertbot.alertmanager import AlertmanagerClient
 from matrix_alertbot.chat_functions import send_text_to_room
 from matrix_alertbot.config import Config
-from matrix_alertbot.errors import AlertmanagerError
+from matrix_alertbot.errors import AlertmanagerError, AlertNotFoundError
 from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher
 
 logger = logging.getLogger(__name__)
@@ -88,8 +88,15 @@ class Command:
 
         logger.debug(f"Read alert fingerprints for event {self.event_id} from cache")
 
-        count_created_silences = 0
+        if self.event_id not in self.cache:
+            raise AlertNotFoundError(
+                f"Cannot find fingerprints for event {self.event_id} in cache"
+            )
+
         alert_fingerprints = self.cache[self.event_id]
+        logger.debug(f"Found {len(alert_fingerprints)} in cache")
+
+        count_created_silences = 0
         for alert_fingerprint in alert_fingerprints:
             logger.debug(
                 f"Create silence for alert with fingerprint {alert_fingerprint} for a duration of {duration}"
@@ -127,8 +134,15 @@ class Command:
         logger.debug("Receiving a command to delete a silence")
         logger.debug(f"Read alert fingerprints for event {self.event_id} from cache")
 
-        count_removed_silences = 0
+        if self.event_id not in self.cache:
+            raise AlertNotFoundError(
+                f"Cannot find fingerprints for event {self.event_id} in cache"
+            )
+
         alert_fingerprints = self.cache[self.event_id]
+        logger.debug(f"Found {len(alert_fingerprints)} in cache")
+
+        count_removed_silences = 0
         for alert_fingerprint in alert_fingerprints:
             logger.debug(
                 f"Delete silence for alert with fingerprint {alert_fingerprint}"
diff --git a/tests/test_command.py b/tests/test_command.py
index be0771e..6dc850e 100644
--- a/tests/test_command.py
+++ b/tests/test_command.py
@@ -42,7 +42,8 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase):
         self.fake_silences = ["silence1", "silence2"]
 
         self.fake_cache = MagicMock(spec=Cache)
-        self.fake_cache.__getitem__ = Mock(return_value=self.fake_fingerprints)
+        self.fake_cache.__getitem__.return_value = self.fake_fingerprints
+        self.fake_cache.__contains__.return_value = True
 
         self.fake_alertmanager = Mock(spec=AlertmanagerClient)
         self.fake_alertmanager.delete_silences.return_value = self.fake_silences