From bbdf648cd9989e1478db155d606f450beb0de4ad Mon Sep 17 00:00:00 2001 From: HgO Date: Sun, 10 Jul 2022 12:51:49 +0200 Subject: [PATCH] delete silence with specific matchers --- matrix_alertbot/alert.py | 3 +- matrix_alertbot/alertmanager.py | 12 ++-- matrix_alertbot/command.py | 21 ++++--- matrix_alertbot/matcher.py | 38 ++++++------ matrix_alertbot/webhook.py | 1 - tests/test_alertmanager.py | 104 ++++++++++++++++++++++++++------ tests/test_command.py | 57 ++++++++++++++--- 7 files changed, 173 insertions(+), 63 deletions(-) diff --git a/matrix_alertbot/alert.py b/matrix_alertbot/alert.py index 06932a0..5d955c5 100644 --- a/matrix_alertbot/alert.py +++ b/matrix_alertbot/alert.py @@ -1,8 +1,7 @@ from __future__ import annotations import logging -import re -from typing import Any, Dict +from typing import Dict logger = logging.getLogger(__name__) diff --git a/matrix_alertbot/alertmanager.py b/matrix_alertbot/alertmanager.py index 778ad6d..5bf53f8 100644 --- a/matrix_alertbot/alertmanager.py +++ b/matrix_alertbot/alertmanager.py @@ -15,7 +15,7 @@ from matrix_alertbot.errors import ( AlertNotFoundError, SilenceNotFoundError, ) -from matrix_alertbot.matcher import AbstractAlertMatcher +from matrix_alertbot.matcher import AlertMatcher class AlertmanagerClient: @@ -46,7 +46,7 @@ class AlertmanagerClient: fingerprint: str, duration: str, user: str, - matchers: List[AbstractAlertMatcher], + matchers: List[AlertMatcher], ) -> str: alert = await self.get_alert(fingerprint) @@ -82,7 +82,9 @@ class AlertmanagerClient: return data["silenceID"] - async def delete_silences(self, fingerprint: str) -> List[str]: + async def delete_silences( + self, fingerprint: str, matchers: List[AlertMatcher] + ) -> List[str]: alert = await self.get_alert(fingerprint) alert_state = alert["status"]["state"] @@ -91,6 +93,8 @@ class AlertmanagerClient: f"Cannot find silences for alert fingerprint {fingerprint} in state {alert_state}" ) + self._match_alert(alert, matchers) + silences = alert["status"]["silencedBy"] for silence in silences: await self._delete_silence(silence) @@ -115,7 +119,7 @@ class AlertmanagerClient: raise AlertNotFoundError(f"Cannot find alert with fingerprint {fingerprint}") @staticmethod - def _match_alert(alert: Dict, matchers: List[AbstractAlertMatcher]) -> None: + def _match_alert(alert: Dict, matchers: List[AlertMatcher]) -> None: labels = alert["labels"] for matcher in matchers: if matcher.label not in labels: diff --git a/matrix_alertbot/command.py b/matrix_alertbot/command.py index 7d95994..6a0d246 100644 --- a/matrix_alertbot/command.py +++ b/matrix_alertbot/command.py @@ -8,11 +8,7 @@ from matrix_alertbot.alertmanager import AlertmanagerClient from matrix_alertbot.chat_functions import react_to_event, send_text_to_room from matrix_alertbot.config import Config from matrix_alertbot.errors import AlertmanagerError -from matrix_alertbot.matcher import ( - AbstractAlertMatcher, - AlertMatcher, - AlertRegexMatcher, -) +from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher logger = logging.getLogger(__name__) @@ -67,7 +63,7 @@ class Command: async def _ack(self) -> None: """Acknowledge an alert and silence it for a certain duration in Alertmanager""" - matchers: List[AbstractAlertMatcher] = [] + matchers: List[AlertMatcher] = [] durations = [] for arg in self.args: if "=~" in arg: @@ -124,6 +120,17 @@ class Command: async def _unack(self) -> None: """Delete an alert's acknowledgement of an alert and remove corresponding silence in Alertmanager""" + matchers: List[AlertMatcher] = [] + for arg in self.args: + if "=~" in arg: + label, regex = arg.split("=~") + regex_matcher = AlertRegexMatcher(label, regex) + matchers.append(regex_matcher) + elif "=" in arg: + label, value = arg.split("=") + matcher = AlertMatcher(label, value) + matchers.append(matcher) + logger.debug( f"Receiving a command to delete a silence | " f"{self.room.user_name(self.event.sender)}: {self.event.body}" @@ -145,7 +152,7 @@ class Command: ) try: removed_silences = await self.alertmanager.delete_silences( - alert_fingerprint + alert_fingerprint, matchers ) count_removed_silences += len(removed_silences) except AlertmanagerError as e: diff --git a/matrix_alertbot/matcher.py b/matrix_alertbot/matcher.py index eac0f94..e386c02 100644 --- a/matrix_alertbot/matcher.py +++ b/matrix_alertbot/matcher.py @@ -2,34 +2,34 @@ import re from typing import Any, Dict -class AbstractAlertMatcher: - def __init__(self, label: str, value: str, op: str) -> None: +class AlertMatcher: + def __init__(self, label: str, value: str) -> None: self.label = label self.value = value - self._op = op - - def match(self, labels: Dict[str, str]) -> bool: - raise NotImplementedError - - def __str__(self) -> str: - return f"{self.label}{self._op}{self.value}" - - def __eq__(self, matcher: Any) -> bool: - return self.label == matcher.label and self.value == matcher.value - - -class AlertMatcher(AbstractAlertMatcher): - def __init__(self, label: str, value: str) -> None: - super().__init__(label, value, "=") def match(self, labels: Dict[str, str]) -> bool: return self.label in labels and self.value == labels[self.label] + def __str__(self) -> str: + return f"{self.label}={self.value}" -class AlertRegexMatcher(AbstractAlertMatcher): + def __repr__(self) -> str: + return f"AlertMatcher({self})" + + def __eq__(self, matcher: Any) -> bool: + return str(self) == str(matcher) + + +class AlertRegexMatcher(AlertMatcher): def __init__(self, label: str, regex: str) -> None: - super().__init__(label, regex, "=~") + super().__init__(label, regex) self.regex = re.compile(regex) + def __str__(self) -> str: + return f"{self.label}=~{self.value}" + + def __repr__(self) -> str: + return f"AlertRegexMatcher({self})" + def match(self, labels: Dict[str, str]) -> bool: return self.label in labels and self.regex.match(labels[self.label]) is not None diff --git a/matrix_alertbot/webhook.py b/matrix_alertbot/webhook.py index b9e2a38..9abedec 100644 --- a/matrix_alertbot/webhook.py +++ b/matrix_alertbot/webhook.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from typing import Any from aiohttp import web, web_request from aiohttp_prometheus_exporter.handler import metrics diff --git a/tests/test_alertmanager.py b/tests/test_alertmanager.py index 920174f..6158dab 100644 --- a/tests/test_alertmanager.py +++ b/tests/test_alertmanager.py @@ -19,11 +19,7 @@ from matrix_alertbot.errors import ( AlertNotFoundError, SilenceNotFoundError, ) -from matrix_alertbot.matcher import ( - AbstractAlertMatcher, - AlertMatcher, - AlertRegexMatcher, -) +from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher class FakeTimeDelta: @@ -253,9 +249,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): @patch("matrix_alertbot.alertmanager.timedelta", side_effect=FakeTimeDelta) async def test_create_silence_with_matchers(self, fake_timedelta: Mock) -> None: - matchers: List[AbstractAlertMatcher] = [ - AlertMatcher(label="alertname", value="alert1") - ] + matchers = [AlertMatcher(label="alertname", value="alert1")] async with FakeAlertmanagerServer() as fake_alertmanager_server: port = fake_alertmanager_server.port @@ -277,7 +271,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): async def test_create_silence_with_regex_matchers( self, fake_timedelta: Mock ) -> None: - matchers: List[AbstractAlertMatcher] = [ + matchers: List[AlertMatcher] = [ AlertRegexMatcher(label="alertname", regex=r"alert\d+") ] @@ -298,7 +292,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): fake_timedelta.assert_called_once_with(seconds=86400) async def test_create_silence_raise_missing_label(self) -> None: - matchers: List[AbstractAlertMatcher] = [ + matchers = [ AlertMatcher(label="alertname", value="alert1"), AlertMatcher(label="severity", value="critical"), ] @@ -318,9 +312,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): ) async def test_create_silence_raise_mismatch_label(self) -> None: - matchers: List[AbstractAlertMatcher] = [ - AlertMatcher(label="alertname", value="alert2") - ] + matchers = [AlertMatcher(label="alertname", value="alert2")] async with FakeAlertmanagerServer() as fake_alertmanager_server: port = fake_alertmanager_server.port @@ -337,7 +329,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): ) async def test_create_silence_raise_mismatch_regex_label(self) -> None: - matchers: List[AbstractAlertMatcher] = [ + matchers: List[AlertMatcher] = [ AlertRegexMatcher(label="alertname", regex=r"alert[^\d]+") ] @@ -377,15 +369,87 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): with self.assertRaises(AlertmanagerServerError): await alertmanager.create_silence("fingerprint1", "1d", "user", []) - async def test_delete_silences_happy(self) -> None: + async def test_delete_silences_without_matchers(self) -> None: async with FakeAlertmanagerServer() as fake_alertmanager_server: port = fake_alertmanager_server.port alertmanager = AlertmanagerClient( f"http://localhost:{port}", self.fake_cache ) async with aiotools.closing_async(alertmanager): - silences = await alertmanager.delete_silences("fingerprint2") - self.assertEqual(["silence1", "silence2"], silences) + silences = await alertmanager.delete_silences("fingerprint2", []) + + self.assertEqual(["silence1", "silence2"], silences) + + async def test_delete_silences_with_matchers(self) -> None: + matchers = [AlertMatcher(label="alertname", value="alert2")] + + async with FakeAlertmanagerServer() as fake_alertmanager_server: + port = fake_alertmanager_server.port + alertmanager = AlertmanagerClient( + f"http://localhost:{port}", self.fake_cache + ) + async with aiotools.closing_async(alertmanager): + silences = await alertmanager.delete_silences("fingerprint2", matchers) + + self.assertEqual(["silence1", "silence2"], silences) + + async def test_delete_silences_with_regex_matchers(self) -> None: + matchers: List[AlertMatcher] = [ + AlertRegexMatcher(label="alertname", regex=r"alert\d+") + ] + + async with FakeAlertmanagerServer() as fake_alertmanager_server: + port = fake_alertmanager_server.port + alertmanager = AlertmanagerClient( + f"http://localhost:{port}", self.fake_cache + ) + async with aiotools.closing_async(alertmanager): + silences = await alertmanager.delete_silences("fingerprint2", matchers) + + self.assertEqual(["silence1", "silence2"], silences) + + async def test_delete_silences_raise_missing_label(self) -> None: + matchers = [ + AlertMatcher(label="alertname", value="alert2"), + AlertMatcher(label="severity", value="critical"), + ] + + async with FakeAlertmanagerServer() as fake_alertmanager_server: + port = fake_alertmanager_server.port + alertmanager = AlertmanagerClient( + f"http://localhost:{port}", self.fake_cache + ) + async with aiotools.closing_async(alertmanager): + with self.assertRaises(AlertMismatchError): + await alertmanager.delete_silences("fingerprint2", matchers) + + async def test_delete_silences_raise_mismatch_label(self) -> None: + matchers = [ + AlertMatcher(label="alertname", value="alert1"), + ] + + async with FakeAlertmanagerServer() as fake_alertmanager_server: + port = fake_alertmanager_server.port + alertmanager = AlertmanagerClient( + f"http://localhost:{port}", self.fake_cache + ) + async with aiotools.closing_async(alertmanager): + with self.assertRaises(AlertMismatchError): + await alertmanager.delete_silences("fingerprint2", matchers) + + async def test_delete_silences_raise_mismatch_regex_label(self) -> None: + matchers: List[AlertMatcher] = [ + AlertRegexMatcher(label="alertname", regex=r"alert[^\d]+"), + ] + + async with FakeAlertmanagerServer() as fake_alertmanager_server: + port = fake_alertmanager_server.port + alertmanager = AlertmanagerClient( + f"http://localhost:{port}", self.fake_cache + ) + async with aiotools.closing_async(alertmanager): + with self.assertRaises(AlertMismatchError): + await alertmanager.delete_silences("fingerprint2", matchers) async def test_delete_silences_raise_silence_not_found(self) -> None: async with FakeAlertmanagerServer() as fake_alertmanager_server: @@ -395,7 +459,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): ) async with aiotools.closing_async(alertmanager): with self.assertRaises(SilenceNotFoundError): - await alertmanager.delete_silences("fingerprint1") + await alertmanager.delete_silences("fingerprint1", []) async def test_delete_silences_raise_alert_not_found(self) -> None: async with FakeAlertmanagerServerWithoutAlert() as fake_alertmanager_server: @@ -405,7 +469,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): ) async with aiotools.closing_async(alertmanager): with self.assertRaises(AlertNotFoundError): - await alertmanager.delete_silences("fingerprint2") + await alertmanager.delete_silences("fingerprint2", []) async def test_delete_silences_raise_alertmanager_error(self) -> None: async with FakeAlertmanagerServerWithErrorDeleteSilence() as fake_alertmanager_server: @@ -417,7 +481,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): await alertmanager.get_alert("fingerprint1") with self.assertRaises(AlertmanagerServerError): - await alertmanager.delete_silences("fingerprint2") + await alertmanager.delete_silences("fingerprint2", []) async def test_find_alert_happy(self) -> None: alertmanager = AlertmanagerClient("http://localhost", self.fake_cache) diff --git a/tests/test_command.py b/tests/test_command.py index 69ada89..3f3b8bf 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -9,20 +9,22 @@ import matrix_alertbot.callback from matrix_alertbot.alertmanager import AlertmanagerClient from matrix_alertbot.command import Command from matrix_alertbot.errors import AlertmanagerError -from matrix_alertbot.matcher import AbstractAlertMatcher, AlertMatcher +from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher from tests.utils import make_awaitable async def create_silence_raise_alertmanager_error( - fingerprint: str, duration: str, user: str, matchers: List[AbstractAlertMatcher] + fingerprint: str, duration: str, user: str, matchers: List[AlertMatcher] ) -> str: if fingerprint == "fingerprint1": raise AlertmanagerError return "silence1" -async def delete_silence_raise_alertmanager_error(fingerprint: str) -> List[str]: +async def delete_silence_raise_alertmanager_error( + fingerprint: str, matchers: List[AlertMatcher] +) -> List[str]: if fingerprint == "fingerprint1": raise AlertmanagerError return ["silence1"] @@ -225,9 +227,9 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) -> None: """Tests the callback for InviteMemberEvents""" # Tests that the bot attempts to join a room after being invited to it - matchers: List[AbstractAlertMatcher] = [ + matchers = [ AlertMatcher(label="alertname", value="alert1"), - AlertMatcher(label="severity", value="critical"), + AlertRegexMatcher(label="severity", regex="critical"), ] self.fake_message_event.source = self.fake_source_in_reply @@ -237,7 +239,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): self.fake_cache, self.fake_alertmanager, self.fake_config, - "ack alertname=alert1 severity=critical", + "ack alertname=alert1 severity=~critical", self.fake_room, self.fake_message_event, ) @@ -300,7 +302,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) -> None: """Tests the callback for InviteMemberEvents""" # Tests that the bot attempts to join a room after being invited to it - matchers: List[AbstractAlertMatcher] = [ + matchers = [ AlertMatcher(label="alertname", value="alert1"), AlertMatcher(label="severity", value="critical"), ] @@ -374,7 +376,9 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) @patch.object(matrix_alertbot.command, "send_text_to_room") - async def test_unack_in_reply(self, fake_send_text_to_room: Mock) -> None: + async def test_unack_in_reply_without_matchers( + self, fake_send_text_to_room: Mock + ) -> None: """Tests the callback for InviteMemberEvents""" # Tests that the bot attempts to join a room after being invited to it @@ -393,7 +397,40 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): # Check that we attempted to create silences self.fake_alertmanager.delete_silences.assert_has_calls( - [call(fingerprint) for fingerprint in self.fake_fingerprints] + [call(fingerprint, []) for fingerprint in self.fake_fingerprints] + ) + fake_send_text_to_room.assert_called_with( + self.fake_client, self.fake_room.room_id, "Removed 4 silences." + ) + + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_unack_in_reply_with_matchers( + self, fake_send_text_to_room: Mock + ) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + + matchers = [ + AlertMatcher(label="alertname", value="alert1"), + AlertRegexMatcher(label="severity", regex="critical"), + ] + + self.fake_message_event.source = self.fake_source_in_reply + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "unack alertname=alert1 severity=~critical", + self.fake_room, + self.fake_message_event, + ) + await command._unack() + + # Check that we attempted to create silences + self.fake_alertmanager.delete_silences.assert_has_calls( + [call(fingerprint, matchers) for fingerprint in self.fake_fingerprints] ) fake_send_text_to_room.assert_called_with( self.fake_client, self.fake_room.room_id, "Removed 4 silences." @@ -425,7 +462,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): # Check that we attempted to create silences self.fake_alertmanager.delete_silences.assert_has_calls( - [call(fingerprint) for fingerprint in self.fake_fingerprints] + [call(fingerprint, []) for fingerprint in self.fake_fingerprints] ) fake_send_text_to_room.assert_called_with( self.fake_client, self.fake_room.room_id, "Removed 1 silences."