tobast/matrix-alertbot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

delete silence with specific matchers

Browse source
This commit is contained in:
HgO 2022-07-10 12:51:49 +02:00
parent af0b9c31ca
commit bbdf648cd9
7 changed files with 173 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
matrix_alertbot/alert.py
View file

@ -1,8 +1,7 @@
from __future__ import annotations
import logging
import re
from typing import Any, Dict
from typing import Dict
logger = logging.getLogger(__name__)

12
matrix_alertbot/alertmanager.py
View file

@ -15,7 +15,7 @@ from matrix_alertbot.errors import (
AlertNotFoundError,
SilenceNotFoundError,
)
from matrix_alertbot.matcher import AbstractAlertMatcher
from matrix_alertbot.matcher import AlertMatcher
class AlertmanagerClient:
@ -46,7 +46,7 @@ class AlertmanagerClient:
fingerprint: str,
duration: str,
user: str,
matchers: List[AbstractAlertMatcher],
matchers: List[AlertMatcher],
) -> str:
alert = await self.get_alert(fingerprint)
@ -82,7 +82,9 @@ class AlertmanagerClient:
return data["silenceID"]
async def delete_silences(self, fingerprint: str) -> List[str]:
async def delete_silences(
self, fingerprint: str, matchers: List[AlertMatcher]
) -> List[str]:
alert = await self.get_alert(fingerprint)
alert_state = alert["status"]["state"]
@ -91,6 +93,8 @@ class AlertmanagerClient:
f"Cannot find silences for alert fingerprint {fingerprint} in state {alert_state}"
)
self._match_alert(alert, matchers)
silences = alert["status"]["silencedBy"]
for silence in silences:
await self._delete_silence(silence)
@ -115,7 +119,7 @@ class AlertmanagerClient:
raise AlertNotFoundError(f"Cannot find alert with fingerprint {fingerprint}")
@staticmethod
def _match_alert(alert: Dict, matchers: List[AbstractAlertMatcher]) -> None:
def _match_alert(alert: Dict, matchers: List[AlertMatcher]) -> None:
labels = alert["labels"]
for matcher in matchers:
if matcher.label not in labels:

21
matrix_alertbot/command.py
View file

@ -8,11 +8,7 @@ from matrix_alertbot.alertmanager import AlertmanagerClient
from matrix_alertbot.chat_functions import react_to_event, send_text_to_room
from matrix_alertbot.config import Config
from matrix_alertbot.errors import AlertmanagerError
from matrix_alertbot.matcher import (
AbstractAlertMatcher,
AlertMatcher,
AlertRegexMatcher,
)
from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher
logger = logging.getLogger(__name__)
@ -67,7 +63,7 @@ class Command:
async def _ack(self) -> None:
"""Acknowledge an alert and silence it for a certain duration in Alertmanager"""
matchers: List[AbstractAlertMatcher] = []
matchers: List[AlertMatcher] = []
durations = []
for arg in self.args:
if "=~" in arg:
@ -124,6 +120,17 @@ class Command:
async def _unack(self) -> None:
"""Delete an alert's acknowledgement of an alert and remove corresponding silence in Alertmanager"""
matchers: List[AlertMatcher] = []
for arg in self.args:
if "=~" in arg:
label, regex = arg.split("=~")
regex_matcher = AlertRegexMatcher(label, regex)
matchers.append(regex_matcher)
elif "=" in arg:
label, value = arg.split("=")
matcher = AlertMatcher(label, value)
matchers.append(matcher)
logger.debug(
f"Receiving a command to delete a silence | "
f"{self.room.user_name(self.event.sender)}: {self.event.body}"
@ -145,7 +152,7 @@ class Command:
)
try:
removed_silences = await self.alertmanager.delete_silences(
alert_fingerprint
alert_fingerprint, matchers
)
count_removed_silences += len(removed_silences)
except AlertmanagerError as e:

38
matrix_alertbot/matcher.py
View file

@ -2,34 +2,34 @@ import re
from typing import Any, Dict
class AbstractAlertMatcher:
def __init__(self, label: str, value: str, op: str) -> None:
class AlertMatcher:
def __init__(self, label: str, value: str) -> None:
self.label = label
self.value = value
self._op = op
def match(self, labels: Dict[str, str]) -> bool:
raise NotImplementedError
def __str__(self) -> str:
return f"{self.label}{self._op}{self.value}"
def __eq__(self, matcher: Any) -> bool:
return self.label == matcher.label and self.value == matcher.value
class AlertMatcher(AbstractAlertMatcher):
def __init__(self, label: str, value: str) -> None:
super().__init__(label, value, "=")
def match(self, labels: Dict[str, str]) -> bool:
return self.label in labels and self.value == labels[self.label]
def __str__(self) -> str:
return f"{self.label}={self.value}"
class AlertRegexMatcher(AbstractAlertMatcher):
def __repr__(self) -> str:
return f"AlertMatcher({self})"
def __eq__(self, matcher: Any) -> bool:
return str(self) == str(matcher)
class AlertRegexMatcher(AlertMatcher):
def __init__(self, label: str, regex: str) -> None:
super().__init__(label, regex, "=~")
super().__init__(label, regex)
self.regex = re.compile(regex)
def __str__(self) -> str:
return f"{self.label}=~{self.value}"
def __repr__(self) -> str:
return f"AlertRegexMatcher({self})"
def match(self, labels: Dict[str, str]) -> bool:
return self.label in labels and self.regex.match(labels[self.label]) is not None

1
matrix_alertbot/webhook.py
View file

@ -1,7 +1,6 @@
from __future__ import annotations
import logging
from typing import Any
from aiohttp import web, web_request
from aiohttp_prometheus_exporter.handler import metrics

104
tests/test_alertmanager.py
View file

@ -19,11 +19,7 @@ from matrix_alertbot.errors import (
AlertNotFoundError,
SilenceNotFoundError,
)
from matrix_alertbot.matcher import (
AbstractAlertMatcher,
AlertMatcher,
AlertRegexMatcher,
)
from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher
class FakeTimeDelta:
@ -253,9 +249,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
@patch("matrix_alertbot.alertmanager.timedelta", side_effect=FakeTimeDelta)
async def test_create_silence_with_matchers(self, fake_timedelta: Mock) -> None:
matchers: List[AbstractAlertMatcher] = [
AlertMatcher(label="alertname", value="alert1")
]
matchers = [AlertMatcher(label="alertname", value="alert1")]
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
@ -277,7 +271,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
async def test_create_silence_with_regex_matchers(
self, fake_timedelta: Mock
) -> None:
matchers: List[AbstractAlertMatcher] = [
matchers: List[AlertMatcher] = [
AlertRegexMatcher(label="alertname", regex=r"alert\d+")
]
@ -298,7 +292,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
fake_timedelta.assert_called_once_with(seconds=86400)
async def test_create_silence_raise_missing_label(self) -> None:
matchers: List[AbstractAlertMatcher] = [
matchers = [
AlertMatcher(label="alertname", value="alert1"),
AlertMatcher(label="severity", value="critical"),
]
@ -318,9 +312,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
)
async def test_create_silence_raise_mismatch_label(self) -> None:
matchers: List[AbstractAlertMatcher] = [
AlertMatcher(label="alertname", value="alert2")
]
matchers = [AlertMatcher(label="alertname", value="alert2")]
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
@ -337,7 +329,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
)
async def test_create_silence_raise_mismatch_regex_label(self) -> None:
matchers: List[AbstractAlertMatcher] = [
matchers: List[AlertMatcher] = [
AlertRegexMatcher(label="alertname", regex=r"alert[^\d]+")
]
@ -377,15 +369,87 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
with self.assertRaises(AlertmanagerServerError):
await alertmanager.create_silence("fingerprint1", "1d", "user", [])
async def test_delete_silences_happy(self) -> None:
async def test_delete_silences_without_matchers(self) -> None:
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
alertmanager = AlertmanagerClient(
f"http://localhost:{port}", self.fake_cache
)
async with aiotools.closing_async(alertmanager):
silences = await alertmanager.delete_silences("fingerprint2")
self.assertEqual(["silence1", "silence2"], silences)
silences = await alertmanager.delete_silences("fingerprint2", [])
self.assertEqual(["silence1", "silence2"], silences)
async def test_delete_silences_with_matchers(self) -> None:
matchers = [AlertMatcher(label="alertname", value="alert2")]
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
alertmanager = AlertmanagerClient(
f"http://localhost:{port}", self.fake_cache
)
async with aiotools.closing_async(alertmanager):
silences = await alertmanager.delete_silences("fingerprint2", matchers)
self.assertEqual(["silence1", "silence2"], silences)
async def test_delete_silences_with_regex_matchers(self) -> None:
matchers: List[AlertMatcher] = [
AlertRegexMatcher(label="alertname", regex=r"alert\d+")
]
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
alertmanager = AlertmanagerClient(
f"http://localhost:{port}", self.fake_cache
)
async with aiotools.closing_async(alertmanager):
silences = await alertmanager.delete_silences("fingerprint2", matchers)
self.assertEqual(["silence1", "silence2"], silences)
async def test_delete_silences_raise_missing_label(self) -> None:
matchers = [
AlertMatcher(label="alertname", value="alert2"),
AlertMatcher(label="severity", value="critical"),
]
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
alertmanager = AlertmanagerClient(
f"http://localhost:{port}", self.fake_cache
)
async with aiotools.closing_async(alertmanager):
with self.assertRaises(AlertMismatchError):
await alertmanager.delete_silences("fingerprint2", matchers)
async def test_delete_silences_raise_mismatch_label(self) -> None:
matchers = [
AlertMatcher(label="alertname", value="alert1"),
]
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
alertmanager = AlertmanagerClient(
f"http://localhost:{port}", self.fake_cache
)
async with aiotools.closing_async(alertmanager):
with self.assertRaises(AlertMismatchError):
await alertmanager.delete_silences("fingerprint2", matchers)
async def test_delete_silences_raise_mismatch_regex_label(self) -> None:
matchers: List[AlertMatcher] = [
AlertRegexMatcher(label="alertname", regex=r"alert[^\d]+"),
]
async with FakeAlertmanagerServer() as fake_alertmanager_server:
port = fake_alertmanager_server.port
alertmanager = AlertmanagerClient(
f"http://localhost:{port}", self.fake_cache
)
async with aiotools.closing_async(alertmanager):
with self.assertRaises(AlertMismatchError):
await alertmanager.delete_silences("fingerprint2", matchers)
async def test_delete_silences_raise_silence_not_found(self) -> None:
async with FakeAlertmanagerServer() as fake_alertmanager_server:
@ -395,7 +459,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
)
async with aiotools.closing_async(alertmanager):
with self.assertRaises(SilenceNotFoundError):
await alertmanager.delete_silences("fingerprint1")
await alertmanager.delete_silences("fingerprint1", [])
async def test_delete_silences_raise_alert_not_found(self) -> None:
async with FakeAlertmanagerServerWithoutAlert() as fake_alertmanager_server:
@ -405,7 +469,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
)
async with aiotools.closing_async(alertmanager):
with self.assertRaises(AlertNotFoundError):
await alertmanager.delete_silences("fingerprint2")
await alertmanager.delete_silences("fingerprint2", [])
async def test_delete_silences_raise_alertmanager_error(self) -> None:
async with FakeAlertmanagerServerWithErrorDeleteSilence() as fake_alertmanager_server:
@ -417,7 +481,7 @@ class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase):
await alertmanager.get_alert("fingerprint1")
with self.assertRaises(AlertmanagerServerError):
await alertmanager.delete_silences("fingerprint2")
await alertmanager.delete_silences("fingerprint2", [])
async def test_find_alert_happy(self) -> None:
alertmanager = AlertmanagerClient("http://localhost", self.fake_cache)

57
tests/test_command.py
View file

@ -9,20 +9,22 @@ import matrix_alertbot.callback
from matrix_alertbot.alertmanager import AlertmanagerClient
from matrix_alertbot.command import Command
from matrix_alertbot.errors import AlertmanagerError
from matrix_alertbot.matcher import AbstractAlertMatcher, AlertMatcher
from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher
from tests.utils import make_awaitable
async def create_silence_raise_alertmanager_error(
fingerprint: str, duration: str, user: str, matchers: List[AbstractAlertMatcher]
fingerprint: str, duration: str, user: str, matchers: List[AlertMatcher]
) -> str:
if fingerprint == "fingerprint1":
raise AlertmanagerError
return "silence1"
async def delete_silence_raise_alertmanager_error(fingerprint: str) -> List[str]:
async def delete_silence_raise_alertmanager_error(
fingerprint: str, matchers: List[AlertMatcher]
) -> List[str]:
if fingerprint == "fingerprint1":
raise AlertmanagerError
return ["silence1"]
@ -225,9 +227,9 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase):
) -> None:
"""Tests the callback for InviteMemberEvents"""
# Tests that the bot attempts to join a room after being invited to it
matchers: List[AbstractAlertMatcher] = [
matchers = [
AlertMatcher(label="alertname", value="alert1"),
AlertMatcher(label="severity", value="critical"),
AlertRegexMatcher(label="severity", regex="critical"),
]
self.fake_message_event.source = self.fake_source_in_reply
@ -237,7 +239,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase):
self.fake_cache,
self.fake_alertmanager,
self.fake_config,
"ack alertname=alert1 severity=critical",
"ack alertname=alert1 severity=~critical",
self.fake_room,
self.fake_message_event,
)
@ -300,7 +302,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase):
) -> None:
"""Tests the callback for InviteMemberEvents"""
# Tests that the bot attempts to join a room after being invited to it
matchers: List[AbstractAlertMatcher] = [
matchers = [
AlertMatcher(label="alertname", value="alert1"),
AlertMatcher(label="severity", value="critical"),
]
@ -374,7 +376,9 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase):
)
@patch.object(matrix_alertbot.command, "send_text_to_room")
async def test_unack_in_reply(self, fake_send_text_to_room: Mock) -> None:
async def test_unack_in_reply_without_matchers(
self, fake_send_text_to_room: Mock
) -> None:
"""Tests the callback for InviteMemberEvents"""
# Tests that the bot attempts to join a room after being invited to it
@ -393,7 +397,40 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase):
# Check that we attempted to create silences
self.fake_alertmanager.delete_silences.assert_has_calls(
[call(fingerprint) for fingerprint in self.fake_fingerprints]
[call(fingerprint, []) for fingerprint in self.fake_fingerprints]
)
fake_send_text_to_room.assert_called_with(
self.fake_client, self.fake_room.room_id, "Removed 4 silences."
)
@patch.object(matrix_alertbot.command, "send_text_to_room")
async def test_unack_in_reply_with_matchers(
self, fake_send_text_to_room: Mock
) -> None:
"""Tests the callback for InviteMemberEvents"""
# Tests that the bot attempts to join a room after being invited to it
matchers = [
AlertMatcher(label="alertname", value="alert1"),
AlertRegexMatcher(label="severity", regex="critical"),
]
self.fake_message_event.source = self.fake_source_in_reply
command = Command(
self.fake_client,
self.fake_cache,
self.fake_alertmanager,
self.fake_config,
"unack alertname=alert1 severity=~critical",
self.fake_room,
self.fake_message_event,
)
await command._unack()
# Check that we attempted to create silences
self.fake_alertmanager.delete_silences.assert_has_calls(
[call(fingerprint, matchers) for fingerprint in self.fake_fingerprints]
)
fake_send_text_to_room.assert_called_with(
self.fake_client, self.fake_room.room_id, "Removed 4 silences."
@ -425,7 +462,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase):
# Check that we attempted to create silences
self.fake_alertmanager.delete_silences.assert_has_calls(
[call(fingerprint) for fingerprint in self.fake_fingerprints]
[call(fingerprint, []) for fingerprint in self.fake_fingerprints]
)
fake_send_text_to_room.assert_called_with(
self.fake_client, self.fake_room.room_id, "Removed 1 silences."