From 2359f6ca7739b5f1157d07576faa439dfd8638fa Mon Sep 17 00:00:00 2001 From: HgO Date: Mon, 11 Jul 2022 23:18:57 +0200 Subject: [PATCH] add unit tests for webhook, alert and missing cache --- matrix_alertbot/alert.py | 15 +-- matrix_alertbot/command.py | 12 +-- matrix_alertbot/webhook.py | 30 ++++-- tests/test_alert.py | 103 +++++++++++++++++++++ tests/test_alertmanager.py | 2 - tests/test_command.py | 181 +++++++++++++++++++++++++++++++++++-- tests/test_config.py | 1 - tests/test_webhook.py | 177 ++++++++++++++++++++++++++++++++++++ 8 files changed, 486 insertions(+), 35 deletions(-) create mode 100644 tests/test_alert.py create mode 100644 tests/test_webhook.py diff --git a/matrix_alertbot/alert.py b/matrix_alertbot/alert.py index 5d955c5..2b8748d 100644 --- a/matrix_alertbot/alert.py +++ b/matrix_alertbot/alert.py @@ -14,23 +14,16 @@ class Alert: self, id: str, url: str, + labels: Dict[str, str], + annotations: Dict[str, str], firing: bool = True, - labels: Dict[str, str] = None, - annotations: Dict[str, str] = None, ): self.id = id self.url = url self.firing = firing - if labels is None: - self.labels = {} - else: - self.labels = labels - - if annotations is None: - self.annotations = {} - else: - self.annotations = annotations + self.labels = labels + self.annotations = annotations if self.firing: self.status = self.labels["severity"] diff --git a/matrix_alertbot/command.py b/matrix_alertbot/command.py index 67e77b3..6f5e570 100644 --- a/matrix_alertbot/command.py +++ b/matrix_alertbot/command.py @@ -7,7 +7,7 @@ from nio import AsyncClient, MatrixRoom from matrix_alertbot.alertmanager import AlertmanagerClient from matrix_alertbot.chat_functions import send_text_to_room from matrix_alertbot.config import Config -from matrix_alertbot.errors import AlertmanagerError, AlertNotFoundError +from matrix_alertbot.errors import AlertmanagerError from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher logger = logging.getLogger(__name__) @@ -89,9 +89,8 @@ class Command: logger.debug(f"Read alert fingerprints for event {self.event_id} from cache") if self.event_id not in self.cache: - raise AlertNotFoundError( - f"Cannot find fingerprints for event {self.event_id} in cache" - ) + logger.error(f"Cannot find fingerprints for event {self.event_id} in cache") + return alert_fingerprints = self.cache[self.event_id] logger.debug(f"Found {len(alert_fingerprints)} in cache") @@ -135,9 +134,8 @@ class Command: logger.debug(f"Read alert fingerprints for event {self.event_id} from cache") if self.event_id not in self.cache: - raise AlertNotFoundError( - f"Cannot find fingerprints for event {self.event_id} in cache" - ) + logger.error(f"Cannot find fingerprints for event {self.event_id} in cache") + return alert_fingerprints = self.cache[self.event_id] logger.debug(f"Found {len(alert_fingerprints)} in cache") diff --git a/matrix_alertbot/webhook.py b/matrix_alertbot/webhook.py index 9abedec..3de0603 100644 --- a/matrix_alertbot/webhook.py +++ b/matrix_alertbot/webhook.py @@ -2,11 +2,12 @@ from __future__ import annotations import logging -from aiohttp import web, web_request +import prometheus_client +from aiohttp import ClientError, web, web_request from aiohttp_prometheus_exporter.handler import metrics from aiohttp_prometheus_exporter.middleware import prometheus_middleware_factory from diskcache import Cache -from nio import AsyncClient, SendRetryError +from nio import AsyncClient, LocalProtocolError from matrix_alertbot.alert import Alert from matrix_alertbot.chat_functions import send_text_to_room @@ -25,10 +26,22 @@ async def create_alert(request: web_request.Request) -> web.Response: config: Config = request.app["config"] cache: Cache = request.app["cache"] + if "alerts" not in data: + return web.Response(status=400, body="Data must contain 'alerts' key.") + + if not isinstance(data["alerts"], list): + return web.Response(status=400, body="Alerts must be a list.") + + if len(data["alerts"]) == 0: + return web.Response(status=400, body="Alerts cannot be empty.") + plaintext = "" html = "" for i, alert in enumerate(data["alerts"]): - alert = Alert.from_dict(alert) + try: + alert = Alert.from_dict(alert) + except KeyError: + return web.Response(status=400, body=f"Invalid alert: {alert}.") if i != 0: plaintext += "\n" @@ -40,9 +53,11 @@ async def create_alert(request: web_request.Request) -> web.Response: event = await send_text_to_room( client, config.room_id, plaintext, html, notice=False ) - except SendRetryError as e: + except (LocalProtocolError, ClientError) as e: logger.error(e) - return web.Response(status=500) + return web.Response( + status=500, body="An error occured when sending alerts to Matrix room." + ) fingerprints = tuple(alert["fingerprint"] for alert in data["alerts"]) cache.set( @@ -59,7 +74,10 @@ class Webhook: self.app["cache"] = cache self.app.add_routes(routes) - self.app.middlewares.append(prometheus_middleware_factory()) + prometheus_registry = prometheus_client.CollectorRegistry(auto_describe=True) + self.app.middlewares.append( + prometheus_middleware_factory(registry=prometheus_registry) + ) self.app.router.add_get("/metrics", metrics()) self.runner = web.AppRunner(self.app) diff --git a/tests/test_alert.py b/tests/test_alert.py new file mode 100644 index 0000000..0c10adc --- /dev/null +++ b/tests/test_alert.py @@ -0,0 +1,103 @@ +import unittest +from typing import Dict + +from matrix_alertbot.alert import Alert + + +class AlertTestCase(unittest.TestCase): + def setUp(self) -> None: + self.alert_dict: Dict = { + "fingerprint": "fingerprint1", + "generatorURL": "http://example.com", + "status": "unknown", + "labels": {"alertname": "alert1", "severity": "critical", "job": "job1"}, + "annotations": {"description": "some description"}, + } + + def test_create_firing_alert_from_dict(self) -> None: + self.alert_dict["status"] = "firing" + alert = Alert.from_dict(self.alert_dict) + + self.assertEqual("fingerprint1", alert.id) + self.assertEqual("http://example.com", alert.url) + self.assertTrue(alert.firing) + self.assertEqual("critical", alert.status) + self.assertDictEqual( + {"alertname": "alert1", "severity": "critical", "job": "job1"}, alert.labels + ) + self.assertDictEqual({"description": "some description"}, alert.annotations) + + def test_create_resolved_alert_from_dict(self) -> None: + self.alert_dict["status"] = "resolved" + alert = Alert.from_dict(self.alert_dict) + + self.assertEqual("resolved", alert.status) + self.assertFalse(alert.firing) + + def test_create_unknown_alert_from_dict(self) -> None: + alert = Alert.from_dict(self.alert_dict) + + self.assertEqual("resolved", alert.status) + self.assertFalse(alert.firing) + + def test_display_firing_critical_alert(self) -> None: + self.alert_dict["status"] = "firing" + alert = Alert.from_dict(self.alert_dict) + alert.labels["severity"] = "critical" + + html = alert.html() + self.assertEqual( + "[🔥 CRITICAL] " + "alert1 (job1)
" + "some description", + html, + ) + + plaintext = alert.plaintext() + self.assertEqual("[🔥 CRITICAL] alert1: some description", plaintext) + + def test_display_firing_warning_alert(self) -> None: + self.alert_dict["status"] = "firing" + self.alert_dict["labels"]["severity"] = "warning" + alert = Alert.from_dict(self.alert_dict) + + html = alert.html() + self.assertEqual( + "[⚠️ WARNING] " + "alert1 (job1)
" + "some description", + html, + ) + + plaintext = alert.plaintext() + self.assertEqual("[⚠️ WARNING] alert1: some description", plaintext) + + def test_display_firing_unknown_alert(self) -> None: + self.alert_dict["status"] = "firing" + self.alert_dict["labels"]["severity"] = "unknown" + alert = Alert.from_dict(self.alert_dict) + + with self.assertRaisesRegex(KeyError, "unknown"): + alert.html() + + with self.assertRaisesRegex(KeyError, "unknown"): + alert.plaintext() + + def test_display_resolved_alert(self) -> None: + self.alert_dict["status"] = "resolved" + alert = Alert.from_dict(self.alert_dict) + + html = alert.html() + self.assertEqual( + "[🥦 RESOLVED] " + "alert1 (job1)
" + "some description", + html, + ) + + plaintext = alert.plaintext() + self.assertEqual("[🥦 RESOLVED] alert1: some description", plaintext) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_alertmanager.py b/tests/test_alertmanager.py index 6158dab..cae48f2 100644 --- a/tests/test_alertmanager.py +++ b/tests/test_alertmanager.py @@ -42,7 +42,6 @@ class AbstractFakeAlertmanagerServer: ) self.runner = web.AppRunner(self.app) - self.response = None async def __aenter__(self) -> AbstractFakeAlertmanagerServer: await self.start() @@ -126,7 +125,6 @@ class FakeAlertmanagerServerWithErrorDeleteSilence(FakeAlertmanagerServer): class AlertmanagerClientTestCase(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self) -> None: - self.fake_fingerprints = Mock(return_value=["fingerprint1", "fingerprint2"]) self.fake_cache = MagicMock(spec=Cache) self.fake_cache.__getitem__ = self.fake_fingerprints diff --git a/tests/test_command.py b/tests/test_command.py index 6dc850e..8992481 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -144,7 +144,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): fake_unknown.assert_called_once() @patch.object(matrix_alertbot.command, "send_text_to_room") - async def test_ack_in_reply_without_duration_nor_matchers( + async def test_ack_without_duration_nor_matchers( self, fake_send_text_to_room: Mock ) -> None: """Tests the callback for InviteMemberEvents""" @@ -176,7 +176,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) @patch.object(matrix_alertbot.command, "send_text_to_room") - async def test_ack_in_reply_without_duration_and_with_matchers( + async def test_ack_without_duration_and_with_matchers( self, fake_send_text_to_room: Mock ) -> None: """Tests the callback for InviteMemberEvents""" @@ -217,7 +217,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) @patch.object(matrix_alertbot.command, "send_text_to_room") - async def test_ack_in_reply_with_duration_and_without_matchers( + async def test_ack_with_duration_and_without_matchers( self, fake_send_text_to_room: Mock ) -> None: """Tests the callback for InviteMemberEvents""" @@ -249,7 +249,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) @patch.object(matrix_alertbot.command, "send_text_to_room") - async def test_ack_in_reply_with_duration_and_matchers( + async def test_ack_with_duration_and_matchers( self, fake_send_text_to_room: Mock ) -> None: """Tests the callback for InviteMemberEvents""" @@ -326,12 +326,36 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) @patch.object(matrix_alertbot.command, "send_text_to_room") - async def test_unack_in_reply_without_matchers( + async def test_ack_with_event_not_found_in_cache( self, fake_send_text_to_room: Mock ) -> None: """Tests the callback for InviteMemberEvents""" # Tests that the bot attempts to join a room after being invited to it + self.fake_cache.__contains__.return_value = False + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "ack", + self.fake_room, + self.fake_sender, + self.fake_event_id, + ) + + await command._ack() + + # Check that we attempted to create silences + self.fake_alertmanager.create_silence.assert_not_called() + fake_send_text_to_room.assert_not_called() + + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_unack_without_matchers(self, fake_send_text_to_room: Mock) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + command = Command( self.fake_client, self.fake_cache, @@ -353,9 +377,7 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): ) @patch.object(matrix_alertbot.command, "send_text_to_room") - async def test_unack_in_reply_with_matchers( - self, fake_send_text_to_room: Mock - ) -> None: + async def test_unack_with_matchers(self, fake_send_text_to_room: Mock) -> None: """Tests the callback for InviteMemberEvents""" # Tests that the bot attempts to join a room after being invited to it @@ -415,6 +437,149 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): self.fake_client, self.fake_room.room_id, "Removed 1 silences." ) + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_unack_with_event_not_found_in_cache( + self, fake_send_text_to_room: Mock + ) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + + self.fake_cache.__contains__.return_value = False + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "unack", + self.fake_room, + self.fake_sender, + self.fake_event_id, + ) + + await command._unack() + + # Check that we attempted to create silences + self.fake_alertmanager.create_silence.assert_not_called() + fake_send_text_to_room.assert_not_called() + + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_help_without_topic(self, fake_send_text_to_room: Mock) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "help", + self.fake_room, + self.fake_sender, + self.fake_event_id, + ) + + await command._show_help() + + # Check that we attempted to create silences + fake_send_text_to_room.assert_called_once() + _, _, text = fake_send_text_to_room.call_args.args + self.assertIn("help commands", text) + + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_help_with_rules_topic(self, fake_send_text_to_room: Mock) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "help rules", + self.fake_room, + self.fake_sender, + self.fake_event_id, + ) + + await command._show_help() + + # Check that we attempted to create silences + fake_send_text_to_room.assert_called_once() + _, _, text = fake_send_text_to_room.call_args.args + self.assertIn("rules!", text) + + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_help_with_commands_topic(self, fake_send_text_to_room: Mock) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "help commands", + self.fake_room, + self.fake_sender, + self.fake_event_id, + ) + + await command._show_help() + + # Check that we attempted to create silences + fake_send_text_to_room.assert_called_once() + _, _, text = fake_send_text_to_room.call_args.args + self.assertIn("Available commands", text) + + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_help_with_unknown_topic(self, fake_send_text_to_room: Mock) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "help unknown", + self.fake_room, + self.fake_sender, + self.fake_event_id, + ) + + await command._show_help() + + # Check that we attempted to create silences + fake_send_text_to_room.assert_called_once() + _, _, text = fake_send_text_to_room.call_args.args + self.assertEqual("Unknown help topic!", text) + + @patch.object(matrix_alertbot.command, "send_text_to_room") + async def test_unknown_command(self, fake_send_text_to_room: Mock) -> None: + """Tests the callback for InviteMemberEvents""" + # Tests that the bot attempts to join a room after being invited to it + + command = Command( + self.fake_client, + self.fake_cache, + self.fake_alertmanager, + self.fake_config, + "", + self.fake_room, + self.fake_sender, + self.fake_event_id, + ) + + await command._unknown_command() + + # Check that we attempted to create silences + fake_send_text_to_room.assert_called_once_with( + self.fake_client, + self.fake_room.room_id, + "Unknown command ''. Try the 'help' command for more information.", + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_config.py b/tests/test_config.py index 4f8f552..cbfc862 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,7 +7,6 @@ import yaml from matrix_alertbot.config import Config from matrix_alertbot.errors import ( - ConfigError, InvalidConfigError, ParseConfigError, RequiredConfigKeyError, diff --git a/tests/test_webhook.py b/tests/test_webhook.py new file mode 100644 index 0000000..569b02c --- /dev/null +++ b/tests/test_webhook.py @@ -0,0 +1,177 @@ +import unittest +from typing import Dict +from unittest.mock import Mock, patch + +import aiohttp.test_utils +import nio +from aiohttp import web +from diskcache import Cache +from nio import LocalProtocolError, RoomSendResponse + +import matrix_alertbot.webhook +from matrix_alertbot.config import Config +from matrix_alertbot.webhook import Webhook + + +def send_text_to_room_raise_error( + client: nio.AsyncClient, room_id: str, plaintext: str, html: str, notice: bool +) -> RoomSendResponse: + raise LocalProtocolError() + + +class WebhookApplicationTestCase(aiohttp.test_utils.AioHTTPTestCase): + async def get_application(self) -> web.Application: + self.fake_client = Mock(spec=nio.AsyncClient) + self.fake_cache = Mock(spec=Cache) + + self.fake_config = Mock(spec=Config) + self.fake_config.port = aiohttp.test_utils.unused_port() + self.fake_config.address = "localhost" + self.fake_config.socket = "webhook.sock" + self.fake_config.room_id = "!abcdefg:example.com" + self.fake_config.cache_expire_time = 0 + + self.fake_alerts = { + "alerts": [ + { + "fingerprint": "fingerprint1", + "generatorURL": "http://example.com/alert1", + "status": "firing", + "labels": { + "alertname": "alert1", + "severity": "critical", + "job": "job1", + }, + "annotations": {"description": "some description1"}, + }, + { + "fingerprint": "fingerprint2", + "generatorURL": "http://example.com/alert2", + "status": "resolved", + "labels": { + "alertname": "alert2", + "severity": "warning", + "job": "job2", + }, + "annotations": {"description": "some description2"}, + }, + ] + } + + webhook = Webhook(self.fake_client, self.fake_cache, self.fake_config) + return webhook.app + + @patch.object(matrix_alertbot.webhook, "send_text_to_room") + async def test_post_alert(self, fake_send_text_to_room: Mock) -> None: + data = self.fake_alerts + async with self.client.request("POST", "/alert", json=data) as response: + self.assertEqual(200, response.status) + fake_send_text_to_room.assert_called_once_with( + self.fake_client, + self.fake_config.room_id, + "[🔥 CRITICAL] alert1: some description1\n" + "[🥦 RESOLVED] alert2: some description2", + "[🔥 CRITICAL] " + "alert1 (job1)
" + "some description1
\n" + "[🥦 RESOLVED] " + "alert2 (job2)
" + "some description2", + notice=False, + ) + + @patch.object(matrix_alertbot.webhook, "send_text_to_room") + async def test_post_alert_with_empty_data( + self, fake_send_text_to_room: Mock + ) -> None: + async with self.client.request("POST", "/alert", json={}) as response: + self.assertEqual(400, response.status) + error_msg = await response.text() + self.assertEqual("Data must contain 'alerts' key.", error_msg) + fake_send_text_to_room.assert_not_called() + + @patch.object(matrix_alertbot.webhook, "send_text_to_room") + async def test_post_alert_with_empty_alerts( + self, fake_send_text_to_room: Mock + ) -> None: + data: Dict = {"alerts": []} + async with self.client.request("POST", "/alert", json=data) as response: + self.assertEqual(400, response.status) + error_msg = await response.text() + self.assertEqual("Alerts cannot be empty.", error_msg) + fake_send_text_to_room.assert_not_called() + + @patch.object(matrix_alertbot.webhook, "send_text_to_room") + async def test_post_alert_with_invalid_alerts( + self, fake_send_text_to_room: Mock + ) -> None: + data = {"alerts": "invalid"} + async with self.client.request("POST", "/alert", json=data) as response: + self.assertEqual(400, response.status) + error_msg = await response.text() + self.assertEqual("Alerts must be a list.", error_msg) + fake_send_text_to_room.assert_not_called() + + @patch.object(matrix_alertbot.webhook, "send_text_to_room") + async def test_post_alert_with_empty_items( + self, fake_send_text_to_room: Mock + ) -> None: + data: Dict = {"alerts": [{}]} + async with self.client.request("POST", "/alert", json=data) as response: + self.assertEqual(400, response.status) + error_msg = await response.text() + self.assertEqual("Invalid alert: {}.", error_msg) + fake_send_text_to_room.assert_not_called() + + @patch.object( + matrix_alertbot.webhook, + "send_text_to_room", + side_effect=send_text_to_room_raise_error, + ) + async def test_post_alert_with_send_error( + self, fake_send_text_to_room: Mock + ) -> None: + data = self.fake_alerts + async with self.client.request("POST", "/alert", json=data) as response: + self.assertEqual(500, response.status) + error_msg = await response.text() + self.assertEqual( + "An error occured when sending alerts to Matrix room.", error_msg + ) + fake_send_text_to_room.assert_called_once() + + +class WebhookServerTestCase(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self) -> None: + self.fake_client = Mock(spec=nio.AsyncClient) + self.fake_cache = Mock(spec=Cache) + + self.fake_config = Mock(spec=Config) + self.fake_config.port = aiohttp.test_utils.unused_port() + self.fake_config.address = "localhost" + self.fake_config.socket = "webhook.sock" + self.fake_config.room_id = "!abcdefg:example.com" + self.fake_config.cache_expire_time = 0 + + @patch.object(matrix_alertbot.webhook.web, "TCPSite", autospec=True) + async def test_webhook_start_address_port(self, fake_tcp_site: Mock) -> None: + webhook = Webhook(self.fake_client, self.fake_cache, self.fake_config) + await webhook.start() + + fake_tcp_site.assert_called_once_with( + webhook.runner, self.fake_config.address, self.fake_config.port + ) + + await webhook.close() + + @patch.object(matrix_alertbot.webhook.web, "UnixSite", autospec=True) + async def test_webhook_start_unix_socket(self, fake_unix_site: Mock) -> None: + self.fake_config.address = None + self.fake_config.port = None + + webhook = Webhook(self.fake_client, self.fake_cache, self.fake_config) + await webhook.start() + + fake_unix_site.assert_called_once_with(webhook.runner, self.fake_config.socket) + + await webhook.close()