import re
import unittest
from typing import Dict
from unittest.mock import Mock, call, patch
import aiohttp.test_utils
import nio
from aiohttp import web, web_request
from diskcache import Cache
import matrix_alertbot.webhook
from matrix_alertbot.alert import Alert, AlertRenderer
from matrix_alertbot.alertmanager import AlertmanagerClient
from matrix_alertbot.config import BiDict, Config
from matrix_alertbot.errors import (
AlertmanagerError,
MatrixClientError,
SilenceExtendError,
SilenceNotFoundError,
)
from matrix_alertbot.matrix import MatrixClientPool
from matrix_alertbot.webhook import Webhook, create_alert
def update_silence_raise_silence_not_found(fingerprint: str) -> str:
raise SilenceNotFoundError
def update_silence_raise_silence_extend_error(fingerprint: str) -> str:
raise SilenceExtendError
def update_silence_raise_alertmanager_error(fingerprint: str) -> str:
raise AlertmanagerError
class WebhookApplicationTestCase(aiohttp.test_utils.AioHTTPTestCase):
async def get_application(self) -> web.Application:
self.fake_matrix_client = Mock(spec=nio.AsyncClient)
self.fake_matrix_client_pool = Mock(spec=MatrixClientPool)
self.fake_matrix_client_pool.matrix_client = self.fake_matrix_client
self.fake_matrix_client_pool.dm_rooms = {
"@fake_dm_user:example.com": "!dmroom:example.com"
}
self.fake_alertmanager_client = Mock(spec=AlertmanagerClient)
self.fake_alert_renderer = Mock(spec=AlertRenderer)
self.fake_cache = Mock(spec=Cache)
self.fake_room_id = "!abcdefg:example.com"
self.fake_config = Mock(spec=Config)
self.fake_config.port = aiohttp.test_utils.unused_port()
self.fake_config.address = "localhost"
self.fake_config.socket = "webhook.sock"
self.fake_config.allowed_rooms = [self.fake_room_id]
self.fake_config.cache_expire_time = 0
self.fake_config.template_dir = None
self.fake_config.dm_select_label = "uuid"
self.fake_config.dm_filter_labels = {"matrix": re.compile("dm")}
self.fake_config.dm_users = BiDict(
{"a7b37c33-574c-45ac-bb07-a3b314c2da54": "@fake_dm_user:example.com"}
)
self.fake_request = Mock(spec=web_request.Request)
self.fake_request.app = {
"alertmanager_client": self.fake_alertmanager_client,
"alert_renderer": self.fake_alert_renderer,
"matrix_client_pool": self.fake_matrix_client_pool,
"cache": self.fake_cache,
"config": self.fake_config,
}
self.fake_alert_1 = {
"fingerprint": "fingerprint1",
"generatorURL": "http://example.com/alert1",
"status": "firing",
"labels": {
"alertname": "alert1",
"severity": "critical",
"job": "job1",
},
"annotations": {"description": "some description1"},
}
self.fake_alert_2 = {
"fingerprint": "fingerprint2",
"generatorURL": "http://example.com/alert2",
"status": "resolved",
"labels": {
"alertname": "alert2",
"severity": "warning",
"job": "job2",
},
"annotations": {"description": "some description2"},
}
self.fake_alerts = {
"alerts": [
self.fake_alert_1,
self.fake_alert_2,
]
}
self.fake_dm_alert = {
"fingerprint": "fingerprint",
"generatorURL": "http://example.com/alert",
"status": "firing",
"labels": {
"alertname": "alert",
"severity": "warning",
"job": "job",
"uuid": "a7b37c33-574c-45ac-bb07-a3b314c2da54",
"matrix": "dm",
},
"annotations": {"description": "some description"},
}
self.fake_dm_alerts = {"alerts": [self.fake_dm_alert]}
webhook = Webhook(
self.fake_matrix_client_pool,
self.fake_alertmanager_client,
self.fake_cache,
self.fake_config,
)
return webhook.app
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_with_silence_not_found(
self, fake_send_text_to_room: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_not_found
)
data = self.fake_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(200, response.status)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
"fingerprint1"
)
self.assertEqual(2, fake_send_text_to_room.call_count)
fake_send_text_to_room.assert_has_calls(
[
call(
self.fake_matrix_client,
self.fake_room_id,
"[🔥 CRITICAL] alert1: some description1",
'\n [🔥 CRITICAL]\n '
'alert1\n (job1)
\n'
"some description1",
notice=False,
),
call(
self.fake_matrix_client,
self.fake_room_id,
"[🥦 RESOLVED] alert2: some description2",
'\n [🥦 RESOLVED]\n '
'alert2\n (job2)
\n'
"some description2",
notice=False,
),
],
any_order=True,
)
self.fake_cache.set.assert_called_once_with(
fake_send_text_to_room.return_value.event_id,
"fingerprint1",
expire=self.fake_config.cache_expire_time,
)
self.assertEqual(2, self.fake_cache.delete.call_count)
self.fake_cache.delete.assert_has_calls(
[call("fingerprint1"), call("fingerprint2")]
)
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_with_silence_extend_error(
self, fake_send_text_to_room: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_extend_error
)
data = self.fake_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(200, response.status)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
"fingerprint1"
)
self.assertEqual(2, fake_send_text_to_room.call_count)
fake_send_text_to_room.assert_has_calls(
[
call(
self.fake_matrix_client,
self.fake_room_id,
"[🔥 CRITICAL] alert1: some description1",
'\n [🔥 CRITICAL]\n '
'alert1\n (job1)
\n'
"some description1",
notice=False,
),
call(
self.fake_matrix_client,
self.fake_room_id,
"[🥦 RESOLVED] alert2: some description2",
'\n [🥦 RESOLVED]\n '
'alert2\n (job2)
\n'
"some description2",
notice=False,
),
],
any_order=True,
)
self.fake_cache.set.assert_called_once_with(
fake_send_text_to_room.return_value.event_id,
"fingerprint1",
expire=self.fake_config.cache_expire_time,
)
self.fake_cache.delete.assert_called_once_with("fingerprint2")
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_with_alertmanager_error(
self, fake_send_text_to_room: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_alertmanager_error
)
data = self.fake_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(500, response.status)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
"fingerprint1"
)
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_with_existing_silence(
self, fake_send_text_to_room: Mock
) -> None:
self.fake_alertmanager_client.update_silence.return_value = "silence1"
data = self.fake_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(200, response.status)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
"fingerprint1"
)
fake_send_text_to_room.assert_called_once_with(
self.fake_matrix_client,
self.fake_room_id,
"[🥦 RESOLVED] alert2: some description2",
'\n [🥦 RESOLVED]\n '
'alert2\n (job2)
\n'
"some description2",
notice=False,
)
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_called_once_with("fingerprint2")
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_in_unauthorized_room(
self, fake_send_text_to_room: Mock
) -> None:
room_id = "!unauthorized_room@example.com"
async with self.client.request(
"POST", f"/alerts/{room_id}", json=self.fake_alerts
) as response:
self.assertEqual(401, response.status)
error_msg = await response.text()
self.assertEqual(
"Cannot send alerts to room ID !unauthorized_room@example.com.", error_msg
)
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_for_dm_user(self, fake_send_text_to_room: Mock) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_not_found
)
data = self.fake_dm_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(200, response.status)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
"fingerprint"
)
self.assertEqual(1, fake_send_text_to_room.call_count)
fake_send_text_to_room.assert_has_calls(
[
call(
self.fake_matrix_client,
self.fake_matrix_client_pool.dm_rooms["@fake_dm_user:example.com"],
"[⚠️ WARNING] alert: some description",
'\n [⚠️ WARNING]\n '
'alert\n (job)
\n'
"some description",
notice=False,
),
]
)
self.fake_cache.set.assert_called_once_with(
fake_send_text_to_room.return_value.event_id,
"fingerprint",
expire=self.fake_config.cache_expire_time,
)
self.assertEqual(1, self.fake_cache.delete.call_count)
self.fake_cache.delete.assert_has_calls([call("fingerprint")])
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_for_unknown_dm_user(
self, fake_send_text_to_room: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_not_found
)
self.fake_config.dm_users = BiDict()
data = self.fake_dm_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(200, response.status)
self.fake_alertmanager_client.update_silence.assert_not_called()
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_for_dm_user_with_unknown_room(
self, fake_send_text_to_room: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_not_found
)
self.fake_matrix_client_pool.dm_rooms = {}
data = self.fake_dm_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(200, response.status)
self.fake_alertmanager_client.update_silence.assert_not_called()
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_with_empty_data(
self, fake_send_text_to_room: Mock
) -> None:
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json={}
) as response:
self.assertEqual(400, response.status)
error_msg = await response.text()
self.assertEqual("Data must contain 'alerts' key.", error_msg)
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_empty_alerts(self, fake_send_text_to_room: Mock) -> None:
data: Dict = {"alerts": []}
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(400, response.status)
error_msg = await response.text()
self.assertEqual("Alerts cannot be empty.", error_msg)
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_invalid_alerts(self, fake_send_text_to_room: Mock) -> None:
data = {"alerts": "invalid"}
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(400, response.status)
error_msg = await response.text()
self.assertEqual("Alerts must be a list, got 'str'.", error_msg)
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room")
async def test_post_alerts_with_empty_items(
self, fake_send_text_to_room: Mock
) -> None:
data: Dict = {"alerts": [{}]}
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(400, response.status)
error_msg = await response.text()
self.assertEqual("Invalid alert: {}.", error_msg)
fake_send_text_to_room.assert_not_called()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "logger", autospec=True)
@patch.object(
matrix_alertbot.webhook,
"send_text_to_room",
side_effect=nio.exceptions.LocalProtocolError("Local protocol error"),
)
async def test_post_alerts_raise_send_error(
self, fake_send_text_to_room: Mock, fake_logger: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_not_found
)
data = self.fake_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(500, response.status)
error_msg = await response.text()
self.assertEqual(
"An error occured when sending alert with fingerprint 'fingerprint1' to Matrix room.",
error_msg,
)
fake_send_text_to_room.assert_called_once()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_called_once_with("fingerprint1")
fake_logger.exception.assert_called_once_with(
"Unable to send alert fingerprint1 to Matrix room !abcdefg:example.com",
exc_info=fake_send_text_to_room.side_effect,
)
@patch.object(matrix_alertbot.webhook, "logger", autospec=True)
@patch.object(
matrix_alertbot.webhook,
"create_alert",
side_effect=MatrixClientError("Matrix client error"),
)
async def test_post_alerts_raise_matrix_client_error(
self, fake_create_alert: Mock, fake_logger: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_not_found
)
data = self.fake_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(500, response.status)
error_msg = await response.text()
self.assertEqual(
"An error occured when sending alert with fingerprint 'fingerprint1' to Matrix room.",
error_msg,
)
fake_create_alert.assert_called_once()
fake_logger.exception.assert_called_once_with(
"Unable to send alert fingerprint1 to Matrix room !abcdefg:example.com",
exc_info=fake_create_alert.side_effect,
)
@patch.object(matrix_alertbot.webhook, "logger", autospec=True)
@patch.object(
matrix_alertbot.webhook,
"send_text_to_room",
side_effect=Exception("Exception"),
)
async def test_post_alerts_raise_exception(
self, fake_send_text_to_room: Mock, fake_logger: Mock
) -> None:
self.fake_alertmanager_client.update_silence.side_effect = (
update_silence_raise_silence_not_found
)
data = self.fake_alerts
async with self.client.request(
"POST", f"/alerts/{self.fake_room_id}", json=data
) as response:
self.assertEqual(500, response.status)
error_msg = await response.text()
self.assertEqual(
"An exception occured when sending alert with fingerprint 'fingerprint1' to Matrix room.",
error_msg,
)
fake_send_text_to_room.assert_called_once()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_called_once_with("fingerprint1")
fake_logger.exception.assert_called_once_with(
"Unable to send alert fingerprint1 to Matrix room !abcdefg:example.com",
exc_info=fake_send_text_to_room.side_effect,
)
async def test_create_alert_update_silence(self) -> None:
fake_alert = Alert(
fingerprint="fingerprint",
url="https://example.com",
firing=True,
labels={"severity": "critical"},
annotations={"description": "dummy description"},
)
await create_alert(fake_alert, self.fake_room_id, self.fake_request)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
fake_alert.fingerprint
)
self.fake_alert_renderer.render.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room", autospec=True)
async def test_create_alert_with_silence_not_found_error(
self, fake_send_text_to_room: Mock
) -> None:
fake_alert = Alert(
fingerprint="fingerprint",
url="https://example.com",
firing=True,
labels={"severity": "critical"},
annotations={"description": "dummy description"},
)
self.fake_alertmanager_client.update_silence.side_effect = SilenceNotFoundError
await create_alert(fake_alert, self.fake_room_id, self.fake_request)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
fake_alert.fingerprint
)
self.fake_alert_renderer.render.assert_has_calls(
[call(fake_alert, html=False), call(fake_alert, html=True)]
)
fake_send_text_to_room.assert_called_once()
self.fake_cache.set.assert_called_once_with(
fake_send_text_to_room.return_value.event_id,
fake_alert.fingerprint,
expire=self.fake_config.cache_expire_time,
)
self.fake_cache.delete.assert_called_once_with(fake_alert.fingerprint)
@patch.object(matrix_alertbot.webhook, "send_text_to_room", autospec=True)
async def test_create_alert_with_silence_extend_error(
self, fake_send_text_to_room: Mock
) -> None:
fake_alert = Alert(
fingerprint="fingerprint",
url="https://example.com",
firing=True,
labels={"severity": "critical"},
annotations={"description": "dummy description"},
)
self.fake_alertmanager_client.update_silence.side_effect = SilenceExtendError
await create_alert(fake_alert, self.fake_room_id, self.fake_request)
self.fake_alertmanager_client.update_silence.assert_called_once_with(
fake_alert.fingerprint
)
self.fake_alert_renderer.render.assert_has_calls(
[call(fake_alert, html=False), call(fake_alert, html=True)]
)
fake_send_text_to_room.assert_called_once()
self.fake_cache.set.assert_called_once_with(
fake_send_text_to_room.return_value.event_id,
fake_alert.fingerprint,
expire=self.fake_config.cache_expire_time,
)
self.fake_cache.delete.assert_not_called()
@patch.object(matrix_alertbot.webhook, "send_text_to_room", autospec=True)
async def test_create_alert_not_firing(self, fake_send_text_to_room: Mock) -> None:
fake_alert = Alert(
fingerprint="fingerprint",
url="https://example.com",
firing=False,
labels={},
annotations={"description": "dummy description"},
)
await create_alert(fake_alert, self.fake_room_id, self.fake_request)
self.fake_alertmanager_client.update_silence.assert_not_called()
self.fake_alert_renderer.render.assert_has_calls(
[call(fake_alert, html=False), call(fake_alert, html=True)]
)
fake_send_text_to_room.assert_called_once()
self.fake_cache.set.assert_not_called()
self.fake_cache.delete.assert_called_once_with(fake_alert.fingerprint)
@patch.object(matrix_alertbot.webhook, "send_text_to_room", autospec=True)
async def test_create_alert_not_firing_raise_matrix_client_error(
self, fake_send_text_to_room: Mock
) -> None:
fake_alert = Alert(
fingerprint="fingerprint",
url="https://example.com",
firing=False,
labels={},
annotations={"description": "dummy description"},
)
self.fake_matrix_client_pool.matrix_client = None
with self.assertRaises(MatrixClientError):
await create_alert(fake_alert, self.fake_room_id, self.fake_request)
self.fake_alertmanager_client.update_silence.assert_not_called()
self.fake_alert_renderer.render.assert_has_calls(
[call(fake_alert, html=False), call(fake_alert, html=True)]
)
fake_send_text_to_room.assert_not_called()
async def test_health(self) -> None:
async with self.client.request("GET", "/health") as response:
self.assertEqual(200, response.status)
async def test_metrics(self) -> None:
async with self.client.request("GET", "/metrics") as response:
self.assertEqual(200, response.status)
class WebhookServerTestCase(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None:
self.fake_matrix_client = Mock(spec=nio.AsyncClient)
self.fake_alertmanager_client = Mock(spec=AlertmanagerClient)
self.fake_cache = Mock(spec=Cache)
self.fake_config = Mock(spec=Config)
self.fake_config.port = aiohttp.test_utils.unused_port()
self.fake_config.address = "localhost"
self.fake_config.socket = "webhook.sock"
self.fake_config.cache_expire_time = 0
self.fake_config.template_dir = None
@patch.object(matrix_alertbot.webhook.web, "TCPSite", autospec=True)
async def test_webhook_start_address_port(self, fake_tcp_site: Mock) -> None:
webhook = Webhook(
self.fake_matrix_client,
self.fake_alertmanager_client,
self.fake_cache,
self.fake_config,
)
await webhook.start()
fake_tcp_site.assert_called_once_with(
webhook.runner, self.fake_config.address, self.fake_config.port
)
await webhook.close()
@patch.object(matrix_alertbot.webhook.web, "UnixSite", autospec=True)
async def test_webhook_start_unix_socket(self, fake_unix_site: Mock) -> None:
self.fake_config.address = None
self.fake_config.port = None
webhook = Webhook(
self.fake_matrix_client,
self.fake_alertmanager_client,
self.fake_cache,
self.fake_config,
)
await webhook.start()
fake_unix_site.assert_called_once_with(webhook.runner, self.fake_config.socket)
await webhook.close()