matrix-alertbot/matrix_alertbot/alertmanager.py

135 lines
4.4 KiB
Python
Raw Normal View History

2022-07-08 22:37:09 +02:00
from __future__ import annotations
2022-07-10 02:40:04 +02:00
from datetime import datetime, timedelta
2022-07-09 15:25:16 +02:00
from typing import Dict, List
import aiohttp
2022-07-10 02:40:04 +02:00
import pytimeparse2
from aiohttp import ClientError
from aiohttp_prometheus_exporter.trace import PrometheusTraceConfig
from diskcache import Cache
2022-07-06 00:54:13 +02:00
from matrix_alertbot.errors import (
2022-07-09 10:38:40 +02:00
AlertmanagerServerError,
2022-07-10 02:40:04 +02:00
AlertMismatchError,
2022-07-06 00:54:13 +02:00
AlertNotFoundError,
SilenceNotFoundError,
)
2022-07-10 12:51:49 +02:00
from matrix_alertbot.matcher import AlertMatcher
2022-07-08 22:46:04 +02:00
class AlertmanagerClient:
def __init__(self, url: str, cache: Cache) -> None:
self.api_url = f"{url}/api/v2"
self.cache = cache
self.session = aiohttp.ClientSession(trace_configs=[PrometheusTraceConfig()])
async def close(self) -> None:
await self.session.close()
async def get_alerts(self) -> List[Dict]:
2022-07-06 00:54:13 +02:00
try:
2022-07-08 23:23:38 +02:00
async with self.session.get(f"{self.api_url}/alerts") as response:
response.raise_for_status()
return await response.json()
except ClientError as e:
2022-07-09 10:38:40 +02:00
raise AlertmanagerServerError(
2022-07-09 15:25:16 +02:00
"Cannot fetch alerts from Alertmanager"
2022-07-09 10:38:40 +02:00
) from e
async def get_alert(self, fingerprint: str) -> Dict:
alerts = await self.get_alerts()
2022-07-06 00:54:13 +02:00
return self._find_alert(fingerprint, alerts)
2022-07-10 02:40:04 +02:00
async def create_silence(
self,
fingerprint: str,
duration: str,
user: str,
2022-07-10 12:51:49 +02:00
matchers: List[AlertMatcher],
2022-07-10 02:40:04 +02:00
) -> str:
alert = await self.get_alert(fingerprint)
2022-07-10 02:40:04 +02:00
self._match_alert(alert, matchers)
2022-07-10 03:03:08 +02:00
silence_matchers = [
{"name": label, "value": value, "isRegex": False, "isEqual": True}
for label, value in alert["labels"].items()
]
2022-07-10 02:40:04 +02:00
start_time = datetime.now()
duration_seconds = pytimeparse2.parse(duration)
duration_delta = timedelta(seconds=duration_seconds)
end_time = start_time + duration_delta
silence = {
2022-07-10 03:03:08 +02:00
"matchers": silence_matchers,
2022-07-05 23:35:19 +02:00
"startsAt": start_time.isoformat(),
"endsAt": end_time.isoformat(),
"createdBy": user,
"comment": "Acknowledge alert from Matrix",
}
2022-07-06 00:54:13 +02:00
try:
async with self.session.post(
f"{self.api_url}/silences", json=silence
) as response:
response.raise_for_status()
data = await response.json()
except ClientError as e:
2022-07-09 10:38:40 +02:00
raise AlertmanagerServerError(
2022-07-06 00:54:13 +02:00
f"Cannot create silence for alert fingerprint {fingerprint}"
) from e
return data["silenceID"]
2022-07-10 12:51:49 +02:00
async def delete_silences(
self, fingerprint: str, matchers: List[AlertMatcher]
) -> List[str]:
alert = await self.get_alert(fingerprint)
2022-07-06 00:54:13 +02:00
alert_state = alert["status"]["state"]
if alert_state != "suppressed":
raise SilenceNotFoundError(
f"Cannot find silences for alert fingerprint {fingerprint} in state {alert_state}"
)
2022-07-10 12:51:49 +02:00
self._match_alert(alert, matchers)
2022-07-06 00:54:13 +02:00
silences = alert["status"]["silencedBy"]
for silence in silences:
2022-07-09 09:56:28 +02:00
await self._delete_silence(silence)
2022-07-09 00:08:51 +02:00
return silences
2022-07-09 09:56:28 +02:00
async def _delete_silence(self, silence: str) -> None:
try:
async with self.session.delete(
f"{self.api_url}/silence/{silence}"
) as response:
2022-07-06 00:54:13 +02:00
response.raise_for_status()
except ClientError as e:
2022-07-09 10:38:40 +02:00
raise AlertmanagerServerError(
f"Cannot delete silence with ID {silence}"
) from e
2022-07-06 00:54:13 +02:00
@staticmethod
2022-07-06 00:54:13 +02:00
def _find_alert(fingerprint: str, alerts: List[Dict]) -> Dict:
for alert in alerts:
if alert["fingerprint"] == fingerprint:
return alert
raise AlertNotFoundError(f"Cannot find alert with fingerprint {fingerprint}")
2022-07-10 02:40:04 +02:00
@staticmethod
2022-07-10 12:51:49 +02:00
def _match_alert(alert: Dict, matchers: List[AlertMatcher]) -> None:
2022-07-10 02:40:04 +02:00
labels = alert["labels"]
for matcher in matchers:
if matcher.label not in labels:
labels_text = ", ".join(labels)
raise AlertMismatchError(
f"Cannot find label {matcher.label} in alert labels: {labels_text}"
)
if not matcher.match(labels):
raise AlertMismatchError(
2022-07-10 03:07:31 +02:00
f"Alert with label {matcher.label}={labels[matcher.label]} does not match {matcher}"
2022-07-10 02:40:04 +02:00
)