2022-07-04 01:03:24 +02:00
|
|
|
import logging
|
2022-07-16 23:08:12 +02:00
|
|
|
from typing import List, Optional
|
2022-07-04 01:03:24 +02:00
|
|
|
|
2022-07-12 00:27:17 +02:00
|
|
|
import pytimeparse2
|
2022-07-06 00:54:13 +02:00
|
|
|
from diskcache import Cache
|
2022-07-10 14:06:36 +02:00
|
|
|
from nio import AsyncClient, MatrixRoom
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2022-07-04 01:03:24 +02:00
|
|
|
from matrix_alertbot.alertmanager import AlertmanagerClient
|
2022-07-10 14:06:36 +02:00
|
|
|
from matrix_alertbot.chat_functions import send_text_to_room
|
2022-06-13 20:55:01 +02:00
|
|
|
from matrix_alertbot.config import Config
|
2022-07-11 23:33:35 +02:00
|
|
|
from matrix_alertbot.errors import (
|
|
|
|
AlertmanagerError,
|
2022-07-12 00:27:17 +02:00
|
|
|
AlertNotFoundError,
|
2022-07-11 23:33:35 +02:00
|
|
|
SilenceNotFoundError,
|
|
|
|
)
|
2022-07-10 12:51:49 +02:00
|
|
|
from matrix_alertbot.matcher import AlertMatcher, AlertRegexMatcher
|
2022-07-04 01:03:24 +02:00
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
2021-01-10 04:30:07 +01:00
|
|
|
|
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
class BaseCommand:
|
2021-01-10 04:30:07 +01:00
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
client: AsyncClient,
|
2022-07-04 01:03:24 +02:00
|
|
|
cache: Cache,
|
|
|
|
alertmanager: AlertmanagerClient,
|
2021-01-10 04:30:07 +01:00
|
|
|
config: Config,
|
2022-07-16 23:08:12 +02:00
|
|
|
cmd: str,
|
2021-01-10 04:30:07 +01:00
|
|
|
room: MatrixRoom,
|
2022-07-10 14:06:36 +02:00
|
|
|
sender: str,
|
|
|
|
event_id: str,
|
2022-07-16 23:08:12 +02:00
|
|
|
) -> None:
|
2021-01-10 04:33:59 +01:00
|
|
|
"""A command made by a user.
|
2019-09-25 14:26:29 +02:00
|
|
|
|
|
|
|
Args:
|
2022-07-16 23:08:12 +02:00
|
|
|
client: The client to communicate with Matrix.
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2022-07-04 01:03:24 +02:00
|
|
|
cache: Bot cache.
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
alertmanager: The client to communicate with Alertmanager.
|
|
|
|
|
2021-01-10 04:33:59 +01:00
|
|
|
config: Bot configuration parameters.
|
2019-10-04 15:44:19 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
cmd: The command and arguments.
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2021-01-10 04:33:59 +01:00
|
|
|
room: The room the command was sent in.
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
sender: The sender of the event
|
|
|
|
|
|
|
|
event_id: The ID of the event describing the command.
|
2019-09-25 14:26:29 +02:00
|
|
|
"""
|
|
|
|
self.client = client
|
2022-07-04 01:03:24 +02:00
|
|
|
self.cache = cache
|
|
|
|
self.alertmanager = alertmanager
|
2019-10-26 01:40:05 +02:00
|
|
|
self.config = config
|
2022-07-16 23:08:12 +02:00
|
|
|
self.cmd = cmd
|
|
|
|
self.args = cmd.split()[1:]
|
2019-09-25 14:26:29 +02:00
|
|
|
self.room = room
|
2022-07-10 14:06:36 +02:00
|
|
|
self.sender = sender
|
|
|
|
self.event_id = event_id
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2022-06-14 23:37:54 +02:00
|
|
|
async def process(self) -> None:
|
2022-07-16 23:08:12 +02:00
|
|
|
raise NotImplementedError
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
|
|
|
|
class BaseAlertCommand(BaseCommand):
|
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
client: AsyncClient,
|
|
|
|
cache: Cache,
|
|
|
|
alertmanager: AlertmanagerClient,
|
|
|
|
config: Config,
|
|
|
|
cmd: str,
|
|
|
|
room: MatrixRoom,
|
|
|
|
sender: str,
|
|
|
|
event_id: str,
|
|
|
|
alert_event_id: str,
|
|
|
|
) -> None:
|
|
|
|
super().__init__(
|
|
|
|
client, cache, alertmanager, config, cmd, room, sender, event_id
|
|
|
|
)
|
|
|
|
|
|
|
|
self.alert_event_id = alert_event_id
|
|
|
|
|
|
|
|
|
|
|
|
class AckAlertCommand(BaseAlertCommand):
|
|
|
|
async def process(self) -> None:
|
2022-07-04 01:03:24 +02:00
|
|
|
"""Acknowledge an alert and silence it for a certain duration in Alertmanager"""
|
2022-07-10 12:51:49 +02:00
|
|
|
matchers: List[AlertMatcher] = []
|
2022-07-10 02:40:04 +02:00
|
|
|
durations = []
|
|
|
|
for arg in self.args:
|
|
|
|
if "=~" in arg:
|
|
|
|
label, regex = arg.split("=~")
|
|
|
|
regex_matcher = AlertRegexMatcher(label, regex)
|
|
|
|
matchers.append(regex_matcher)
|
|
|
|
elif "=" in arg:
|
|
|
|
label, value = arg.split("=")
|
|
|
|
matcher = AlertMatcher(label, value)
|
|
|
|
matchers.append(matcher)
|
|
|
|
else:
|
|
|
|
durations.append(arg)
|
|
|
|
|
|
|
|
if len(durations) > 0:
|
|
|
|
duration = " ".join(durations)
|
2022-07-04 01:03:24 +02:00
|
|
|
else:
|
|
|
|
duration = "1d"
|
2022-07-10 02:40:04 +02:00
|
|
|
|
2022-07-04 01:03:24 +02:00
|
|
|
logger.debug(
|
2022-07-10 14:06:36 +02:00
|
|
|
f"Receiving a command to create a silence for a duration of {duration}"
|
2022-07-04 01:03:24 +02:00
|
|
|
)
|
|
|
|
|
2022-07-12 00:27:17 +02:00
|
|
|
duration_seconds = pytimeparse2.parse(duration)
|
|
|
|
if duration_seconds is None:
|
|
|
|
logger.error(f"Unable to create silence: Invalid duration '{duration}'")
|
|
|
|
await send_text_to_room(
|
|
|
|
self.client,
|
|
|
|
self.room.room_id,
|
2022-07-12 00:29:46 +02:00
|
|
|
f"I tried really hard, but I can't convert the duration '{duration}' to a number of seconds.",
|
2022-07-12 00:27:17 +02:00
|
|
|
)
|
|
|
|
return
|
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
logger.debug(
|
|
|
|
f"Read alert fingerprints for alert event {self.alert_event_id} from cache"
|
|
|
|
)
|
2022-07-05 23:35:19 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
if self.alert_event_id not in self.cache:
|
|
|
|
logger.error(
|
|
|
|
f"Cannot find fingerprints for alert event {self.alert_event_id} in cache"
|
|
|
|
)
|
2022-07-11 23:18:57 +02:00
|
|
|
return
|
2022-07-10 15:11:25 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
alert_fingerprints = self.cache[self.alert_event_id]
|
2022-07-10 15:11:25 +02:00
|
|
|
logger.debug(f"Found {len(alert_fingerprints)} in cache")
|
|
|
|
|
2022-07-11 23:33:35 +02:00
|
|
|
count_alert_not_found = 0
|
2022-07-16 23:08:12 +02:00
|
|
|
created_silences = []
|
2022-07-05 23:35:19 +02:00
|
|
|
for alert_fingerprint in alert_fingerprints:
|
|
|
|
logger.debug(
|
|
|
|
f"Create silence for alert with fingerprint {alert_fingerprint} for a duration of {duration}"
|
|
|
|
)
|
2022-07-12 00:27:17 +02:00
|
|
|
|
2022-07-06 00:54:13 +02:00
|
|
|
try:
|
2022-07-16 23:08:12 +02:00
|
|
|
silence_id = await self.alertmanager.create_silence(
|
2022-07-10 02:40:04 +02:00
|
|
|
alert_fingerprint,
|
2022-07-12 00:27:17 +02:00
|
|
|
duration_seconds,
|
2022-07-10 14:06:36 +02:00
|
|
|
self.room.user_name(self.sender),
|
2022-07-10 02:40:04 +02:00
|
|
|
matchers,
|
2022-07-06 00:54:13 +02:00
|
|
|
)
|
2022-07-16 23:08:12 +02:00
|
|
|
created_silences.append(silence_id)
|
2022-07-11 23:33:35 +02:00
|
|
|
except AlertNotFoundError as e:
|
|
|
|
logger.warning(f"Unable to create silence: {e}")
|
|
|
|
count_alert_not_found += 1
|
2022-07-09 10:38:40 +02:00
|
|
|
except AlertmanagerError as e:
|
2022-07-08 23:22:31 +02:00
|
|
|
logger.exception(f"Unable to create silence: {e}", exc_info=e)
|
2022-07-06 00:54:13 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
self.cache.set(self.event_id, tuple(created_silences), expire=duration_seconds)
|
|
|
|
|
2022-07-11 23:33:35 +02:00
|
|
|
if count_alert_not_found > 0:
|
|
|
|
await send_text_to_room(
|
|
|
|
self.client,
|
|
|
|
self.room.room_id,
|
2022-07-11 23:45:15 +02:00
|
|
|
f"Sorry, I couldn't find {count_alert_not_found} alerts, therefore I couldn't create their silence.",
|
2022-07-11 23:33:35 +02:00
|
|
|
)
|
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
if len(created_silences) > 0:
|
2022-07-11 23:43:27 +02:00
|
|
|
await send_text_to_room(
|
|
|
|
self.client,
|
|
|
|
self.room.room_id,
|
2022-07-16 23:08:12 +02:00
|
|
|
f"Created {len(created_silences)} silences with a duration of {duration}.",
|
2022-07-11 23:43:27 +02:00
|
|
|
)
|
2022-07-06 00:54:13 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
|
|
|
|
class UnackAlertCommand(BaseAlertCommand):
|
|
|
|
async def process(self) -> None:
|
2022-07-06 00:54:13 +02:00
|
|
|
"""Delete an alert's acknowledgement of an alert and remove corresponding silence in Alertmanager"""
|
2022-07-10 12:51:49 +02:00
|
|
|
matchers: List[AlertMatcher] = []
|
|
|
|
for arg in self.args:
|
|
|
|
if "=~" in arg:
|
|
|
|
label, regex = arg.split("=~")
|
|
|
|
regex_matcher = AlertRegexMatcher(label, regex)
|
|
|
|
matchers.append(regex_matcher)
|
|
|
|
elif "=" in arg:
|
|
|
|
label, value = arg.split("=")
|
|
|
|
matcher = AlertMatcher(label, value)
|
|
|
|
matchers.append(matcher)
|
|
|
|
|
2022-07-10 14:06:36 +02:00
|
|
|
logger.debug("Receiving a command to delete a silence")
|
2022-07-16 23:08:12 +02:00
|
|
|
logger.debug(
|
|
|
|
f"Read alert fingerprints for alert event {self.alert_event_id} from cache"
|
|
|
|
)
|
2022-07-06 00:54:13 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
if self.alert_event_id not in self.cache:
|
|
|
|
logger.error(
|
|
|
|
f"Cannot find fingerprints for event {self.alert_event_id} in cache"
|
|
|
|
)
|
2022-07-11 23:18:57 +02:00
|
|
|
return
|
2022-07-10 15:11:25 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
alert_fingerprints = self.cache[self.alert_event_id]
|
2022-07-10 15:11:25 +02:00
|
|
|
logger.debug(f"Found {len(alert_fingerprints)} in cache")
|
|
|
|
|
2022-07-11 23:33:35 +02:00
|
|
|
count_alert_not_found = 0
|
2022-07-10 15:11:25 +02:00
|
|
|
count_removed_silences = 0
|
2022-07-06 00:54:13 +02:00
|
|
|
for alert_fingerprint in alert_fingerprints:
|
|
|
|
logger.debug(
|
|
|
|
f"Delete silence for alert with fingerprint {alert_fingerprint}"
|
2022-07-05 23:35:19 +02:00
|
|
|
)
|
2022-07-06 00:54:13 +02:00
|
|
|
try:
|
2022-07-09 09:56:28 +02:00
|
|
|
removed_silences = await self.alertmanager.delete_silences(
|
2022-07-10 12:51:49 +02:00
|
|
|
alert_fingerprint, matchers
|
2022-07-09 00:08:51 +02:00
|
|
|
)
|
2022-07-09 09:56:28 +02:00
|
|
|
count_removed_silences += len(removed_silences)
|
2022-07-11 23:33:35 +02:00
|
|
|
except (AlertNotFoundError, SilenceNotFoundError) as e:
|
2022-07-11 23:41:30 +02:00
|
|
|
logger.error(f"Unable to delete silence: {e}")
|
2022-07-11 23:33:35 +02:00
|
|
|
count_alert_not_found += 1
|
2022-07-09 10:38:40 +02:00
|
|
|
except AlertmanagerError as e:
|
2022-07-08 23:22:31 +02:00
|
|
|
logger.exception(f"Unable to delete silence: {e}", exc_info=e)
|
2022-07-06 00:54:13 +02:00
|
|
|
|
2022-07-11 23:33:35 +02:00
|
|
|
if count_alert_not_found > 0:
|
|
|
|
await send_text_to_room(
|
|
|
|
self.client,
|
|
|
|
self.room.room_id,
|
2022-07-11 23:45:15 +02:00
|
|
|
f"Sorry, I couldn't find {count_alert_not_found} alerts, therefore I couldn't remove their silences.",
|
2022-07-11 23:33:35 +02:00
|
|
|
)
|
|
|
|
|
2022-07-11 23:43:27 +02:00
|
|
|
if count_removed_silences > 0:
|
|
|
|
await send_text_to_room(
|
|
|
|
self.client,
|
|
|
|
self.room.room_id,
|
|
|
|
f"Removed {count_removed_silences} silences.",
|
|
|
|
)
|
2019-09-25 14:26:29 +02:00
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
|
|
|
|
class HelpCommand(BaseCommand):
|
|
|
|
async def process(self) -> None:
|
2019-09-25 14:26:29 +02:00
|
|
|
"""Show the help text"""
|
2022-07-10 14:06:36 +02:00
|
|
|
logger.debug(f"Displaying help to room {self.room.display_name}")
|
2019-09-25 14:26:29 +02:00
|
|
|
if not self.args:
|
2020-08-10 00:02:07 +02:00
|
|
|
text = (
|
|
|
|
"Hello, I am a bot made with matrix-nio! Use `help commands` to view "
|
|
|
|
"available commands."
|
|
|
|
)
|
2019-09-25 14:26:29 +02:00
|
|
|
await send_text_to_room(self.client, self.room.room_id, text)
|
|
|
|
return
|
|
|
|
|
|
|
|
topic = self.args[0]
|
|
|
|
if topic == "rules":
|
|
|
|
text = "These are the rules!"
|
|
|
|
elif topic == "commands":
|
2021-01-10 04:33:59 +01:00
|
|
|
text = "Available commands: ..."
|
2019-09-25 14:26:29 +02:00
|
|
|
else:
|
|
|
|
text = "Unknown help topic!"
|
|
|
|
await send_text_to_room(self.client, self.room.room_id, text)
|
|
|
|
|
2022-07-16 23:08:12 +02:00
|
|
|
|
|
|
|
class UnknownCommand(BaseCommand):
|
|
|
|
async def process(self) -> None:
|
2022-07-04 01:03:24 +02:00
|
|
|
logger.debug(
|
2022-07-10 14:06:36 +02:00
|
|
|
f"Sending unknown command response to room {self.room.display_name}"
|
2022-07-04 01:03:24 +02:00
|
|
|
)
|
2019-09-25 14:26:29 +02:00
|
|
|
await send_text_to_room(
|
|
|
|
self.client,
|
|
|
|
self.room.room_id,
|
2022-07-16 23:08:12 +02:00
|
|
|
f"Unknown command '{self.cmd}'. Try the 'help' command for more information.",
|
2019-09-25 14:26:29 +02:00
|
|
|
)
|
2022-07-16 23:08:12 +02:00
|
|
|
|
|
|
|
|
|
|
|
class CommandFactory:
|
|
|
|
@staticmethod
|
|
|
|
def create(
|
|
|
|
cmd: str,
|
|
|
|
client: AsyncClient,
|
|
|
|
cache: Cache,
|
|
|
|
alertmanager: AlertmanagerClient,
|
|
|
|
config: Config,
|
|
|
|
room: MatrixRoom,
|
|
|
|
sender: str,
|
|
|
|
event_id: str,
|
|
|
|
reacted_to_event_id: Optional[str] = None,
|
|
|
|
) -> BaseCommand:
|
|
|
|
if cmd.startswith("ack"):
|
|
|
|
assert reacted_to_event_id is not None
|
|
|
|
return AckAlertCommand(
|
|
|
|
client,
|
|
|
|
cache,
|
|
|
|
alertmanager,
|
|
|
|
config,
|
|
|
|
cmd,
|
|
|
|
room,
|
|
|
|
sender,
|
|
|
|
event_id,
|
|
|
|
reacted_to_event_id,
|
|
|
|
)
|
|
|
|
elif cmd.startswith("unack") or cmd.startswith("nack"):
|
|
|
|
assert reacted_to_event_id is not None
|
|
|
|
return UnackAlertCommand(
|
|
|
|
client,
|
|
|
|
cache,
|
|
|
|
alertmanager,
|
|
|
|
config,
|
|
|
|
cmd,
|
|
|
|
room,
|
|
|
|
sender,
|
|
|
|
event_id,
|
|
|
|
reacted_to_event_id,
|
|
|
|
)
|
|
|
|
elif cmd.startswith("help"):
|
|
|
|
return HelpCommand(
|
|
|
|
client, cache, alertmanager, config, cmd, room, sender, event_id
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
return UnknownCommand(
|
|
|
|
client, cache, alertmanager, config, cmd, room, sender, event_id
|
|
|
|
)
|