diff --git a/README.md b/README.md index 9c5eda2..7e7b0fa 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,9 @@ repository's secrets: Keeps one issue in femiwiki/infra per Grafana alert rule labelled `impact=operators`. Grafana's webhook contact point calls the function URL from femiwiki/infra `aws/lambda.tf`; the first firing opens an issue named after the -rule, and later notifications become comments. `deploy.yml` sets: +rule and its start time in KST, such as `Disk almost full (10/3 21:55 KST)`, and +later notifications become comments while it is open. New issues get the +`alert` label. `deploy.yml` sets: - `GITHUB_APP_CLIENT_ID`: the alerts GitHub App, installed on femiwiki/infra with Issues read and write diff --git a/grafana-github/lambda_function.py b/grafana-github/lambda_function.py index 5df985b..082a6f1 100644 --- a/grafana-github/lambda_function.py +++ b/grafana-github/lambda_function.py @@ -1,9 +1,12 @@ """Keep one GitHub issue per Grafana alert rule that only operators need to act on. Grafana's webhook contact point posts its default payload to this function's -URL. The first firing opens an issue titled with the rule's name; later -notifications for the same rule, repeats and resolves alike, become comments. -A new issue mentions GITHUB_MENTION, a team, so its members get a push. +URL. The first firing opens an issue titled with the rule's name and the time it +started in KST, so issues for the same rule can be told apart; later +notifications for the same rule, repeats and resolves alike, become comments +while that issue is open. +A new issue carries the label LABEL and mentions GITHUB_MENTION, a team, so its +members get a push. The pull request that fixes the cause closes the issue. """ @@ -11,11 +14,15 @@ import hmac import json import os +from datetime import UTC, datetime, timedelta, timezone from typing import Any from github import Auth, GithubIntegration from github.Repository import Repository +KST = timezone(timedelta(hours=9), "KST") +LABEL = "alert" + def lambda_handler(event: dict[str, Any], context: Any) -> dict[str, int]: headers = {k.lower(): v for k, v in (event.get("headers") or {}).items()} @@ -52,17 +59,39 @@ def authorized(header: str, token: str) -> bool: def record( repo: Repository, bot: str, payload: dict[str, Any], mention: str = "" ) -> None: - title = ( + rule = ( payload.get("groupLabels", {}).get("alertname") or payload["commonLabels"]["alertname"] ) text = render(payload) for issue in repo.get_issues(state="open"): - if issue.title == title and issue.user.login == bot and not issue.pull_request: + if ( + is_for(issue.title, rule) + and issue.user.login == bot + and not issue.pull_request + ): issue.create_comment(text) return if payload["status"] == "firing": - repo.create_issue(title=title, body=f"{text}\n\n{mention}".strip()) + repo.create_issue( + title=title(rule, payload), + body=f"{text}\n\n{mention}".strip(), + labels=[LABEL], + ) + + +def is_for(title: str, rule: str) -> bool: + """Whether an issue title is the rule's, with or without the start time.""" + return title == rule or title.startswith(f"{rule} (") + + +def title(rule: str, payload: dict[str, Any]) -> str: + """The rule's name and when its earliest firing alert started, in KST.""" + starts = min( + alert["startsAt"] for alert in payload["alerts"] if alert["status"] == "firing" + ) + start = datetime.fromisoformat(starts[:16]).replace(tzinfo=UTC).astimezone(KST) + return f"{rule} ({start.month}/{start.day} {start:%H:%M} KST)" def render(payload: dict[str, Any]) -> str: diff --git a/grafana-github/test_lambda_function.py b/grafana-github/test_lambda_function.py index 91c8fa4..ce47bad 100644 --- a/grafana-github/test_lambda_function.py +++ b/grafana-github/test_lambda_function.py @@ -4,17 +4,19 @@ from unittest.mock import MagicMock, patch import lambda_function -from lambda_function import authorized, lambda_handler, record, render +from lambda_function import authorized, lambda_handler, record, render, title BOT = "femiwiki-alerts[bot]" -def alert(status: str = "firing", **annotations: str) -> dict: +def alert( + status: str = "firing", starts: str = "2026-10-01T12:00:00.123Z", **annotations: str +) -> dict: return { "status": status, "labels": {"alertname": "Disk almost full"}, "annotations": {"summary": "/ is 92% full", **annotations}, - "startsAt": "2026-10-01T12:00:00.123Z", + "startsAt": starts, "endsAt": "2026-10-01T13:30:00Z" if status == "resolved" else "0001-01-01T00:00:00Z", @@ -59,8 +61,9 @@ def test_opens_an_issue_on_first_firing(self): record(r, BOT, payload()) r.get_issues.assert_called_once_with(state="open") kwargs = r.create_issue.call_args.kwargs - self.assertEqual(kwargs["title"], "Disk almost full") + self.assertEqual(kwargs["title"], "Disk almost full (10/1 21:00 KST)") self.assertIn("/ is 92% full", kwargs["body"]) + self.assertEqual(kwargs["labels"], ["alert"]) def test_mentions_the_team_only_when_opening(self): r = repo([]) @@ -79,6 +82,20 @@ def test_comments_on_the_open_issue_for_the_same_rule(self): existing.create_comment.assert_called_once() r.create_issue.assert_not_called() + def test_comments_on_an_open_issue_titled_with_a_start_time(self): + existing = issue("Disk almost full (9/30 08:15 KST)") + r = repo([existing]) + record(r, BOT, payload()) + existing.create_comment.assert_called_once() + r.create_issue.assert_not_called() + + def test_ignores_a_rule_whose_name_extends_this_one(self): + other = issue("Disk almost full on /data (9/30 08:15 KST)") + r = repo([other]) + record(r, BOT, payload()) + other.create_comment.assert_not_called() + r.create_issue.assert_called_once() + def test_ignores_pull_requests_other_titles_and_people(self): pr = issue("Disk almost full", pull_request=object()) other = issue("Out of memory") @@ -101,6 +118,19 @@ def test_resolve_without_an_open_issue_does_nothing(self): r.create_issue.assert_not_called() +class TestTitle(unittest.TestCase): + def test_uses_the_earliest_firing_start_in_kst(self): + p = payload() + p["alerts"] = [ + alert(starts="2026-10-01T18:00:00Z"), + alert(starts="2026-10-01T16:30:00.123456789Z"), + alert("resolved", starts="2026-10-01T10:00:00Z"), + ] + self.assertEqual( + title("Disk almost full", p), "Disk almost full (10/2 01:30 KST)" + ) + + class TestRender(unittest.TestCase): def test_lists_start_rule_logs_and_id(self): p = payload()