From a0f042cf2316907b921262fdc8d351342c8ec79a Mon Sep 17 00:00:00 2001 From: lens0021 Date: Sun, 4 Oct 2026 09:50:46 +0900 Subject: [PATCH 1/2] feat(grafana-github): put the start time in KST in new alert issue titles An issue is titled " (10/3 21:55 KST)" from the earliest firing alert, set once at creation. Open issues still match by rule, with or without the time, so the open old-style issue keeps its comments. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hv8vVU7X8zGPFbo3CvktmJ --- README.md | 3 ++- grafana-github/lambda_function.py | 35 ++++++++++++++++++++---- grafana-github/test_lambda_function.py | 37 +++++++++++++++++++++++--- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9c5eda2..fc09d25 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ repository's secrets: Keeps one issue in femiwiki/infra per Grafana alert rule labelled `impact=operators`. Grafana's webhook contact point calls the function URL from femiwiki/infra `aws/lambda.tf`; the first firing opens an issue named after the -rule, and later notifications become comments. `deploy.yml` sets: +rule and its start time in KST, such as `Disk almost full (10/3 21:55 KST)`, and +later notifications become comments while it is open. `deploy.yml` sets: - `GITHUB_APP_CLIENT_ID`: the alerts GitHub App, installed on femiwiki/infra with Issues read and write diff --git a/grafana-github/lambda_function.py b/grafana-github/lambda_function.py index 5df985b..ba6bc25 100644 --- a/grafana-github/lambda_function.py +++ b/grafana-github/lambda_function.py @@ -1,8 +1,10 @@ """Keep one GitHub issue per Grafana alert rule that only operators need to act on. Grafana's webhook contact point posts its default payload to this function's -URL. The first firing opens an issue titled with the rule's name; later -notifications for the same rule, repeats and resolves alike, become comments. +URL. The first firing opens an issue titled with the rule's name and the time it +started in KST, so issues for the same rule can be told apart; later +notifications for the same rule, repeats and resolves alike, become comments +while that issue is open. A new issue mentions GITHUB_MENTION, a team, so its members get a push. The pull request that fixes the cause closes the issue. """ @@ -11,11 +13,14 @@ import hmac import json import os +from datetime import UTC, datetime, timedelta, timezone from typing import Any from github import Auth, GithubIntegration from github.Repository import Repository +KST = timezone(timedelta(hours=9), "KST") + def lambda_handler(event: dict[str, Any], context: Any) -> dict[str, int]: headers = {k.lower(): v for k, v in (event.get("headers") or {}).items()} @@ -52,17 +57,37 @@ def authorized(header: str, token: str) -> bool: def record( repo: Repository, bot: str, payload: dict[str, Any], mention: str = "" ) -> None: - title = ( + rule = ( payload.get("groupLabels", {}).get("alertname") or payload["commonLabels"]["alertname"] ) text = render(payload) for issue in repo.get_issues(state="open"): - if issue.title == title and issue.user.login == bot and not issue.pull_request: + if ( + is_for(issue.title, rule) + and issue.user.login == bot + and not issue.pull_request + ): issue.create_comment(text) return if payload["status"] == "firing": - repo.create_issue(title=title, body=f"{text}\n\n{mention}".strip()) + repo.create_issue( + title=title(rule, payload), body=f"{text}\n\n{mention}".strip() + ) + + +def is_for(title: str, rule: str) -> bool: + """Whether an issue title is the rule's, with or without the start time.""" + return title == rule or title.startswith(f"{rule} (") + + +def title(rule: str, payload: dict[str, Any]) -> str: + """The rule's name and when its earliest firing alert started, in KST.""" + starts = min( + alert["startsAt"] for alert in payload["alerts"] if alert["status"] == "firing" + ) + start = datetime.fromisoformat(starts[:16]).replace(tzinfo=UTC).astimezone(KST) + return f"{rule} ({start.month}/{start.day} {start:%H:%M} KST)" def render(payload: dict[str, Any]) -> str: diff --git a/grafana-github/test_lambda_function.py b/grafana-github/test_lambda_function.py index 91c8fa4..2d89bab 100644 --- a/grafana-github/test_lambda_function.py +++ b/grafana-github/test_lambda_function.py @@ -4,17 +4,19 @@ from unittest.mock import MagicMock, patch import lambda_function -from lambda_function import authorized, lambda_handler, record, render +from lambda_function import authorized, lambda_handler, record, render, title BOT = "femiwiki-alerts[bot]" -def alert(status: str = "firing", **annotations: str) -> dict: +def alert( + status: str = "firing", starts: str = "2026-10-01T12:00:00.123Z", **annotations: str +) -> dict: return { "status": status, "labels": {"alertname": "Disk almost full"}, "annotations": {"summary": "/ is 92% full", **annotations}, - "startsAt": "2026-10-01T12:00:00.123Z", + "startsAt": starts, "endsAt": "2026-10-01T13:30:00Z" if status == "resolved" else "0001-01-01T00:00:00Z", @@ -59,7 +61,7 @@ def test_opens_an_issue_on_first_firing(self): record(r, BOT, payload()) r.get_issues.assert_called_once_with(state="open") kwargs = r.create_issue.call_args.kwargs - self.assertEqual(kwargs["title"], "Disk almost full") + self.assertEqual(kwargs["title"], "Disk almost full (10/1 21:00 KST)") self.assertIn("/ is 92% full", kwargs["body"]) def test_mentions_the_team_only_when_opening(self): @@ -79,6 +81,20 @@ def test_comments_on_the_open_issue_for_the_same_rule(self): existing.create_comment.assert_called_once() r.create_issue.assert_not_called() + def test_comments_on_an_open_issue_titled_with_a_start_time(self): + existing = issue("Disk almost full (9/30 08:15 KST)") + r = repo([existing]) + record(r, BOT, payload()) + existing.create_comment.assert_called_once() + r.create_issue.assert_not_called() + + def test_ignores_a_rule_whose_name_extends_this_one(self): + other = issue("Disk almost full on /data (9/30 08:15 KST)") + r = repo([other]) + record(r, BOT, payload()) + other.create_comment.assert_not_called() + r.create_issue.assert_called_once() + def test_ignores_pull_requests_other_titles_and_people(self): pr = issue("Disk almost full", pull_request=object()) other = issue("Out of memory") @@ -101,6 +117,19 @@ def test_resolve_without_an_open_issue_does_nothing(self): r.create_issue.assert_not_called() +class TestTitle(unittest.TestCase): + def test_uses_the_earliest_firing_start_in_kst(self): + p = payload() + p["alerts"] = [ + alert(starts="2026-10-01T18:00:00Z"), + alert(starts="2026-10-01T16:30:00.123456789Z"), + alert("resolved", starts="2026-10-01T10:00:00Z"), + ] + self.assertEqual( + title("Disk almost full", p), "Disk almost full (10/2 01:30 KST)" + ) + + class TestRender(unittest.TestCase): def test_lists_start_rule_logs_and_id(self): p = payload() From a78bf968b3ba7aea1132363e31ea9f5c3b379a1c Mon Sep 17 00:00:00 2001 From: lens0021 Date: Sun, 4 Oct 2026 09:51:01 +0900 Subject: [PATCH 2/2] feat(grafana-github): label new alert issues alert Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hv8vVU7X8zGPFbo3CvktmJ --- README.md | 3 ++- grafana-github/lambda_function.py | 8 ++++++-- grafana-github/test_lambda_function.py | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fc09d25..7e7b0fa 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,8 @@ Keeps one issue in femiwiki/infra per Grafana alert rule labelled `impact=operators`. Grafana's webhook contact point calls the function URL from femiwiki/infra `aws/lambda.tf`; the first firing opens an issue named after the rule and its start time in KST, such as `Disk almost full (10/3 21:55 KST)`, and -later notifications become comments while it is open. `deploy.yml` sets: +later notifications become comments while it is open. New issues get the +`alert` label. `deploy.yml` sets: - `GITHUB_APP_CLIENT_ID`: the alerts GitHub App, installed on femiwiki/infra with Issues read and write diff --git a/grafana-github/lambda_function.py b/grafana-github/lambda_function.py index ba6bc25..082a6f1 100644 --- a/grafana-github/lambda_function.py +++ b/grafana-github/lambda_function.py @@ -5,7 +5,8 @@ started in KST, so issues for the same rule can be told apart; later notifications for the same rule, repeats and resolves alike, become comments while that issue is open. -A new issue mentions GITHUB_MENTION, a team, so its members get a push. +A new issue carries the label LABEL and mentions GITHUB_MENTION, a team, so its +members get a push. The pull request that fixes the cause closes the issue. """ @@ -20,6 +21,7 @@ from github.Repository import Repository KST = timezone(timedelta(hours=9), "KST") +LABEL = "alert" def lambda_handler(event: dict[str, Any], context: Any) -> dict[str, int]: @@ -72,7 +74,9 @@ def record( return if payload["status"] == "firing": repo.create_issue( - title=title(rule, payload), body=f"{text}\n\n{mention}".strip() + title=title(rule, payload), + body=f"{text}\n\n{mention}".strip(), + labels=[LABEL], ) diff --git a/grafana-github/test_lambda_function.py b/grafana-github/test_lambda_function.py index 2d89bab..ce47bad 100644 --- a/grafana-github/test_lambda_function.py +++ b/grafana-github/test_lambda_function.py @@ -63,6 +63,7 @@ def test_opens_an_issue_on_first_firing(self): kwargs = r.create_issue.call_args.kwargs self.assertEqual(kwargs["title"], "Disk almost full (10/1 21:00 KST)") self.assertIn("/ is 92% full", kwargs["body"]) + self.assertEqual(kwargs["labels"], ["alert"]) def test_mentions_the_team_only_when_opening(self): r = repo([])