From 7e9cd6f72d9708198053fb7f51ff4cd2da0c9e54 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 2 Oct 2026 01:21:20 +0200 Subject: [PATCH 1/2] feat(mcp): preview_alert_rule dry-runs a rule before it is saved An agent writing a raw-SQL rule had no way to see what it would do short of creating it and reading checks later, which is how a rule that matches nothing ships looking quiet. preview_alert_rule replays a saved rule (rule_id, optionally with overrides) or a create_alert_rule definition over past data through AlertsService.previewRule: per window the value, sample count and verdict, skips with their reason, and when it would have fired. It warns when every window had no data or fell below the minimum sample count. Preview points now carry skipReason (skip_reason on the v2 API). create/update/get_alert_rule and list_alert_checks point at it. --- apps/ai/src/mcp/lib/alert-rules.ts | 5 + apps/ai/src/mcp/resources/instructions.ts | 2 +- .../mcp/tools/__tests__/alert-tools.test.ts | 75 ++++ apps/ai/src/mcp/tools/create-alert-rule.ts | 8 +- apps/ai/src/mcp/tools/get-alert-rule.ts | 5 + apps/ai/src/mcp/tools/list-alert-checks.ts | 4 +- apps/ai/src/mcp/tools/preview-alert-rule.ts | 346 ++++++++++++++++++ apps/ai/src/mcp/tools/registry.ts | 2 + apps/ai/src/mcp/tools/update-alert-rule.ts | 9 +- apps/api/src/routes/v2/alert-rules.http.ts | 1 + .../MapleAPI/Sources/MapleAPI/openapi.json | 1 + .../landing/src/content/docs/reference/mcp.md | 1 + .../components/ai-elements/tool-metadata.ts | 2 + apps/web/src/lib/alerts/form-utils.ts | 3 + .../src/services/alerts/AlertsService.ts | 4 +- packages/domain/src/http/alerts.ts | 2 + packages/domain/src/http/v2/alert-rules.ts | 6 + packages/domain/src/mcp-outputs/alerts.ts | 44 +++ packages/domain/src/mcp-outputs/catalog.ts | 2 + 19 files changed, 512 insertions(+), 10 deletions(-) create mode 100644 apps/ai/src/mcp/tools/preview-alert-rule.ts diff --git a/apps/ai/src/mcp/lib/alert-rules.ts b/apps/ai/src/mcp/lib/alert-rules.ts index c249864c01..f6b2d05712 100644 --- a/apps/ai/src/mcp/lib/alert-rules.ts +++ b/apps/ai/src/mcp/lib/alert-rules.ts @@ -143,6 +143,11 @@ export const renderRuleWrite = ( ], next: [ doc.next("get_alert_rule", { rule_id: rule.id }, "full configuration"), + doc.next( + "preview_alert_rule", + { rule_id: rule.id }, + "replay it over the last day to see what it would have done", + ), doc.next("list_alert_checks", { rule_id: rule.id }, "its evaluations once the scheduler picks it up"), ], }) diff --git a/apps/ai/src/mcp/resources/instructions.ts b/apps/ai/src/mcp/resources/instructions.ts index 360a8ff15d..0a4c97e235 100644 --- a/apps/ai/src/mcp/resources/instructions.ts +++ b/apps/ai/src/mcp/resources/instructions.ts @@ -105,7 +105,7 @@ first occurrences ──> (candidate) ──> issue in \`triage\` - Performance analysis: find_slow_traces -> inspect_trace -> get_service_top_operations - Trend analysis: query_data (timeseries or breakdown) - Service discovery: list_services -> diagnose_service -- Alert management: list_alert_rules / list_alert_destinations -> get_alert_rule -> create_alert_rule / update_alert_rule / delete_alert_rule -> list_alert_incidents +- Alert management: list_alert_rules / list_alert_destinations -> get_alert_rule -> preview_alert_rule (dry-run before saving; a window with no data is a blind rule, not a healthy one) -> create_alert_rule / update_alert_rule / delete_alert_rule -> list_alert_incidents - AI agent sessions (LLM agent traces — NOT browser replays): list_agent_sessions -> get_agent_session -> inspect_span (decodes an AI span's messages and tool calls) - Agent tool health: get_agent_tools_overview -> get_agent_tools_overview tool="" (its failure groups) -> get_agent_tool_error - Product analytics / conversion: list_product_events -> query_funnel (steps over page views, \`track()\` events and server events, stitched per person; \`breakdown_by\` a UTM/referrer dimension or an event attribute) -> add_dashboard_widget with \`panel_type: "funnel"\` and \`display_json.funnel.steps\` to pin it (\`display_json.funnel.variant: "dropoff"\` for the step-by-step drop-off view), or \`panel_type: "paths"\` with \`display_json.paths.anchor\` for what people do after / before one event diff --git a/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts b/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts index b6c55f62c2..442c0c00b0 100644 --- a/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts +++ b/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts @@ -10,6 +10,8 @@ import { AlertIncidentsListResponse, AlertRuleDocument, AlertRuleNotFoundError, + AlertRulePreviewRequest, + AlertRulePreviewResponse, AlertRulesListResponse, AlertRuleUpsertRequest, OrgId, @@ -22,6 +24,7 @@ import { ListAlertDestinationsOutput, ListAlertIncidentsOutput, ListAlertRulesOutput, + PreviewAlertRuleOutput, } from "@maple/domain/mcp-outputs" import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService" import { AlertsService } from "@maple/backend/services/alerts/AlertsService" @@ -99,6 +102,7 @@ const incident = (overrides: Record) => ({ interface Seen { created?: AlertRuleUpsertRequest updated?: AlertRuleUpsertRequest + previewed?: AlertRulePreviewRequest incidentOptions?: unknown checkOptions?: unknown } @@ -134,6 +138,36 @@ const layer = (seen: Seen) => ), } as never), Layer.succeed(AlertsService, { + previewRule: (_o: unknown, _r: unknown, request: AlertRulePreviewRequest) => { + seen.previewed = request + const point = (bucket: string) => ({ + bucket, + value: null, + sampleCount: 0, + status: "skipped", + skipReason: "no_data", + }) + return Effect.succeed( + Schema.decodeUnknownSync(AlertRulePreviewResponse)({ + bucketSeconds: 300, + windowMinutes: 5, + threshold: 0.05, + thresholdUpper: null, + comparator: "gt", + truncatedToStart: null, + series: [ + { + groupKey: "__total__", + points: [ + point("2026-09-24T09:50:00.000Z"), + point("2026-09-24T09:55:00.000Z"), + ], + }, + ], + wouldFire: [], + }), + ) + }, updateRule: ( _o: unknown, _u: unknown, @@ -457,6 +491,47 @@ describe("alert tools", () => { ).toBeUndefined() }) + it("preview_alert_rule replays a draft and flags a window range with no data", async () => { + const seen: Seen = {} + const result = await ok( + "preview_alert_rule", + { + signal_type: "raw_query", + comparator: "gt", + threshold: 0.05, + raw_query_sql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + start_time: "2026-09-24 09:00:00", + end_time: "2026-09-24 10:00:00", + }, + seen, + ) + expect(seen.previewed?.rule).toMatchObject({ + name: "Preview", + destinationIds: [], + signalType: "raw_query", + }) + expect(seen.previewed?.startTime).toBe("2026-09-24T09:00:00Z") + const output = Schema.decodeUnknownSync(PreviewAlertRuleOutput)(result.structuredContent) + expect(output.groups[0]).toMatchObject({ windows: 2, noData: 2, breached: 0 }) + expect(output.points[0]?.skipReason).toBe("no_data") + expect(output.warnings[0]).toContain("Every window had no data") + expect(text(result)).toContain("| no data |") + }) + + it("preview_alert_rule overlays overrides on a saved rule", async () => { + const seen: Seen = {} + await ok("preview_alert_rule", { rule_id: RULE_ID, threshold: 0.2 }, seen) + expect(seen.previewed?.rule).toMatchObject({ + name: "Checkout errors", + threshold: 0.2, + signalType: "error_rate", + }) + + const missing = await run("preview_alert_rule", { rule_id: "8f1c2d3e-4b5a-4c6d-8e9f-0a1b2c3d4e5f" }) + expect(missing._tag === "Failure" && missing.failure.parameter).toBe("rule_id") + }) + it("update_alert_rule overlays only the given fields", async () => { const seen: Seen = {} await ok( diff --git a/apps/ai/src/mcp/tools/create-alert-rule.ts b/apps/ai/src/mcp/tools/create-alert-rule.ts index bad06f276c..d3726b6dab 100644 --- a/apps/ai/src/mcp/tools/create-alert-rule.ts +++ b/apps/ai/src/mcp/tools/create-alert-rule.ts @@ -60,7 +60,7 @@ const ALERT_TEMPLATES = { throughput_drop: { signalType: "throughput", comparator: "lt", defaultThreshold: 100 }, } satisfies Record -const Parameters = Schema.Struct({ +export const CreateAlertRuleParameters = Schema.Struct({ name: P.text("Rule name"), destination_ids: P.list( "Destination IDs to notify (use list_alert_destinations to find IDs). Pass an empty list for none.", @@ -139,7 +139,9 @@ const invalid = (message: string, parameter: string, example?: string) => ) /** The upsert request the params describe, before the domain schema checks it. */ -const buildAlertRuleRequest = Effect.fnUntraced(function* (params: typeof Parameters.Type) { +export const buildAlertRuleRequest = Effect.fnUntraced(function* ( + params: typeof CreateAlertRuleParameters.Type, +) { const template: AlertTemplate | undefined = params.template === undefined || params.template === "custom" ? undefined @@ -272,7 +274,7 @@ export function registerCreateAlertRuleTool(server: McpToolRegistrar) { description: "Create an alert rule. Pick a `template` for the common cases; otherwise pass signal_type, comparator and threshold yourself. " + "Use list_alert_destinations for destination_ids.", - parameters: Parameters, + parameters: CreateAlertRuleParameters, aliases: { service_names: "services" }, output: CreateAlertRuleOutput, hints: { readOnly: false, destructive: false, idempotent: false }, diff --git a/apps/ai/src/mcp/tools/get-alert-rule.ts b/apps/ai/src/mcp/tools/get-alert-rule.ts index 963f84d470..148df3a727 100644 --- a/apps/ai/src/mcp/tools/get-alert-rule.ts +++ b/apps/ai/src/mcp/tools/get-alert-rule.ts @@ -142,6 +142,11 @@ export function registerGetAlertRuleTool(server: McpToolRegistrar) { "recent evaluations: observed values and near-misses", ), doc.next("get_incident_timeline", { rule_id: rule.id }, "incident history for this rule"), + doc.next( + "preview_alert_rule", + { rule_id: rule.id }, + "replay it over past data, with or without changes", + ), ], } }, diff --git a/apps/ai/src/mcp/tools/list-alert-checks.ts b/apps/ai/src/mcp/tools/list-alert-checks.ts index 183af03e9c..c0cb6b5024 100644 --- a/apps/ai/src/mcp/tools/list-alert-checks.ts +++ b/apps/ai/src/mcp/tools/list-alert-checks.ts @@ -165,9 +165,9 @@ export function registerListAlertChecksTool(server: McpToolRegistrar) { if (output.noData > 0 && output.noData === output.total) { next.push( doc.next( - "get_alert_rule", + "preview_alert_rule", { rule_id: output.ruleId }, - "every check had no data (the query matched nothing), so the rule is blind, not healthy: check its filters", + "every check had no data (the query matched nothing), so the rule is blind, not healthy: replay it with adjusted filters", ), ) } else if (output.belowMinSamples > 0) { diff --git a/apps/ai/src/mcp/tools/preview-alert-rule.ts b/apps/ai/src/mcp/tools/preview-alert-rule.ts new file mode 100644 index 0000000000..978760cc4e --- /dev/null +++ b/apps/ai/src/mcp/tools/preview-alert-rule.ts @@ -0,0 +1,346 @@ +import { McpInvalidInputError, McpQueryBudgetError, type McpToolRegistrar } from "./types" +import { Effect, Schema, Struct } from "effect" +import { IsoDateTimeString } from "@maple/domain" +import { AlertRulePreviewRequest, type AlertRulePreviewResponse } from "@maple/domain/http" +import { PreviewAlertRuleOutput } from "@maple/domain/mcp-outputs" +import { rawAlertSampleCountWarning } from "@maple/domain/raw-sql" +import { AlertsService } from "@maple/backend/services/alerts/AlertsService" +import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService" +import { toMcpHttpError } from "../lib/map-http-error" +import { warehouseReadToMcpHandlers } from "../lib/map-warehouse-error" +import { CurrentMcpTenant } from "../lib/query-warehouse" +import { formatCondition, ruleNotFound } from "../lib/alert-rules" +import { truncate } from "../lib/format" +import * as P from "../lib/params" +import { doc, type DocBlock, type NextCall } from "../lib/tool-doc" +import { buildAlertRuleRequest, CreateAlertRuleParameters } from "./create-alert-rule" +import { buildUpdatedRequest } from "./update-alert-rule" + +const WINDOW = P.timeWindow({ defaultHours: 24, maxHours: 7 * 24 }) +/** Most recent windows the text shows per group; the rest stay in the structured output. */ +const SHOWN_POINTS = 12 +const SHOWN_GROUPS = 20 + +type Output = typeof PreviewAlertRuleOutput.Type + +const decodePreviewRequest = Schema.decodeUnknownEffect(AlertRulePreviewRequest) +const decodeIsoDateTimeString = Schema.decodeUnknownSync(IsoDateTimeString) + +/** Window bounds are `YYYY-MM-DD HH:mm:ss` UTC; the preview takes ISO 8601. */ +const toIso = (value: string) => decodeIsoDateTimeString(`${value.replace(" ", "T")}Z`) + +const Parameters = Schema.Struct({ + rule_id: P.optionalText( + "Existing rule to preview (list_alert_rules). Definition params you also pass override its saved config, so a change can be tried before update_alert_rule.", + ), + // The rule definition, exactly as create_alert_rule takes it; naming and delivery do not affect evaluation. + ...Struct.omit(CreateAlertRuleParameters.fields, [ + "name", + "destination_ids", + "severity", + "enabled", + "notification_title", + "notification_body", + ]), + ...WINDOW.fields, +}) + +const skipLabel = (reason: string | null): string => { + switch (reason) { + case "no_data": + return "no data" + case "below_min_samples": + return "below min samples" + case "no_value": + return "no value" + default: + return "skipped" + } +} + +const summarizeGroups = (preview: AlertRulePreviewResponse): Output["groups"] => + preview.series.map((series) => { + const values = series.points.flatMap((p) => (p.value === null ? [] : [p.value])) + const count = (predicate: (p: (typeof series.points)[number]) => boolean) => + series.points.filter(predicate).length + return { + groupKey: series.groupKey, + windows: series.points.length, + breached: count((p) => p.status === "breached"), + healthy: count((p) => p.status === "healthy"), + noData: count((p) => p.skipReason === "no_data"), + belowMinSamples: count((p) => p.skipReason === "below_min_samples"), + noValue: count((p) => p.skipReason === "no_value"), + minValue: values.length > 0 ? Math.min(...values) : null, + maxValue: values.length > 0 ? Math.max(...values) : null, + totalSamples: series.points.reduce((sum, p) => sum + p.sampleCount, 0), + } + }) + +/** What an agent would otherwise only learn after saving the rule and waiting for checks. */ +const previewWarnings = ( + groups: Output["groups"], + rule: { + readonly signalType: string + readonly rawQuerySql?: string | null + readonly minimumSampleCount: number + }, +): ReadonlyArray => { + const windows = groups.reduce((sum, g) => sum + g.windows, 0) + const noData = groups.reduce((sum, g) => sum + g.noData, 0) + const belowMin = groups.reduce((sum, g) => sum + g.belowMinSamples, 0) + const warnings: Array = [] + if (windows > 0 && noData === windows) { + warnings.push( + "Every window had no data: the query matched nothing in this range. Saved as is, every check would be skipped, which reads as quiet, not healthy. Check the filters, or widen start_time.", + ) + } else if (windows > 0 && belowMin === windows) { + warnings.push( + `Every window had fewer than ${rule.minimumSampleCount} samples, so every check would be skipped. Lower minimum_sample_count or widen window_minutes.`, + ) + } + if (rule.signalType === "raw_query" && rule.rawQuerySql != null) { + const warning = rawAlertSampleCountWarning(rule.rawQuerySql, rule.minimumSampleCount) + if (warning !== null) warnings.push(warning) + } + return warnings +} + +const formatValue = (value: number | null): string => (value === null ? "-" : String(value)) + +export function registerPreviewAlertRuleTool(server: McpToolRegistrar) { + server.define({ + name: "preview_alert_rule", + title: "Preview Alert Rule", + description: + "Replay an alert rule over past data without saving it: per evaluation window, the value, sample count and verdict the scheduler would produce (breached, healthy, or skipped with the reason), plus when it would have fired. " + + "Pass rule_id to preview a saved rule, optionally with overrides, or a create_alert_rule definition to dry-run a new one. Run it before saving a raw_query rule.", + parameters: Parameters, + aliases: { service_names: "services", since: "start_time", until: "end_time" }, + output: PreviewAlertRuleOutput, + hints: { readOnly: true }, + phrases: ["Previewing an alert rule"], + handler: Effect.fn("McpTool.previewAlertRule")(function* (params) { + const tenant = yield* CurrentMcpTenant + const { st, et } = yield* WINDOW.resolve(params, "preview_alert_rule") + + let rule: unknown + if (params.rule_id !== undefined) { + const rules = yield* AlertRulesService + const list = yield* rules + .listRules(tenant.orgId) + .pipe(Effect.mapError(toMcpHttpError("preview_alert_rule"))) + const current = list.rules.find((r) => r.id === params.rule_id) + if (!current) return yield* ruleNotFound(params.rule_id) + rule = buildUpdatedRequest(current, { ...params, rule_id: current.id }) + } else { + rule = yield* buildAlertRuleRequest({ ...params, name: "Preview", destination_ids: [] }) + } + + const request = yield* decodePreviewRequest({ + rule, + startTime: toIso(st), + endTime: toIso(et), + }).pipe( + Effect.mapError( + (error) => new McpInvalidInputError({ message: `Invalid alert rule: ${String(error)}` }), + ), + ) + + const alerts = yield* AlertsService + const preview = yield* alerts.previewRule(tenant.orgId, tenant.roles, request).pipe( + Effect.catchTags({ + "@maple/http/errors/AlertValidationError": (error) => + Effect.fail( + new McpInvalidInputError({ + message: [`Invalid alert rule: ${error.message}`, ...error.details].join( + "\n", + ), + }), + ), + "@maple/http/errors/AlertForbiddenError": (error) => + Effect.fail(toMcpHttpError("preview_alert_rule")(error)), + "@maple/http/errors/QueryEngineValidationError": (error) => + Effect.fail( + new McpInvalidInputError({ + message: + error.details.length > 0 + ? `${error.message}\n${error.details.join("\n")}` + : error.message, + }), + ), + "@maple/http/errors/QueryEngineTimeoutError": () => + Effect.fail( + new McpQueryBudgetError({ + message: "The preview ran past its time limit. Narrow start_time/end_time.", + pipeName: "preview_alert_rule", + setting: "max_execution_time", + }), + ), + ...warehouseReadToMcpHandlers("preview_alert_rule"), + // Raw-SQL rules mint a per-org warehouse token; ordinary reads do not. + "@maple/http/errors/TinybirdOrgTokenConfigError": (error) => + Effect.fail(toMcpHttpError("preview_alert_rule")(error)), + "@maple/http/errors/TinybirdOrgTokenMintError": (error) => + Effect.fail(toMcpHttpError("preview_alert_rule")(error)), + }), + ) + + const groups = summarizeGroups(preview) + const ruleRequest = request.rule + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + "maple.alert.signal_type": ruleRequest.signalType, + "result.groupCount": groups.length, + }) + + return { + ...(params.rule_id === undefined ? undefined : { ruleId: params.rule_id }), + timeRange: { start: st, end: et }, + truncatedToStart: preview.truncatedToStart, + windowMinutes: preview.windowMinutes, + comparator: preview.comparator, + threshold: preview.threshold, + thresholdUpper: preview.thresholdUpper, + minimumSampleCount: ruleRequest.minimumSampleCount ?? 0, + groups, + points: preview.series.flatMap((series) => + series.points.map((p) => ({ + groupKey: series.groupKey, + bucket: p.bucket, + status: p.status, + skipReason: p.skipReason ?? null, + value: p.value, + sampleCount: p.sampleCount, + provisional: p.provisional === true, + })), + ), + wouldFire: preview.wouldFire.map((span) => ({ + groupKey: span.groupKey, + start: span.start, + end: span.end, + })), + warnings: [ + ...previewWarnings(groups, { + signalType: ruleRequest.signalType, + rawQuerySql: ruleRequest.rawQuerySql, + minimumSampleCount: ruleRequest.minimumSampleCount ?? 0, + }), + ], + } + }), + render: (output) => { + const shownGroups = output.groups.slice(0, SHOWN_GROUPS) + const blocks: Array = [ + doc.fields([ + ["Condition", formatCondition(output)], + ["Window", `${output.windowMinutes}m`], + ["Minimum samples", output.minimumSampleCount], + [ + "Would fire", + output.wouldFire.length > 0 ? `${output.wouldFire.length} time(s)` : "never", + ], + ]), + doc.table( + [ + "Group", + "Windows", + "Breached", + "Healthy", + "No data", + "Below min", + "Min", + "Max", + "Samples", + ], + shownGroups.map((g) => [ + truncate(g.groupKey, 30), + String(g.windows), + String(g.breached), + String(g.healthy), + String(g.noData), + String(g.belowMinSamples), + formatValue(g.minValue), + formatValue(g.maxValue), + String(g.totalSamples), + ]), + ), + ] + + // The latest windows of the first group: enough to see the shape without the whole series. + const first = shownGroups[0] + if (first !== undefined) { + const recent = output.points.filter((p) => p.groupKey === first.groupKey).slice(-SHOWN_POINTS) + blocks.push( + doc.heading(`Latest windows: ${truncate(first.groupKey, 40)}`), + doc.table( + ["Window end", "Verdict", "Value", "Samples"], + recent.map((p) => [ + `${p.bucket.slice(0, 19)}${p.provisional ? " (in progress)" : ""}`, + p.status === "skipped" ? skipLabel(p.skipReason) : p.status, + formatValue(p.value), + String(p.sampleCount), + ]), + ), + ) + } + + if (output.wouldFire.length > 0) { + blocks.push( + doc.heading("Would have fired"), + doc.table( + ["Group", "From", "To"], + output.wouldFire + .slice(0, SHOWN_GROUPS) + .map((span) => [ + truncate(span.groupKey, 30), + span.start.slice(0, 19), + span.end.slice(0, 19), + ]), + ), + ) + } + + const next: Array = + output.ruleId === undefined + ? [ + doc.next( + "list_alert_destinations", + {}, + "destination ids for create_alert_rule once the preview looks right", + ), + ] + : [ + doc.next( + "list_alert_checks", + { rule_id: output.ruleId }, + "what the saved rule actually recorded", + ), + ] + + return { + title: "Alert Rule Preview", + scope: [ + ["Rule", output.ruleId], + ["Time range", `${output.timeRange.start} to ${output.timeRange.end}`], + ["Clamped to start", output.truncatedToStart ?? undefined], + ], + ...(output.warnings.length > 0 ? { notices: output.warnings } : undefined), + ...(output.groups.length === 0 + ? { empty: { message: "The preview returned no series for this range." } } + : undefined), + blocks: output.groups.length === 0 ? [] : blocks, + ...(output.groups.length > shownGroups.length + ? { + truncation: { + shown: shownGroups.length, + total: output.groups.length, + noun: "groups", + }, + } + : undefined), + next, + } + }, + }) +} diff --git a/apps/ai/src/mcp/tools/registry.ts b/apps/ai/src/mcp/tools/registry.ts index bd6a56bd8f..bf6a3e7f96 100644 --- a/apps/ai/src/mcp/tools/registry.ts +++ b/apps/ai/src/mcp/tools/registry.ts @@ -6,6 +6,7 @@ import { registerAddDashboardWidgetTool } from "./add-dashboard-widget" import { registerDescribeWarehouseTablesTool } from "./describe-warehouse-tables" import { registerComparePeriodsTool } from "./compare-periods" import { registerCreateAlertRuleTool } from "./create-alert-rule" +import { registerPreviewAlertRuleTool } from "./preview-alert-rule" import { registerUpdateAlertRuleTool } from "./update-alert-rule" import { registerDeleteAlertRuleTool } from "./delete-alert-rule" import { registerCreateDashboardTool } from "./create-dashboard" @@ -378,6 +379,7 @@ const collectMapleToolDefinitions = (): ReadonlyArray => { registerGetAlertRuleTool(registrar) registerListAlertIncidentsTool(registrar) registerListAlertChecksTool(registrar) + registerPreviewAlertRuleTool(registrar) registerGetIncidentTimelineTool(registrar) registerCreateAlertRuleTool(registrar) registerUpdateAlertRuleTool(registrar) diff --git a/apps/ai/src/mcp/tools/update-alert-rule.ts b/apps/ai/src/mcp/tools/update-alert-rule.ts index 29c3777bcd..f5df8cbe66 100644 --- a/apps/ai/src/mcp/tools/update-alert-rule.ts +++ b/apps/ai/src/mcp/tools/update-alert-rule.ts @@ -26,7 +26,7 @@ import * as P from "../lib/params" const decodeAlertRuleRequest = Schema.decodeUnknownEffect(AlertRuleUpsertRequest) -const Parameters = Schema.Struct({ +export const UpdateAlertRuleParameters = Schema.Struct({ rule_id: P.text("Alert rule ID to update (use list_alert_rules to find IDs)"), name: P.optionalText("New rule name"), severity: P.optionalOneOf(ALERT_SEVERITIES, "Alert severity"), @@ -80,7 +80,10 @@ const Parameters = Schema.Struct({ * rule's current config and overlaid with only the params the caller provided. The service's * `normalizeRule` validates the merged result. */ -function buildUpdatedRequest(current: AlertRuleDocument, params: typeof Parameters.Type) { +export function buildUpdatedRequest( + current: AlertRuleDocument, + params: typeof UpdateAlertRuleParameters.Type, +) { // Merge notification template fields onto the existing template (null-safe). const touchesTemplate = params.notification_title !== undefined || params.notification_body !== undefined const title = params.notification_title ?? current.notificationTemplate?.title ?? undefined @@ -136,7 +139,7 @@ export function registerUpdateAlertRuleTool(server: McpToolRegistrar) { description: "Update an alert rule. Pass only the fields to change; the rest keep their current value (get_alert_rule shows it). " + "Ids from list_alert_rules and list_alert_destinations.", - parameters: Parameters, + parameters: UpdateAlertRuleParameters, aliases: { service_names: "services" }, output: UpdateAlertRuleOutput, hints: { readOnly: false, destructive: false, idempotent: true }, diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index a5b41c9c57..9f80da44e8 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -323,6 +323,7 @@ const toV2PreviewResult = (preview: AlertRulePreviewResponse): V2AlertRulePrevie value: point.value, sample_count: point.sampleCount, status: point.status, + ...(point.skipReason !== undefined ? { skip_reason: point.skipReason } : undefined), ...(point.provisional !== undefined ? { provisional: point.provisional } : undefined), })), })), diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index 6a62757329..03c9002a2e 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -3447,6 +3447,7 @@ "type": "string" }, "_maple_AlertSkipReason": { + "description": "Set on `skipped` points: `no_data` (the window had no data), `below_min_samples`, or `no_value`.", "enum": [ "no_data", "below_min_samples", diff --git a/apps/landing/src/content/docs/reference/mcp.md b/apps/landing/src/content/docs/reference/mcp.md index 848bfbe201..ddc6084e87 100644 --- a/apps/landing/src/content/docs/reference/mcp.md +++ b/apps/landing/src/content/docs/reference/mcp.md @@ -103,6 +103,7 @@ The server exposes the tools below. Read-only tools are marked **read**; the res | `list_alert_incidents` | read | Triggered alert incidents, open and resolved, with the last observed value | | `get_incident_timeline` | read | Trigger, notification and resolution times for one rule's incidents, or all rules' | | `list_alert_checks` | read | A rule's recent evaluations with observed value, threshold and sample count | +| `preview_alert_rule` | read | Replay a saved or draft rule over past data: each window's verdict and skip reason | | `create_alert_rule` | write | Create an alert rule from a template or from signal type, comparator and threshold | | `update_alert_rule` | write | Change the fields you pass on an alert rule | | `delete_alert_rule` | write | Delete an alert rule and its incident history. Requires `confirm=true` | diff --git a/apps/web/src/components/ai-elements/tool-metadata.ts b/apps/web/src/components/ai-elements/tool-metadata.ts index 071abf78bd..832d69f36e 100644 --- a/apps/web/src/components/ai-elements/tool-metadata.ts +++ b/apps/web/src/components/ai-elements/tool-metadata.ts @@ -112,6 +112,7 @@ const toolLabels: Record = { delete_alert_rule: "Delete Alert Rule", list_alert_incidents: "List Alert Incidents", list_alert_checks: "List Alert Checks", + preview_alert_rule: "Preview Alert Rule", get_incident_timeline: "Incident Timeline", update_error_notification_policy: "Notification Policy", // sessions @@ -177,6 +178,7 @@ const toolIcons: Record = { delete_alert_rule: TrashIcon, list_alert_incidents: AlertWarningIcon, list_alert_checks: CircleCheckIcon, + preview_alert_rule: HistoryIcon, get_incident_timeline: HistoryIcon, update_error_notification_policy: BellIcon, run_sql: DatabaseIcon, diff --git a/apps/web/src/lib/alerts/form-utils.ts b/apps/web/src/lib/alerts/form-utils.ts index 7fd10c1df1..e30f2574f5 100644 --- a/apps/web/src/lib/alerts/form-utils.ts +++ b/apps/web/src/lib/alerts/form-utils.ts @@ -688,6 +688,9 @@ export function v2PreviewToResponse(result: V2AlertRulePreviewResult): AlertRule value: point.value, sampleCount: point.sample_count, status: point.status, + ...(point.skip_reason !== undefined + ? { skipReason: point.skip_reason } + : undefined), ...(point.provisional !== undefined ? { provisional: point.provisional } : undefined), diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index c7488159fd..f4780dc500 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -1328,6 +1328,7 @@ export class AlertsService extends Context.Service + ({ bucketMs, status, skipReason, value, sampleCount, provisional }) => new AlertRulePreviewPoint({ bucket: iso(bucketMs), value, sampleCount, status, + ...(skipReason === undefined ? undefined : { skipReason }), ...(provisional ? { provisional } : undefined), }), ), diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index df0fd86cb2..3b3e128297 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -617,6 +617,8 @@ export class AlertRulePreviewPoint extends Schema.Class(" value: Schema.NullOr(Schema.Number), sampleCount: Schema.Number, status: AlertEvaluationStatus, + /** Set on `status: "skipped"` points. */ + skipReason: Schema.optionalKey(AlertSkipReason), /** * The trailing in-progress window: evaluated over less than a full * `windowMinutes`, so its value may still move as data arrives. diff --git a/packages/domain/src/http/v2/alert-rules.ts b/packages/domain/src/http/v2/alert-rules.ts index 0b72e8b5b6..9751f3f5d2 100644 --- a/packages/domain/src/http/v2/alert-rules.ts +++ b/packages/domain/src/http/v2/alert-rules.ts @@ -451,6 +451,12 @@ const V2AlertRulePreviewPoint = Schema.Struct({ value: Schema.NullOr(Schema.Number), sample_count: Schema.Number, status: AlertEvaluationStatus, + skip_reason: Schema.optionalKey( + AlertSkipReason.annotate({ + description: + "Set on `skipped` points: `no_data` (the window had no data), `below_min_samples`, or `no_value`.", + }), + ), provisional: Schema.optionalKey( Schema.Boolean.annotate({ description: diff --git a/packages/domain/src/mcp-outputs/alerts.ts b/packages/domain/src/mcp-outputs/alerts.ts index efa96a82ad..651aa418e6 100644 --- a/packages/domain/src/mcp-outputs/alerts.ts +++ b/packages/domain/src/mcp-outputs/alerts.ts @@ -86,6 +86,50 @@ export const AlertRuleDetailRow = Schema.Struct({ export const GetAlertRuleOutput = Schema.Struct({ rule: AlertRuleDetailRow }) +/** One evaluation window of a preview: what the scheduler would have observed and decided. */ +export const AlertPreviewPointRow = Schema.Struct({ + groupKey: Schema.String, + bucket: Schema.String, + status: Schema.String, + skipReason: NullableString, + value: NullableNumber, + sampleCount: Schema.Number, + provisional: Schema.Boolean, +}) + +/** One group's windows, counted by verdict. */ +export const AlertPreviewGroupRow = Schema.Struct({ + groupKey: Schema.String, + windows: Schema.Number, + breached: Schema.Number, + healthy: Schema.Number, + noData: Schema.Number, + belowMinSamples: Schema.Number, + noValue: Schema.Number, + minValue: NullableNumber, + maxValue: NullableNumber, + totalSamples: Schema.Number, +}) + +export const PreviewAlertRuleOutput = Schema.Struct({ + /** Set when an existing rule was previewed. */ + ruleId: Schema.optionalKey(Schema.String), + timeRange: OutputTimeRange, + /** Set when the range was clamped to the preview's window cap. */ + truncatedToStart: NullableString, + windowMinutes: Schema.Number, + comparator: Schema.String, + threshold: Schema.Number, + thresholdUpper: NullableNumber, + minimumSampleCount: Schema.Number, + groups: Schema.Array(AlertPreviewGroupRow), + points: Schema.Array(AlertPreviewPointRow), + wouldFire: Schema.Array( + Schema.Struct({ groupKey: Schema.String, start: Schema.String, end: Schema.String }), + ), + warnings: Schema.Array(Schema.String), +}) + export const AlertIncidentRow = Schema.Struct({ id: Schema.String, ruleId: Schema.String, diff --git a/packages/domain/src/mcp-outputs/catalog.ts b/packages/domain/src/mcp-outputs/catalog.ts index 1d96455c48..74153941a2 100644 --- a/packages/domain/src/mcp-outputs/catalog.ts +++ b/packages/domain/src/mcp-outputs/catalog.ts @@ -9,6 +9,7 @@ import { GetAlertRuleOutput, GetIncidentTimelineOutput, ListAlertChecksOutput, + PreviewAlertRuleOutput, ListAlertDestinationsOutput, ListAlertIncidentsOutput, ListAlertRulesOutput, @@ -120,6 +121,7 @@ export const McpToolOutputs = { link_pull_request: LinkPullRequestOutput, list_agent_sessions: ListAgentSessionsOutput, list_alert_checks: ListAlertChecksOutput, + preview_alert_rule: PreviewAlertRuleOutput, list_alert_destinations: ListAlertDestinationsOutput, list_alert_incidents: ListAlertIncidentsOutput, list_alert_rules: ListAlertRulesOutput, From 3bb3e2bb60d2687a4ea98d8c0e3292500069eb78 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 2 Oct 2026 19:42:28 +0200 Subject: [PATCH 2/2] fix(alerts): preview a no-data series when a rule's query matches nothing The scheduler turns an empty result into one no-data observation per tick (per service in multi-service mode), but the preview only charted that for ungrouped plans, so a raw-SQL rule matching nothing previewed as no series and preview_alert_rule could never warn about it. preview_alert_rule also keeps the latest 200 windows per group in its structured output, labels the window column by its start, and tailors the advice once the range was clamped. --- apps/ai/src/mcp/tools/preview-alert-rule.ts | 13 +++++--- .../src/services/alerts/AlertsService.test.ts | 31 +++++++++++++++++++ .../src/services/alerts/AlertsService.ts | 24 ++++++++------ packages/domain/src/mcp-outputs/alerts.ts | 1 + 4 files changed, 55 insertions(+), 14 deletions(-) diff --git a/apps/ai/src/mcp/tools/preview-alert-rule.ts b/apps/ai/src/mcp/tools/preview-alert-rule.ts index 978760cc4e..1b390dba99 100644 --- a/apps/ai/src/mcp/tools/preview-alert-rule.ts +++ b/apps/ai/src/mcp/tools/preview-alert-rule.ts @@ -20,6 +20,8 @@ const WINDOW = P.timeWindow({ defaultHours: 24, maxHours: 7 * 24 }) /** Most recent windows the text shows per group; the rest stay in the structured output. */ const SHOWN_POINTS = 12 const SHOWN_GROUPS = 20 +/** Latest windows per group kept in the structured output; group counts cover the whole range. */ +const POINTS_PER_GROUP = 200 type Output = typeof PreviewAlertRuleOutput.Type @@ -31,7 +33,7 @@ const toIso = (value: string) => decodeIsoDateTimeString(`${value.replace(" ", " const Parameters = Schema.Struct({ rule_id: P.optionalText( - "Existing rule to preview (list_alert_rules). Definition params you also pass override its saved config, so a change can be tried before update_alert_rule.", + "Existing rule to preview (list_alert_rules). Definition params you also pass override its saved config, so a change can be tried before update_alert_rule. `template` is ignored with rule_id.", ), // The rule definition, exactly as create_alert_rule takes it; naming and delivery do not affect evaluation. ...Struct.omit(CreateAlertRuleParameters.fields, [ @@ -80,6 +82,7 @@ const summarizeGroups = (preview: AlertRulePreviewResponse): Output["groups"] => /** What an agent would otherwise only learn after saving the rule and waiting for checks. */ const previewWarnings = ( groups: Output["groups"], + clamped: boolean, rule: { readonly signalType: string readonly rawQuerySql?: string | null @@ -92,7 +95,7 @@ const previewWarnings = ( const warnings: Array = [] if (windows > 0 && noData === windows) { warnings.push( - "Every window had no data: the query matched nothing in this range. Saved as is, every check would be skipped, which reads as quiet, not healthy. Check the filters, or widen start_time.", + `Every window had no data: the query matched nothing in this range. Saved as is, every check would be skipped, which reads as quiet, not healthy. Check the filters${clamped ? "; the range was already clamped to the preview cap, so try a longer window_minutes rather than an earlier start_time" : ", or widen start_time"}.`, ) } else if (windows > 0 && belowMin === windows) { warnings.push( @@ -205,7 +208,7 @@ export function registerPreviewAlertRuleTool(server: McpToolRegistrar) { minimumSampleCount: ruleRequest.minimumSampleCount ?? 0, groups, points: preview.series.flatMap((series) => - series.points.map((p) => ({ + series.points.slice(-POINTS_PER_GROUP).map((p) => ({ groupKey: series.groupKey, bucket: p.bucket, status: p.status, @@ -221,7 +224,7 @@ export function registerPreviewAlertRuleTool(server: McpToolRegistrar) { end: span.end, })), warnings: [ - ...previewWarnings(groups, { + ...previewWarnings(groups, preview.truncatedToStart !== null, { signalType: ruleRequest.signalType, rawQuerySql: ruleRequest.rawQuerySql, minimumSampleCount: ruleRequest.minimumSampleCount ?? 0, @@ -274,7 +277,7 @@ export function registerPreviewAlertRuleTool(server: McpToolRegistrar) { blocks.push( doc.heading(`Latest windows: ${truncate(first.groupKey, 40)}`), doc.table( - ["Window end", "Verdict", "Value", "Samples"], + ["Window start", "Verdict", "Value", "Samples"], recent.map((p) => [ `${p.bucket.slice(0, 19)}${p.provisional ? " (in progress)" : ""}`, p.status === "skipped" ? skipLabel(p.skipReason) : p.status, diff --git a/packages/backend/src/services/alerts/AlertsService.test.ts b/packages/backend/src/services/alerts/AlertsService.test.ts index ff3e6931c4..46bdda4d75 100644 --- a/packages/backend/src/services/alerts/AlertsService.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.test.ts @@ -3780,6 +3780,37 @@ describe("AlertsService.previewRule", () => { }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) }) + it.effect("charts a no-data series for a raw-SQL rule whose query matches nothing", () => { + const testDb = createTestDb(trackedDbs) + const state = { rawQueryRows: [] } + + return Effect.gen(function* () { + const alerts = yield* AlertsService + const request = decodePreviewRequest({ + rule: { + name: "Raw preview empty", + severity: "warning", + signalType: "raw_query", + rawQuerySql: + "SELECT count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp)", + comparator: "gt", + threshold: 10, + windowMinutes: 5, + destinationIds: [], + }, + startTime: "2026-01-01T00:00:00.000Z", + endTime: "2026-01-01T00:30:00.000Z", + }) + + // The scheduler sees one no-data observation per tick here, so the preview must too. + const preview = yield* alerts.previewRule(asOrgId("org_preview_raw_empty"), adminRoles, request) + assert.lengthOf(preview.series, 1) + const points = preview.series[0]?.points ?? [] + assert.lengthOf(points, 6) + assert.isTrue(points.every((p) => p.status === "skipped" && p.skipReason === "no_data")) + }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) + }) + it.effect("dedupes rule destinations and environments, preserving order", () => { const testDb = createTestDb(trackedDbs) // Guards the `Arr.dedupe` calls in normalizeRule. A destination listed twice diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index f4780dc500..9e8e71046d 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -7,7 +7,12 @@ import { planAlertLifecycle, type AlertLifecycleInput, } from "@maple/alerting-core" -import { formatWarehouseDateTime, snapAlertWindowEndMs, warehouseDateTime64 } from "@maple/query-engine" +import { + ENGINE_UNGROUPED_GROUP_KEY, + formatWarehouseDateTime, + snapAlertWindowEndMs, + warehouseDateTime64, +} from "@maple/query-engine" import { AlertComparator as AlertComparatorSchema, type AlertComparator, @@ -1299,14 +1304,15 @@ export class AlertsService extends Context.Service 1) { + for (const serviceName of normalized.serviceNames) { + if (!obsByGroup.has(serviceName)) obsByGroup.set(serviceName, new Map()) + } + } else if (obsByGroup.size === 0) { + obsByGroup.set(toStorageGroupKey(plan, ENGINE_UNGROUPED_GROUP_KEY), new Map()) } const NO_DATA: PreviewObs = { diff --git a/packages/domain/src/mcp-outputs/alerts.ts b/packages/domain/src/mcp-outputs/alerts.ts index 651aa418e6..b6baff08b4 100644 --- a/packages/domain/src/mcp-outputs/alerts.ts +++ b/packages/domain/src/mcp-outputs/alerts.ts @@ -123,6 +123,7 @@ export const PreviewAlertRuleOutput = Schema.Struct({ thresholdUpper: NullableNumber, minimumSampleCount: Schema.Number, groups: Schema.Array(AlertPreviewGroupRow), + /** The latest 200 windows per group; `groups` counts the whole range. */ points: Schema.Array(AlertPreviewPointRow), wouldFire: Schema.Array( Schema.Struct({ groupKey: Schema.String, start: Schema.String, end: Schema.String }),