From 3b728dd007047cb68c9b88cb2623581f30ec5ebc Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 2 Oct 2026 01:29:45 +0200 Subject: [PATCH 1/2] feat(alerts): alert when a rule's window has no data A window with no data was always skipped (bar low-throughput rules, which read it as zero), so a rule whose query stopped matching, typically a raw SQL rule, went quiet with nothing to say it was blind. Rules take an opt-in alertOnNoData (alert_on_no_data on the v2 API and the MCP create/update tools, a switch under Evaluation timing in the app). It compiles to noDataBehavior "alert": an empty window evaluates as a breach with no value, so it opens and resolves an incident through the usual breach and healthy counts. Notifications read "No data" where the value would be. The stored no_data_behavior column already held the compiled value, so no migration is needed. get_alert_rule shows the behavior, the v2 rule exposes alert_on_no_data, and the IaC AlertRule resource accepts it. The alert rule docs now cover the samples column, the new switch, and how skipped checks report why. --- .../mcp/tools/__tests__/alert-tools.test.ts | 4 ++ apps/ai/src/mcp/tools/create-alert-rule.ts | 4 ++ apps/ai/src/mcp/tools/get-alert-rule.ts | 16 +++++++ apps/ai/src/mcp/tools/update-alert-rule.ts | 2 + apps/api/src/routes/v2/alert-rules.http.ts | 5 +++ apps/ios/Maple/Fixtures/FixtureAPI.swift | 1 + .../MapleAPI/Sources/MapleAPI/openapi.json | 14 ++++++- .../src/content/docs/alerting/alert-rules.md | 6 +-- .../alerts/signal-and-threshold-section.tsx | 17 ++++++++ apps/web/src/lib/alerts/diagnosis.ts | 4 +- apps/web/src/lib/alerts/form-utils.ts | 5 +++ packages/alchemy-maple/src/AlertRule.ts | 2 + packages/alerting-core/src/index.test.ts | 39 +++++++++++++++++ packages/alerting-core/src/index.ts | 18 +++++++- .../src/services/alerts/AlertRuleModel.ts | 11 ++++- .../src/services/alerts/AlertsService.test.ts | 42 +++++++++++++++++++ .../src/services/alerts/alert-formatting.ts | 5 ++- packages/domain/src/http/alerts.ts | 2 + packages/domain/src/http/v2/alert-rules.ts | 14 ++++++- .../domain/src/http/v2/v2-contract.test.ts | 1 + packages/domain/src/mcp-outputs/alerts.ts | 2 + packages/domain/src/query-engine.ts | 6 ++- 22 files changed, 208 insertions(+), 12 deletions(-) diff --git a/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts b/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts index 442c0c00b0..3c9364a290 100644 --- a/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts +++ b/apps/ai/src/mcp/tools/__tests__/alert-tools.test.ts @@ -402,6 +402,7 @@ describe("alert tools", () => { template: "high_error_rate", destination_ids: DEST_ID, environments: ["production"], + alert_on_no_data: true, }, seen, ) @@ -413,6 +414,7 @@ describe("alert tools", () => { destinationIds: [DEST_ID], environments: ["production"], groupBy: ["service.name"], + alertOnNoData: true, }) expect(text(result)).toContain("## Alert Rule Created") @@ -544,6 +546,8 @@ describe("alert tools", () => { name: "Checkout errors", serviceNames: ["checkout"], groupBy: ["service.name", "attr.http.route"], + // Carried over from the saved rule, which skips empty windows. + alertOnNoData: false, }) }) diff --git a/apps/ai/src/mcp/tools/create-alert-rule.ts b/apps/ai/src/mcp/tools/create-alert-rule.ts index d3726b6dab..0086b8e3c2 100644 --- a/apps/ai/src/mcp/tools/create-alert-rule.ts +++ b/apps/ai/src/mcp/tools/create-alert-rule.ts @@ -101,6 +101,9 @@ export const CreateAlertRuleParameters = Schema.Struct({ minimum_sample_count: P.optionalNumber( "Skip evaluation below this many samples in the window (default: 0). For raw_query it sums the `samples` column; without one each returned row counts as 1, so it gates on buckets, not events.", ), + alert_on_no_data: P.optionalFlag( + "Count a window with no data as a breach, so a rule whose query stops matching opens an incident instead of going quiet (default false: such windows are skipped).", + ), consecutive_breaches: P.optionalNumber("Consecutive breaches before alerting (default: 2)"), consecutive_healthy: P.optionalNumber("Consecutive healthy evaluations before resolving (default: 2)"), renotify_interval_minutes: P.optionalNumber("Re-notification interval in minutes (default: 30)"), @@ -232,6 +235,7 @@ export const buildAlertRuleRequest = Effect.fnUntraced(function* ( ...(params.minimum_sample_count === undefined ? undefined : { minimumSampleCount: params.minimum_sample_count }), + ...(params.alert_on_no_data === undefined ? undefined : { alertOnNoData: params.alert_on_no_data }), ...(params.consecutive_breaches === undefined ? undefined : { consecutiveBreachesRequired: params.consecutive_breaches }), diff --git a/apps/ai/src/mcp/tools/get-alert-rule.ts b/apps/ai/src/mcp/tools/get-alert-rule.ts index 148df3a727..30e89a7d90 100644 --- a/apps/ai/src/mcp/tools/get-alert-rule.ts +++ b/apps/ai/src/mcp/tools/get-alert-rule.ts @@ -8,6 +8,20 @@ import { formatCondition, ruleConfigWarnings, ruleNotFound, toAlertRuleRow } fro import * as P from "../lib/params" import { doc, type DocBlock } from "../lib/tool-doc" +/** How the rule treats an empty window, in words. */ +const noDataLabel = (behavior: string): string => { + switch (behavior) { + case "skip": + return "skip the check (never breaches)" + case "zero": + return "read as 0" + case "alert": + return "breach (alerts when the rule goes blind)" + default: + return behavior + } +} + export function registerGetAlertRuleTool(server: McpToolRegistrar) { server.define({ name: "get_alert_rule", @@ -37,6 +51,7 @@ export function registerGetAlertRuleTool(server: McpToolRegistrar) { excludeServiceNames: [...rule.excludeServiceNames], groupBy: rule.groupBy ? [...rule.groupBy] : null, minimumSampleCount: rule.minimumSampleCount, + noDataBehavior: rule.noDataBehavior, consecutiveBreachesRequired: rule.consecutiveBreachesRequired, consecutiveHealthyRequired: rule.consecutiveHealthyRequired, renotifyIntervalMinutes: rule.renotifyIntervalMinutes, @@ -86,6 +101,7 @@ export function registerGetAlertRuleTool(server: McpToolRegistrar) { doc.heading("Evaluation"), doc.fields([ ["Minimum Sample Count", rule.minimumSampleCount], + ["When No Data", noDataLabel(rule.noDataBehavior)], ["Consecutive Breaches Required", rule.consecutiveBreachesRequired], ["Consecutive Healthy Required", rule.consecutiveHealthyRequired], ["Renotify Interval", `${rule.renotifyIntervalMinutes}m`], diff --git a/apps/ai/src/mcp/tools/update-alert-rule.ts b/apps/ai/src/mcp/tools/update-alert-rule.ts index f5df8cbe66..792c94e346 100644 --- a/apps/ai/src/mcp/tools/update-alert-rule.ts +++ b/apps/ai/src/mcp/tools/update-alert-rule.ts @@ -52,6 +52,7 @@ export const UpdateAlertRuleParameters = Schema.Struct({ minimum_sample_count: P.optionalNumber( "Skip evaluation below this many samples in the window. For raw_query it sums the `samples` column; without one each returned row counts as 1.", ), + alert_on_no_data: P.optionalFlag("Count a window with no data as a breach instead of skipping it"), consecutive_breaches: P.optionalNumber("Consecutive breaches before alerting"), consecutive_healthy: P.optionalNumber("Consecutive healthy evaluations before resolving"), renotify_interval_minutes: P.optionalNumber("Re-notification interval in minutes"), @@ -128,6 +129,7 @@ export function buildUpdatedRequest( queryBuilderDraft: params.query_builder_draft ?? current.queryBuilderDraft, rawQuerySql: params.raw_query_sql ?? current.rawQuerySql, rawQueryReducer: params.raw_query_reducer ?? current.rawQueryReducer, + alertOnNoData: params.alert_on_no_data ?? current.noDataBehavior === "alert", destinationIds: [...(params.destination_ids ?? current.destinationIds)], } } diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 9f80da44e8..86aa0d54e4 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -87,6 +87,7 @@ const toV2Rule = (doc: AlertRuleDocument): V2AlertRule => ({ raw_query_reducer: doc.rawQueryReducer, destination_ids: doc.destinationIds, no_data_behavior: doc.noDataBehavior, + alert_on_no_data: doc.noDataBehavior === "alert", last_evaluation_error: doc.lastEvaluationError, last_evaluated_at: doc.lastEvaluatedAt, last_scheduled_at: doc.lastScheduledAt, @@ -212,6 +213,9 @@ const toUpsertRequest = ( minimumSampleCount: params.minimum_sample_count, } : undefined), + ...(params.alert_on_no_data !== undefined + ? { alertOnNoData: params.alert_on_no_data } + : undefined), ...(params.consecutive_breaches_required !== undefined ? { consecutiveBreachesRequired: params.consecutive_breaches_required, @@ -304,6 +308,7 @@ const mergeUpsertRequest = ( queryBuilderDraft, rawQuerySql, rawQueryReducer, + alertOnNoData: patch.alert_on_no_data ?? doc.noDataBehavior === "alert", destinationIds: patch.destination_ids ?? doc.destinationIds, }) }) diff --git a/apps/ios/Maple/Fixtures/FixtureAPI.swift b/apps/ios/Maple/Fixtures/FixtureAPI.swift index a4014d59ed..574b258a69 100644 --- a/apps/ios/Maple/Fixtures/FixtureAPI.swift +++ b/apps/ios/Maple/Fixtures/FixtureAPI.swift @@ -308,6 +308,7 @@ struct FixtureAPI: MapleAPI { threshold: Double, window: Int, breaches: Int, healthy: Int ) -> AlertRule { AlertRule( + alertOnNoData: false, comparator: .gt, consecutiveBreachesRequired: breaches, consecutiveHealthyRequired: healthy, diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index 03c9002a2e..d51844013e 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -482,6 +482,7 @@ "description": "A monitor that evaluates a built-in signal, query-builder query, or raw SQL over a rolling window and opens incidents when the threshold condition holds for enough consecutive checks. Metrics use the same query-builder path as metric charts. Notifications are delivered to the referenced alert destinations.", "examples": [ { + "alert_on_no_data": false, "apdex_threshold_ms": null, "comparator": "gt", "consecutive_breaches_required": 2, @@ -527,6 +528,13 @@ } ], "properties": { + "alert_on_no_data": { + "description": "Whether a window with no data counts as a breach (`no_data_behavior` is `alert`).", + "examples": [ + false + ], + "type": "boolean" + }, "apdex_threshold_ms": { "description": "For `apdex` rules: the satisfied-latency threshold in milliseconds.", "exclusiveMinimum": 0, @@ -743,6 +751,7 @@ "renotify_interval_minutes", "destination_ids", "no_data_behavior", + "alert_on_no_data", "created_at", "updated_at", "created_by", @@ -3641,10 +3650,11 @@ "type": "string" }, "_maple_QueryEngineNoDataBehavior": { - "description": "What the evaluator does when the window has no data: `skip` the check or treat the value as `zero`.", + "description": "What the evaluator does when the window has no data: `skip` the check, treat the value as `zero`, or `alert` (count it as a breach; set with `alert_on_no_data`).", "enum": [ "skip", - "zero" + "zero", + "alert" ], "examples": [ "skip" diff --git a/apps/landing/src/content/docs/alerting/alert-rules.md b/apps/landing/src/content/docs/alerting/alert-rules.md index 6d61c5fdb0..071a1e6267 100644 --- a/apps/landing/src/content/docs/alerting/alert-rules.md +++ b/apps/landing/src/content/docs/alerting/alert-rules.md @@ -62,7 +62,7 @@ A built-in signal only sees entry-point spans. A service that records a failure **Query** (`builder_query`) uses the same query builder as dashboard charts. It can read traces, logs, metrics or product events, with its own filters and group-by. The quickest way to build one is from a chart: open a dashboard, open the chart's menu and choose **Create alert**. -**Raw SQL** (`raw_query`) runs your own SQL. The query must include `$__orgFilter` and a `$__timeFilter(...)` on the time column, and return a time bucket and a value. **Reduce buckets by** turns the buckets in the window into one value: **Last bucket**, **Sum**, **Average**, **Minimum** or **Maximum**. To evaluate several groups, return a group column. See the [SQL reference](/docs/reference/sql). +**Raw SQL** (`raw_query`) runs your own SQL. The query must include `$__orgFilter` and a `$__timeFilter(...)` on the time column, and return a time bucket and a value. **Reduce buckets by** turns the buckets in the window into one value: **Last bucket**, **Sum**, **Average**, **Minimum** or **Maximum**. To evaluate several groups, return a group column. Return a `samples` column with the number of events behind each row: **Min samples** sums it, and without it every row counts as one sample, so the minimum counts buckets rather than events. See the [SQL reference](/docs/reference/sql). For **Query** and **Raw SQL** rules the query carries its own filters, so the **Scope** section is hidden. @@ -97,7 +97,7 @@ Maple evaluates every enabled rule once a minute. Each check aggregates the last A skipped check counts neither as a breach nor as healthy. It leaves the breach and healthy counters where they were. -A window with no data at all is skipped, with one exception: a **Throughput** rule with `<` or `<=` treats an empty window as zero. +A window with no data at all is skipped, with two exceptions: a **Throughput** rule with `<` or `<=` treats an empty window as zero, and a rule with **Alert when there is no data** on (`alert_on_no_data`) counts it as a breach. Turn it on for **Raw SQL** rules, where a query that stops matching otherwise goes quiet instead of firing. The **Min samples** check runs before the threshold comparison, and that zero still counts as zero samples. For **Throughput** the sample count is the signal, so a drop rule with the blank form's default of 50 skips every window below 50 requests, including a full outage, and cannot fire for those windows. Set **Min samples** to 0 for throughput drop rules, as the **Throughput drop** template does. Then traffic stopping entirely fires the rule. @@ -171,7 +171,7 @@ The response lists the value and status of each window per group in `series`, an ## Troubleshooting -- **The rule never fires.** Open the rule and look at its checks. If every check is skipped, the window has fewer samples than **Min samples**, or the scope matches no data. Check the service names and environments against what is arriving. +- **The rule never fires.** Open the rule and look at its checks. If every check is skipped, each one says why: **no data** means the scope or query matches nothing, **below min samples** means the window has fewer samples than **Min samples**. Check the service names and environments against what is arriving. - **The rule fires and resolves over and over.** Raise **Breaches to fire** and **Healthy to resolve**, or widen the window. - **Save is disabled.** The action bar lists what is missing, such as a rule name or a destination. diff --git a/apps/web/src/components/alerts/signal-and-threshold-section.tsx b/apps/web/src/components/alerts/signal-and-threshold-section.tsx index 1a18eeb8a5..a9c1c2913a 100644 --- a/apps/web/src/components/alerts/signal-and-threshold-section.tsx +++ b/apps/web/src/components/alerts/signal-and-threshold-section.tsx @@ -13,6 +13,7 @@ import { Card } from "@maple/ui/components/ui/card" import { Input } from "@maple/ui/components/ui/input" import { Label } from "@maple/ui/components/ui/label" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@maple/ui/components/ui/select" +import { Switch } from "@maple/ui/components/ui/switch" import { cn } from "@maple/ui/lib/utils" import { AlertSegmentedSelect } from "@/components/alerts/alert-segmented-select" @@ -381,6 +382,22 @@ export function SignalAndThresholdSection({ })) } /> +
+ + onChange((c) => ({ ...c, alertOnNoData: checked })) + } + /> +
+ +

+ Count a window with no data as a breach. Off, those windows are + skipped and the rule goes quiet when its query stops matching. +

+
+
)} diff --git a/apps/web/src/lib/alerts/diagnosis.ts b/apps/web/src/lib/alerts/diagnosis.ts index 2c7c0f8669..80b8f72260 100644 --- a/apps/web/src/lib/alerts/diagnosis.ts +++ b/apps/web/src/lib/alerts/diagnosis.ts @@ -184,7 +184,9 @@ export function buildDiagnosis(input: DiagnosisInput): DiagnosisStage[] { const noDataExplainer = rule.noDataBehavior === "zero" ? "No-data windows are treated as 0 (they still evaluate)." - : "No-data windows are skipped (they never breach)." + : rule.noDataBehavior === "alert" + ? "No-data windows count as breaches." + : "No-data windows are skipped (they never breach). Turn on alerting when there is no data to hear about it." // Checks recorded before skip reasons existed carry none and count as neither. const belowMinChecks = groupChecks.filter((c) => c.skipReason === "below_min_samples").length const noDataChecks = groupChecks.filter((c) => c.skipReason === "no_data").length diff --git a/apps/web/src/lib/alerts/form-utils.ts b/apps/web/src/lib/alerts/form-utils.ts index e30f2574f5..7e4a9da240 100644 --- a/apps/web/src/lib/alerts/form-utils.ts +++ b/apps/web/src/lib/alerts/form-utils.ts @@ -79,6 +79,8 @@ export type RuleFormState = { thresholdUpper: string windowMinutes: string minimumSampleCount: string + /** Count an empty window as a breach instead of skipping it. */ + alertOnNoData: boolean consecutiveBreachesRequired: string consecutiveHealthyRequired: string renotifyIntervalMinutes: string @@ -250,6 +252,7 @@ export function defaultRuleForm(serviceName?: string): RuleFormState { thresholdUpper: "", windowMinutes: "5", minimumSampleCount: "50", + alertOnNoData: false, consecutiveBreachesRequired: "2", consecutiveHealthyRequired: "2", renotifyIntervalMinutes: "30", @@ -282,6 +285,7 @@ export function ruleToFormState(rule: AlertRuleDocument): RuleFormState { rule.thresholdUpper == null ? "" : domainThresholdToForm(rule.signalType, rule.thresholdUpper), windowMinutes: String(rule.windowMinutes), minimumSampleCount: String(rule.minimumSampleCount), + alertOnNoData: rule.noDataBehavior === "alert", consecutiveBreachesRequired: String(rule.consecutiveBreachesRequired), consecutiveHealthyRequired: String(rule.consecutiveHealthyRequired), renotifyIntervalMinutes: String(rule.renotifyIntervalMinutes), @@ -364,6 +368,7 @@ export function buildRuleCreateParamsV2(form: RuleFormState): V2AlertRuleCreateP : null, window_minutes: parsePositiveNumber(form.windowMinutes, 5), minimum_sample_count: parseNonNegativeNumber(form.minimumSampleCount, 0), + alert_on_no_data: form.alertOnNoData, consecutive_breaches_required: parsePositiveNumber(form.consecutiveBreachesRequired, 2), consecutive_healthy_required: parsePositiveNumber(form.consecutiveHealthyRequired, 2), renotify_interval_minutes: parsePositiveNumber(form.renotifyIntervalMinutes, 30), diff --git a/packages/alchemy-maple/src/AlertRule.ts b/packages/alchemy-maple/src/AlertRule.ts index ad544f9b03..1f956b14f3 100644 --- a/packages/alchemy-maple/src/AlertRule.ts +++ b/packages/alchemy-maple/src/AlertRule.ts @@ -61,6 +61,8 @@ export interface AlertRuleProps { query_builder_draft?: Record | null raw_query_sql?: string | null raw_query_reducer?: "identity" | "sum" | "avg" | "min" | "max" | null + /** Count a window with no data as a breach instead of skipping it. */ + alert_on_no_data?: boolean notification_template?: Record | null } diff --git a/packages/alerting-core/src/index.test.ts b/packages/alerting-core/src/index.test.ts index 169c766ec0..110065c277 100644 --- a/packages/alerting-core/src/index.test.ts +++ b/packages/alerting-core/src/index.test.ts @@ -217,6 +217,45 @@ describe("planAlertLifecycle", () => { }) }) + it("breaches on an empty window when the rule alerts on no data, and opens an incident", () => { + const evaluation = evaluateAlertObservation( + { + comparator: "gt", + threshold: 10, + thresholdUpper: null, + minimumSampleCount: 50, + noDataBehavior: "alert", + }, + { value: null, sampleCount: 0, hasData: false }, + "above threshold", + ) + expect(evaluation).toMatchObject({ status: "breached", value: null, derivedFromNoData: true }) + expect(evaluation.skipReason).toBeUndefined() + expect( + planAlertLifecycle({ + policy: { ...policy, consecutiveBreachesRequired: 1 }, + evaluation, + state: null, + openIncident: null, + nowMs: 1_000, + }), + ).toMatchObject({ transition: "opened", eventType: "trigger" }) + + // Data that is present but thin is still gated by the minimum, not read as no data. + const thin = evaluateAlertObservation( + { + comparator: "gt", + threshold: 10, + thresholdUpper: null, + minimumSampleCount: 50, + noDataBehavior: "alert", + }, + { value: 20, sampleCount: 3, hasData: true }, + "above threshold", + ) + expect(thin).toMatchObject({ status: "skipped", skipReason: "below_min_samples" }) + }) + it("never turns insufficient samples or invalid scalars into empty-window recovery", () => { for (const observation of [ { value: 5, sampleCount: 1, hasData: true }, diff --git a/packages/alerting-core/src/index.ts b/packages/alerting-core/src/index.ts index 28145a6af2..4076a09598 100644 --- a/packages/alerting-core/src/index.ts +++ b/packages/alerting-core/src/index.ts @@ -21,7 +21,7 @@ export interface AlertEvaluationPolicy { readonly threshold: number readonly thresholdUpper: number | null readonly minimumSampleCount: number - readonly noDataBehavior: "skip" | "zero" + readonly noDataBehavior: "skip" | "zero" | "alert" } export interface AlertEvaluation extends Pick< @@ -35,7 +35,7 @@ export interface AlertEvaluation extends Pick< | "reason" | "skipReason" > { - /** A healthy result derived from an empty window synthesized as zero. */ + /** Derived from an empty window: healthy when read as zero, breached when the rule alerts on no data. */ readonly derivedFromNoData: boolean } @@ -95,6 +95,20 @@ export const evaluateAlertObservation = ( } } + // The window is empty and the rule asked to hear about it: a breach with no value. + if (!observation.hasData && policy.noDataBehavior === "alert") { + return { + status: "breached", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "No data in the selected window", + derivedFromNoData: true, + } + } + if (sampleCount < policy.minimumSampleCount) { return { status: "skipped", diff --git a/packages/backend/src/services/alerts/AlertRuleModel.ts b/packages/backend/src/services/alerts/AlertRuleModel.ts index 4769257409..d6904eb728 100644 --- a/packages/backend/src/services/alerts/AlertRuleModel.ts +++ b/packages/backend/src/services/alerts/AlertRuleModel.ts @@ -76,6 +76,7 @@ export interface NormalizedRule { readonly queryBuilderDraft: QueryBuilderQueryDraftPayload | null readonly rawQuerySql: string | null readonly rawQueryReducer: QueryEngineAlertReducer | null + readonly alertOnNoData: boolean readonly destinationIds: ReadonlyArray readonly compiledPlan: Schema.Schema.Type readonly createdAt: number @@ -248,6 +249,7 @@ export const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(functi readonly comparator: AlertComparator readonly windowMinutes: number readonly groupBy: AlertGroupBy | null + readonly alertOnNoData: boolean }): Effect.fn.Return, AlertValidationError> { const bucketSeconds = alertWindowBucketSeconds(rule.windowMinutes) const envFilter = rule.environments.length > 0 ? { environments: rule.environments } : {} @@ -256,8 +258,13 @@ export const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(functi ...envFilter, } + // A low-throughput rule already breaches on an empty window read as zero. const noDataBehavior: QueryEngineNoDataBehavior = - rule.signalType === "throughput" && ["lt", "lte"].includes(rule.comparator) ? "zero" : "skip" + rule.signalType === "throughput" && ["lt", "lte"].includes(rule.comparator) + ? "zero" + : rule.alertOnNoData + ? "alert" + : "skip" const traceSignalMetrics: Record = { error_rate: "error_rate", p95_latency: "p95_duration", @@ -648,6 +655,7 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => { queryBuilderDraft: stored.queryBuilderDraft, rawQuerySql: row.rawQuerySql ?? null, rawQueryReducer: decoded.rawQueryReducer, + alertOnNoData: row.noDataBehavior === "alert", destinationIds: yield* decodeStoredAlertRuleDestinationIds(row.id, row.destinationIdsJson), compiledPlan: yield* parseCompiledPlan(row), createdAt: dateToMs(row.createdAt), @@ -779,6 +787,7 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => { rawQuerySql: request.signalType === "raw_query" ? normalizeOptionalString(request.rawQuerySql) : null, rawQueryReducer: request.signalType === "raw_query" ? (request.rawQueryReducer ?? null) : null, + alertOnNoData: request.alertOnNoData ?? false, destinationIds, createdAt: nowMs, updatedAt: nowMs, diff --git a/packages/backend/src/services/alerts/AlertsService.test.ts b/packages/backend/src/services/alerts/AlertsService.test.ts index 46bdda4d75..19b48e2552 100644 --- a/packages/backend/src/services/alerts/AlertsService.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.test.ts @@ -3505,6 +3505,48 @@ describe("AlertsService evaluation error persistence", () => { }).pipe(Effect.provide(makeLayer(testDb, failingWarehouseStub(state), { fetch: okFetch }))) }) + it.effect("opens an incident on empty windows when the rule alerts on no data", () => { + const testDb = createTestDb(trackedDbs) + const state = { failing: false, rows: [], ingested: [] as Array> } + + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const orgId = asOrgId("org_alert_on_no_data") + const userId = asUserId("user_alert_on_no_data") + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const rule = yield* alerts.createRule( + orgId, + userId, + adminRoles, + new AlertRuleUpsertRequest({ + name: "Checkout goes blind", + severity: "critical", + serviceNames: ["checkout"], + signalType: "error_rate", + comparator: "gt", + threshold: 5, + windowMinutes: 5, + minimumSampleCount: 10, + consecutiveBreachesRequired: 1, + alertOnNoData: true, + destinationIds: [destination.id], + }), + ) + assert.strictEqual(rule.noDataBehavior, "alert") + + yield* alerts.runSchedulerTick() + + const breached = state.ingested.filter((row) => row.Status === "breached") + assert.lengthOf(breached, 1) + assert.isNull(breached[0]?.ObservedValue) + assert.strictEqual(breached[0]?.IncidentTransition, "opened") + const incidents = yield* alerts.listIncidents(orgId) + assert.lengthOf(incidents.incidents, 1) + assert.strictEqual(incidents.incidents[0]?.status, "open") + }).pipe(Effect.provide(makeLayer(testDb, failingWarehouseStub(state), { fetch: okFetch }))) + }) + it.effect("records why a check skipped when the window has no data", () => { const testDb = createTestDb(trackedDbs) const state = { failing: false, rows: [], ingested: [] as Array> } diff --git a/packages/backend/src/services/alerts/alert-formatting.ts b/packages/backend/src/services/alerts/alert-formatting.ts index 510f4ebc62..07fea76273 100644 --- a/packages/backend/src/services/alerts/alert-formatting.ts +++ b/packages/backend/src/services/alerts/alert-formatting.ts @@ -180,8 +180,11 @@ export const formatThresholdSummary = (context: ThresholdContext): string => { : `${formatComparator(context.comparator)} ${formatSignalMetric(context.threshold, display)}` } +/** A breach without a value is a rule alerting on an empty window. */ export const formatObservedSummary = (context: ObservedContext): string => - `${formatSignalMetric(context.value, signalDisplayOf(context))} ${formatThresholdSummary(context)}` + context.value === null + ? `No data (threshold ${formatThresholdSummary(context)})` + : `${formatSignalMetric(context.value, signalDisplayOf(context))} ${formatThresholdSummary(context)}` export const comparatorBreachPhrase = (context: ThresholdContext): string => { const display = signalDisplayOf(context) diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index 3b3e128297..7ca4456774 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -567,6 +567,8 @@ export class AlertRuleUpsertRequest extends Schema.Class queryBuilderDraft: Schema.optionalKey(Schema.NullOr(QueryBuilderQueryDraftSchema)), rawQuerySql: Schema.optionalKey(Schema.NullOr(Schema.String)), rawQueryReducer: Schema.optionalKey(Schema.NullOr(QueryEngineAlertReducer)), + /** Count a window with no data as a breach instead of skipping it. Default false. */ + alertOnNoData: Schema.optionalKey(Schema.Boolean), destinationIds: Schema.Array(AlertDestinationId), }) {} diff --git a/packages/domain/src/http/v2/alert-rules.ts b/packages/domain/src/http/v2/alert-rules.ts index 9751f3f5d2..2a87a41da4 100644 --- a/packages/domain/src/http/v2/alert-rules.ts +++ b/packages/domain/src/http/v2/alert-rules.ts @@ -86,6 +86,7 @@ const alertRuleExample = { raw_query_reducer: null, destination_ids: ["dest_oybbpTBhtSFGShMjjLiCrh"], no_data_behavior: "skip", + alert_on_no_data: false, last_evaluation_error: null, last_evaluated_at: "2026-07-15T09:10:00.000Z", last_scheduled_at: "2026-07-15T09:10:00.000Z", @@ -196,9 +197,13 @@ export const V2AlertRule = Schema.Struct({ }), no_data_behavior: QueryEngineNoDataBehavior.annotate({ description: - "What the evaluator does when the window has no data: `skip` the check or treat the value as `zero`.", + "What the evaluator does when the window has no data: `skip` the check, treat the value as `zero`, or `alert` (count it as a breach; set with `alert_on_no_data`).", examples: ["skip"], }), + alert_on_no_data: Schema.Boolean.annotate({ + description: "Whether a window with no data counts as a breach (`no_data_behavior` is `alert`).", + examples: [false], + }), last_evaluation_error: Schema.NullOr(Schema.String).annotate({ description: "The most recent evaluation error for this rule, or `null` if the last evaluation succeeded.", @@ -314,6 +319,12 @@ const createParamsFields = { query_builder_draft: Schema.optionalKey(Schema.NullOr(QueryBuilderDraftPassthrough)), raw_query_sql: Schema.optionalKey(Schema.NullOr(Schema.String)), raw_query_reducer: Schema.optionalKey(Schema.NullOr(QueryEngineAlertReducer)), + alert_on_no_data: Schema.optionalKey( + Schema.Boolean.annotate({ + description: + "Count a window with no data (e.g. a raw query returning no rows) as a breach, so a rule that goes blind opens an incident. Default `false`: such windows are skipped.", + }), + ), destination_ids: Schema.Array(AlertDestinationPublicId).annotate({ description: "The alert destinations (`dest_…`) to notify. May be empty.", }), @@ -365,6 +376,7 @@ export const V2AlertRuleUpdateParams = Schema.Struct({ query_builder_draft: createParamsFields.query_builder_draft, raw_query_sql: createParamsFields.raw_query_sql, raw_query_reducer: createParamsFields.raw_query_reducer, + alert_on_no_data: createParamsFields.alert_on_no_data, }).annotate({ identifier: "AlertRuleUpdateParams", title: "Alert rule update parameters", diff --git a/packages/domain/src/http/v2/v2-contract.test.ts b/packages/domain/src/http/v2/v2-contract.test.ts index b83401c60c..aef7cf714c 100644 --- a/packages/domain/src/http/v2/v2-contract.test.ts +++ b/packages/domain/src/http/v2/v2-contract.test.ts @@ -374,6 +374,7 @@ describe("V2 alerts wire format", () => { raw_query_reducer: null, destination_ids: [encodePublicId("dest", DEST_UUID)], no_data_behavior: "skip", + alert_on_no_data: false, last_evaluation_error: null, last_evaluated_at: null, last_scheduled_at: null, diff --git a/packages/domain/src/mcp-outputs/alerts.ts b/packages/domain/src/mcp-outputs/alerts.ts index b6baff08b4..c8e1795368 100644 --- a/packages/domain/src/mcp-outputs/alerts.ts +++ b/packages/domain/src/mcp-outputs/alerts.ts @@ -68,6 +68,8 @@ export const AlertRuleDetailRow = Schema.Struct({ excludeServiceNames: Schema.Array(Schema.String), groupBy: Schema.NullOr(Schema.Array(Schema.String)), minimumSampleCount: Schema.Number, + /** What an empty window does: skip, zero, or alert. */ + noDataBehavior: Schema.String, consecutiveBreachesRequired: Schema.Number, consecutiveHealthyRequired: Schema.Number, renotifyIntervalMinutes: Schema.Number, diff --git a/packages/domain/src/query-engine.ts b/packages/domain/src/query-engine.ts index 09a794fd1b..2b90d48779 100644 --- a/packages/domain/src/query-engine.ts +++ b/packages/domain/src/query-engine.ts @@ -704,7 +704,11 @@ export const QueryEngineSampleCountStrategy = Schema.Literals([ }) export type QueryEngineSampleCountStrategy = Schema.Schema.Type -export const QueryEngineNoDataBehavior = Schema.Literals(["skip", "zero"]).annotate({ +/** + * What an empty window evaluates to: `skip` the check, read it as `zero`, or + * `alert`, which counts it as a breach so a rule that goes blind opens an incident. + */ +export const QueryEngineNoDataBehavior = Schema.Literals(["skip", "zero", "alert"]).annotate({ identifier: "@maple/QueryEngineNoDataBehavior", }) export type QueryEngineNoDataBehavior = Schema.Schema.Type From cd80cc09b33f425d47cdffa69abbc5c0be068fa3 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 2 Oct 2026 19:47:45 +0200 Subject: [PATCH 2/2] fix(alerts): make alert-on-no-data hold up on throughput and grouped rules - alertOnNoData now wins over the low-throughput zero read. Read as zero, an empty window was skipped under a minimum sample count instead of breaching, and the flag read back as off on every rebuild (edits, the v2 rule, IaC drift). - Grouped rules reject it: an empty result breached under the engine's "all" key, never a real group, and a group that stops reporting is already held by its incident's telemetry check. The app disables the switch on grouped rules. - The preview treats a group's missing windows as skips, as the scheduler never evaluates them, and charts the empty-result series where nothing reported. - Changing the no-data behavior resolves open incidents, the testRule fallback goes through applyEvaluationLogic, the summary line reads "has no data" in place of "n/a", alert_on_no_data is in the audit diff, and the Electric alert rows read an unknown behavior as skip. --- apps/ai/src/mcp/tools/create-alert-rule.ts | 2 +- apps/ai/src/mcp/tools/preview-alert-rule.ts | 2 +- apps/ai/src/mcp/tools/update-alert-rule.ts | 4 +- apps/api/src/routes/v2/alert-rules.http.ts | 1 + .../src/content/docs/alerting/alert-rules.md | 2 +- .../alerts/signal-and-threshold-section.tsx | 10 +- apps/web/src/lib/alerts/form-utils.test.ts | 12 +++ apps/web/src/lib/alerts/form-utils.ts | 11 ++- apps/web/src/lib/collections/alerts.ts | 5 +- .../alerts/AlertDeliveryDispatch.test.ts | 6 ++ .../services/alerts/AlertDeliveryDispatch.ts | 4 + .../src/services/alerts/AlertRuleModel.ts | 25 +++-- .../src/services/alerts/AlertsService.test.ts | 95 +++++++++++++++++++ .../src/services/alerts/AlertsService.ts | 82 +++++++++------- packages/domain/src/http/v2/alert-rules.ts | 2 +- 15 files changed, 210 insertions(+), 53 deletions(-) diff --git a/apps/ai/src/mcp/tools/create-alert-rule.ts b/apps/ai/src/mcp/tools/create-alert-rule.ts index 0086b8e3c2..5eb622ef16 100644 --- a/apps/ai/src/mcp/tools/create-alert-rule.ts +++ b/apps/ai/src/mcp/tools/create-alert-rule.ts @@ -102,7 +102,7 @@ export const CreateAlertRuleParameters = Schema.Struct({ "Skip evaluation below this many samples in the window (default: 0). For raw_query it sums the `samples` column; without one each returned row counts as 1, so it gates on buckets, not events.", ), alert_on_no_data: P.optionalFlag( - "Count a window with no data as a breach, so a rule whose query stops matching opens an incident instead of going quiet (default false: such windows are skipped).", + "Count a window with no data as a breach, so a rule whose query stops matching opens an incident instead of going quiet (default false: such windows are skipped). Not supported with group_by.", ), consecutive_breaches: P.optionalNumber("Consecutive breaches before alerting (default: 2)"), consecutive_healthy: P.optionalNumber("Consecutive healthy evaluations before resolving (default: 2)"), diff --git a/apps/ai/src/mcp/tools/preview-alert-rule.ts b/apps/ai/src/mcp/tools/preview-alert-rule.ts index 1b390dba99..0666de9f42 100644 --- a/apps/ai/src/mcp/tools/preview-alert-rule.ts +++ b/apps/ai/src/mcp/tools/preview-alert-rule.ts @@ -95,7 +95,7 @@ const previewWarnings = ( const warnings: Array = [] if (windows > 0 && noData === windows) { warnings.push( - `Every window had no data: the query matched nothing in this range. Saved as is, every check would be skipped, which reads as quiet, not healthy. Check the filters${clamped ? "; the range was already clamped to the preview cap, so try a longer window_minutes rather than an earlier start_time" : ", or widen start_time"}.`, + `Every window had no data: the query matched nothing in this range. Saved as is, every check would be skipped, which reads as quiet, not healthy (alert_on_no_data makes it fire instead). Check the filters${clamped ? "; the range was already clamped to the preview cap, so try a longer window_minutes rather than an earlier start_time" : ", or widen start_time"}.`, ) } else if (windows > 0 && belowMin === windows) { warnings.push( diff --git a/apps/ai/src/mcp/tools/update-alert-rule.ts b/apps/ai/src/mcp/tools/update-alert-rule.ts index 792c94e346..e331f11275 100644 --- a/apps/ai/src/mcp/tools/update-alert-rule.ts +++ b/apps/ai/src/mcp/tools/update-alert-rule.ts @@ -52,7 +52,9 @@ export const UpdateAlertRuleParameters = Schema.Struct({ minimum_sample_count: P.optionalNumber( "Skip evaluation below this many samples in the window. For raw_query it sums the `samples` column; without one each returned row counts as 1.", ), - alert_on_no_data: P.optionalFlag("Count a window with no data as a breach instead of skipping it"), + alert_on_no_data: P.optionalFlag( + "Count a window with no data as a breach instead of skipping it. Not supported with group_by.", + ), consecutive_breaches: P.optionalNumber("Consecutive breaches before alerting"), consecutive_healthy: P.optionalNumber("Consecutive healthy evaluations before resolving"), renotify_interval_minutes: P.optionalNumber("Re-notification interval in minutes"), diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 86aa0d54e4..f2b7fabd1e 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -123,6 +123,7 @@ const ruleAuditDiff = auditDiff onChange((c) => ({ ...c, alertOnNoData: checked })) } @@ -393,8 +396,9 @@ export function SignalAndThresholdSection({

- Count a window with no data as a breach. Off, those windows are - skipped and the rule goes quiet when its query stops matching. + {grouped + ? "Not available on grouped rules: a group that stops reporting keeps its incident open until telemetry returns." + : "Count a window with no data as a breach. Off, those windows are skipped and the rule goes quiet when its query stops matching."}

diff --git a/apps/web/src/lib/alerts/form-utils.test.ts b/apps/web/src/lib/alerts/form-utils.test.ts index c5bc528bc9..c5f211ffa8 100644 --- a/apps/web/src/lib/alerts/form-utils.test.ts +++ b/apps/web/src/lib/alerts/form-utils.test.ts @@ -13,6 +13,7 @@ import { buildRuleCreateParamsV2, defaultDestinationForm, defaultRuleForm, + ruleFormIsGrouped, deriveRuleQueryIssues, domainThresholdToForm, formThresholdToDomain, @@ -389,3 +390,14 @@ describe("v2 response mappers", () => { }) }) }) + +describe("alert on no data", () => { + it("is sent only for rules that are not grouped", () => { + const form = { ...defaultRuleForm(), name: "A", alertOnNoData: true } + expect(buildRuleCreateParamsV2(form).alert_on_no_data).toBe(true) + const grouped = { ...form, serviceNames: [], groupBy: ["service.name"] } + expect(ruleFormIsGrouped(grouped)).toBe(true) + expect(buildRuleCreateParamsV2(grouped).alert_on_no_data).toBe(false) + expect(ruleFormIsGrouped({ ...grouped, signalType: "raw_query" })).toBe(false) + }) +}) diff --git a/apps/web/src/lib/alerts/form-utils.ts b/apps/web/src/lib/alerts/form-utils.ts index 7e4a9da240..5a1714d305 100644 --- a/apps/web/src/lib/alerts/form-utils.ts +++ b/apps/web/src/lib/alerts/form-utils.ts @@ -266,6 +266,15 @@ export function defaultRuleForm(serviceName?: string): RuleFormState { } } +/** Grouped rules cannot alert on no data: a group that stops reporting is held by its incident. */ +export function ruleFormIsGrouped(form: RuleFormState): boolean { + if (form.signalType === "raw_query") return false + if (form.signalType === "builder_query") { + return form.queryBuilderDraft.groupBy.some((token) => token !== "none" && token.length > 0) + } + return form.groupBy.length > 0 +} + export function ruleToFormState(rule: AlertRuleDocument): RuleFormState { const queryBuilderDraft = normalizeRuleQueryDraft(rule.queryBuilderDraft) return { @@ -368,7 +377,7 @@ export function buildRuleCreateParamsV2(form: RuleFormState): V2AlertRuleCreateP : null, window_minutes: parsePositiveNumber(form.windowMinutes, 5), minimum_sample_count: parseNonNegativeNumber(form.minimumSampleCount, 0), - alert_on_no_data: form.alertOnNoData, + alert_on_no_data: form.alertOnNoData && !ruleFormIsGrouped(form), consecutive_breaches_required: parsePositiveNumber(form.consecutiveBreachesRequired, 2), consecutive_healthy_required: parsePositiveNumber(form.consecutiveHealthyRequired, 2), renotify_interval_minutes: parsePositiveNumber(form.renotifyIntervalMinutes, 30), diff --git a/apps/web/src/lib/collections/alerts.ts b/apps/web/src/lib/collections/alerts.ts index 4a71ff5725..c74e47589f 100644 --- a/apps/web/src/lib/collections/alerts.ts +++ b/apps/web/src/lib/collections/alerts.ts @@ -40,7 +40,8 @@ const asIncidentStatus = Schema.decodeUnknownSync(AlertIncidentStatus) const asHoldReason = Schema.decodeUnknownSync(AlertIncidentHoldReason) const asEventType = Schema.decodeUnknownSync(AlertEventType) const asReducer = Schema.decodeUnknownSync(QueryEngineAlertReducer) -const asNoDataBehavior = Schema.decodeUnknownSync(QueryEngineNoDataBehavior) +// Lenient: a behavior added by a newer server reads as skip rather than breaking the alerts list. +const decodeNoDataBehavior = Schema.decodeUnknownOption(QueryEngineNoDataBehavior) const decodeNotificationTemplate = Schema.decodeUnknownOption(AlertNotificationTemplate) const decodeQueryBuilderDraft = Schema.decodeUnknownOption(QueryBuilderQueryDraftSchema) @@ -216,7 +217,7 @@ export const rowToAlertRuleDocument = ( rawQuerySql: row.raw_query_sql ?? null, rawQueryReducer: row.signal_type === "raw_query" ? asReducer(row.reducer) : null, destinationIds: safeParseStringArray(row.destination_ids_json).map((id) => decodeDestinationId(id)), - noDataBehavior: asNoDataBehavior(row.no_data_behavior), + noDataBehavior: Option.getOrElse(decodeNoDataBehavior(row.no_data_behavior), () => "skip" as const), lastEvaluationError: state?.last_error ?? null, lastEvaluatedAt: state?.last_evaluated_at != null ? decodeIso(state.last_evaluated_at) : null, lastScheduledAt: row.last_scheduled_at != null ? decodeIso(row.last_scheduled_at) : null, diff --git a/packages/backend/src/services/alerts/AlertDeliveryDispatch.test.ts b/packages/backend/src/services/alerts/AlertDeliveryDispatch.test.ts index df1f83bd11..f2ace4c5ad 100644 --- a/packages/backend/src/services/alerts/AlertDeliveryDispatch.test.ts +++ b/packages/backend/src/services/alerts/AlertDeliveryDispatch.test.ts @@ -149,6 +149,12 @@ describe("buildSummaryLine", () => { * metric name and its value as a bare unpunctuated integer — * "*builder_query* is *1041923*". */ + it("says a breach without a value is a window with no data", () => { + const line = bold({ ...baseContext, value: null }) + assert.include(line, "has no data over the last") + assert.notInclude(line, "n/a") + }) + it("names what a builder_query rule measures instead of its query kind", () => { const line = bold({ ...baseContext, diff --git a/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts b/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts index bc719c5b34..6b13c5c0a4 100644 --- a/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts +++ b/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts @@ -135,6 +135,10 @@ export const buildSummaryLine = (context: SummaryLineContext, em: (value: string const now = context.value != null ? ` — now ${em(observed)}` : "" return `${em(signal)} is back within its threshold (${formatThresholdSummary(context)})${now}.` } + // A breach without a value is a rule that alerts when its window has no data. + if (context.value == null) { + return `${em(signal)} has no data over the last ${window} (alert threshold ${formatThresholdSummary(context)}).` + } return `${em(signal)} is ${em(observed)} — ${comparatorBreachPhrase(context)}, measured over the last ${window}.` } diff --git a/packages/backend/src/services/alerts/AlertRuleModel.ts b/packages/backend/src/services/alerts/AlertRuleModel.ts index d6904eb728..370e21f2fa 100644 --- a/packages/backend/src/services/alerts/AlertRuleModel.ts +++ b/packages/backend/src/services/alerts/AlertRuleModel.ts @@ -258,13 +258,13 @@ export const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(functi ...envFilter, } - // A low-throughput rule already breaches on an empty window read as zero. - const noDataBehavior: QueryEngineNoDataBehavior = - rule.signalType === "throughput" && ["lt", "lte"].includes(rule.comparator) + // An explicit alert-on-no-data wins: a low-throughput rule reads an empty window + // as zero, but a minimum sample count then skips it instead of breaching. + const noDataBehavior: QueryEngineNoDataBehavior = rule.alertOnNoData + ? "alert" + : rule.signalType === "throughput" && ["lt", "lte"].includes(rule.comparator) ? "zero" - : rule.alertOnNoData - ? "alert" - : "skip" + : "skip" const traceSignalMetrics: Record = { error_rate: "error_rate", p95_latency: "p95_duration", @@ -792,10 +792,17 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => { createdAt: nowMs, updatedAt: nowMs, } - return { - ...normalizedBase, - compiledPlan: yield* compileRulePlan(normalizedBase), + const compiledPlan = yield* compileRulePlan(normalizedBase) + // A grouped rule's missing group is already held open by its incident's + // liveness check; an empty result would breach under no real group. + if (normalizedBase.alertOnNoData && planGroupingTokens(compiledPlan) != null) { + return yield* Effect.fail( + makeAlertValidationError("Invalid alert rule", [ + "alertOnNoData is not supported on grouped rules: a group that stops reporting is already held open by its incident's telemetry check", + ]), + ) } + return { ...normalizedBase, compiledPlan } }) return { normalizeRule, normalizeRuleRow } diff --git a/packages/backend/src/services/alerts/AlertsService.test.ts b/packages/backend/src/services/alerts/AlertsService.test.ts index 19b48e2552..e5d68a413f 100644 --- a/packages/backend/src/services/alerts/AlertsService.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.test.ts @@ -3547,6 +3547,57 @@ describe("AlertsService evaluation error persistence", () => { }).pipe(Effect.provide(makeLayer(testDb, failingWarehouseStub(state), { fetch: okFetch }))) }) + it.effect("keeps alertOnNoData on a low-throughput rule and rejects it on a grouped one", () => { + const testDb = createTestDb(trackedDbs) + const state = { failing: false, rows: [], ingested: [] as Array> } + + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const orgId = asOrgId("org_alert_on_no_data_shapes") + const userId = asUserId("user_alert_on_no_data_shapes") + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const request = (fields: Partial[0]>) => + new AlertRuleUpsertRequest({ + name: "No data shapes", + severity: "critical", + signalType: "throughput", + comparator: "lt", + threshold: 10, + windowMinutes: 5, + minimumSampleCount: 50, + consecutiveBreachesRequired: 1, + alertOnNoData: true, + destinationIds: [destination.id], + ...fields, + }) + + // Read as zero, an empty window would be skipped under the minimum; alert wins. + const throughput = yield* alerts.createRule( + orgId, + userId, + adminRoles, + request({ serviceNames: ["checkout"] }), + ) + assert.strictEqual(throughput.noDataBehavior, "alert") + yield* alerts.runSchedulerTick() + assert.lengthOf( + state.ingested.filter((row) => row.Status === "breached" && row.ObservedValue === null), + 1, + ) + + const grouped = yield* alerts + .createRule( + orgId, + userId, + adminRoles, + request({ name: "Grouped", groupBy: ["service.name"] }), + ) + .pipe(Effect.flip) + assert.instanceOf(grouped, AlertValidationError) + }).pipe(Effect.provide(makeLayer(testDb, failingWarehouseStub(state), { fetch: okFetch }))) + }) + it.effect("records why a check skipped when the window has no data", () => { const testDb = createTestDb(trackedDbs) const state = { failing: false, rows: [], ingested: [] as Array> } @@ -3853,6 +3904,50 @@ describe("AlertsService.previewRule", () => { }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) }) + it.effect("previews a grouped raw-SQL rule that alerts on no data the way the scheduler fires", () => { + const testDb = createTestDb(trackedDbs) + const state = { + rawQueryRows: [ + { bucket: "2026-01-01 00:00:00", group: "a", value: 1, samples: 5 }, + { bucket: "2026-01-01 00:05:00", group: "a", value: 1, samples: 5 }, + ], + } + + return Effect.gen(function* () { + const alerts = yield* AlertsService + const request = decodePreviewRequest({ + rule: { + name: "Raw grouped no data", + severity: "warning", + signalType: "raw_query", + rawQuerySql: + "SELECT $__timeGroup(Timestamp) AS bucket, ServiceName AS group, count() AS value FROM traces WHERE $__orgFilter AND $__timeFilter(Timestamp) GROUP BY bucket, group", + comparator: "gt", + threshold: 10, + windowMinutes: 5, + consecutiveBreachesRequired: 1, + alertOnNoData: true, + destinationIds: [], + }, + startTime: "2026-01-01T00:00:00.000Z", + endTime: "2026-01-01T00:30:00.000Z", + }) + + const preview = yield* alerts.previewRule(asOrgId("org_preview_raw_grouped"), adminRoles, request) + const byGroup = new Map(preview.series.map((series) => [series.groupKey, series.points])) + // A group missing from a tick is not evaluated by the scheduler, so it never breaches. + const a = byGroup.get("a") ?? [] + assert.isTrue(a.slice(2).every((p) => p.status === "skipped" && p.skipReason === "no_data")) + // Ticks where nothing reported at all breach under the empty-result key. + const empty = byGroup.get("all") ?? [] + assert.deepStrictEqual( + empty.map((p) => p.status), + ["healthy", "healthy", "breached", "breached", "breached", "breached"], + ) + assert.isTrue(preview.wouldFire.every((span) => span.groupKey === "all")) + }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(state), { fetch: okFetch }))) + }) + it.effect("dedupes rule destinations and environments, preserving order", () => { const testDb = createTestDb(trackedDbs) // Guards the `Arr.dedupe` calls in normalizeRule. A destination listed twice diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 9e8e71046d..469dfed6bb 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -653,6 +653,9 @@ export class AlertsService extends Context.Service, @@ -1051,31 +1054,16 @@ export class AlertsService extends Context.Service r.status === "breached") ?? - results[0] ?? { - status: "skipped" as const, - value: null, - sampleCount: 0, - threshold: normalized.threshold, - thresholdUpper: normalized.thresholdUpper, - comparator: normalized.comparator, - reason: "No data", - skipReason: "no_data" as const, - } + evaluation = + results.find((r) => r.status === "breached") ?? + results[0] ?? + applyEvaluationLogic(normalized, EMPTY_OBSERVATION) } else { // Uniform grouped/ungrouped path — mirrors runSchedulerTick: the // compiled plan decides groupedness, and a breaching group (if any) @@ -1086,17 +1074,10 @@ export class AlertsService extends Context.Service !HashSet.has(excludeSet, r.groupKey)) : allResults const breached = results.find((r) => r.evaluation.status === "breached") - evaluation = breached?.evaluation ?? - results[0]?.evaluation ?? { - status: "skipped" as const, - value: null, - sampleCount: 0, - threshold: normalized.threshold, - thresholdUpper: normalized.thresholdUpper, - comparator: normalized.comparator, - reason: "No data", - skipReason: "no_data" as const, - } + evaluation = + breached?.evaluation ?? + results[0]?.evaluation ?? + applyEvaluationLogic(normalized, EMPTY_OBSERVATION) } if (sendNotification) { @@ -1322,15 +1303,48 @@ export class AlertsService extends Context.Service decodeIsoDateTimeStringSync(new Date(ms).toISOString()) + // The scheduler never evaluates a group that is missing from a tick; with + // alert-on-no-data it breaches only when the whole result is empty, under the + // engine's ungrouped key. Model both so per-group gaps do not read as breaches. + const groupedAlert = isGroupedPlan(plan) && plan.noDataBehavior === "alert" + const skipGapsRule: NormalizedRule = { + ...normalized, + compiledPlan: { ...plan, noDataBehavior: "skip" }, + } + const emptyKey = toStorageGroupKey(plan, ENGINE_UNGROUPED_GROUP_KEY) + if (groupedAlert && !obsByGroup.has(emptyKey)) { + const emptyTicks = pointBuckets.filter( + (bucketMs) => + ![...obsByGroup.values()].some( + (buckets) => buckets.get(bucketMs)?.hasData === true, + ), + ) + if (emptyTicks.length > 0) obsByGroup.set(emptyKey, new Map()) + } + const series: AlertRulePreviewSeries[] = [] const wouldFire: AlertRulePreviewFiringSpan[] = [] for (const [groupKey, buckets] of obsByGroup) { + const isEmptyResultSeries = groupedAlert && groupKey === emptyKey && buckets.size === 0 + const tickHasData = (bucketMs: number) => + [...obsByGroup.values()].some((other) => other.get(bucketMs)?.hasData === true) // Every window in the grid, judged by the same `applyEvaluationLogic` // the scheduler runs per tick — no-data windows included, filled from // `NO_DATA` so a gap is an evaluated skip rather than a missing point. const evaluations = pointBuckets.map((bucketMs) => { const obs = buckets.get(bucketMs) ?? NO_DATA - const evaluation = applyEvaluationLogic(normalized, obs) + const evaluation: EvaluatedRule = isEmptyResultSeries + ? tickHasData(bucketMs) + ? // Groups reported this tick: the empty-result incident resolves. + { + ...applyEvaluationLogic(skipGapsRule, NO_DATA), + status: "healthy", + skipReason: undefined, + } + : applyEvaluationLogic(normalized, NO_DATA) + : groupedAlert && !buckets.has(bucketMs) + ? applyEvaluationLogic(skipGapsRule, NO_DATA) + : applyEvaluationLogic(normalized, obs) return { bucketMs, status: evaluation.status, @@ -2388,6 +2402,8 @@ export class AlertsService extends Context.Service { if (!groupByEqual(effectiveGroupByKeys(oldRule), effectiveGroupByKeys(newRule))) return true if (oldRule.signalType !== newRule.signalType) return true + // An incident opened on an empty window cannot resolve once empty windows are skipped again. + if (oldRule.compiledPlan.noDataBehavior !== newRule.compiledPlan.noDataBehavior) return true const mode = (r: NormalizedRule) => isGroupedPlan(r.compiledPlan) ? "grouped" : r.serviceNames.length > 1 ? "multi" : "single" return mode(oldRule) !== mode(newRule) diff --git a/packages/domain/src/http/v2/alert-rules.ts b/packages/domain/src/http/v2/alert-rules.ts index 2a87a41da4..b76a48b0df 100644 --- a/packages/domain/src/http/v2/alert-rules.ts +++ b/packages/domain/src/http/v2/alert-rules.ts @@ -322,7 +322,7 @@ const createParamsFields = { alert_on_no_data: Schema.optionalKey( Schema.Boolean.annotate({ description: - "Count a window with no data (e.g. a raw query returning no rows) as a breach, so a rule that goes blind opens an incident. Default `false`: such windows are skipped.", + "Count a window with no data (e.g. a raw query returning no rows) as a breach, so a rule that goes blind opens an incident. Default `false`: such windows are skipped. Not supported on grouped rules, where a group that stops reporting keeps its incident open until telemetry returns.", }), ), destination_ids: Schema.Array(AlertDestinationPublicId).annotate({