From 74fd00bf81879c40ef91feb6015ef5f32276cefb Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 10:45:45 +0000 Subject: [PATCH] Nightly traffic digest, on by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The performance report already had weekly and monthly cadences, an hourly TZ-aware cron and a settings control. This adds 'daily' to that rather than a second mechanism beside it. What the nightly one is for is traffic, so the report gains a traffic section and the email leads with it: overall humans and bots first, then every property most to least busy, with each one's bot share. The subject carries the human count, because that is the number worth seeing on a phone. Counting is by tracker bucket — a bucket starting 'bot:' is a crawler and everything else is a human, AI referrals included. That split only exists on tracker_daily_stats; tracker_event_daily_stats has no bucket column and its pageview counts include crawlers. The migration makes 'daily' the column default so a new account gets the digest without hunting for the setting. Existing rows keep whatever their owner chose: a default applies to inserts, and moving somebody from weekly to nightly would be changing a preference, not honouring one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WYMJH7N4d2qRct5Q5q2YWQ --- app/(app)/dashboard/settings/form.tsx | 8 +- app/(app)/dashboard/settings/page.tsx | 3 +- app/actions/settings.ts | 2 +- lib/perfReport.ts | 159 +++++++++++++++++- .../20260906100000_perf_report_daily.sql | 26 +++ tests/perf-report-daily.test.ts | 98 +++++++++++ 6 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 supabase/migrations/20260906100000_perf_report_daily.sql create mode 100644 tests/perf-report-daily.test.ts diff --git a/app/(app)/dashboard/settings/form.tsx b/app/(app)/dashboard/settings/form.tsx index 54efe34a..9e670b0c 100644 --- a/app/(app)/dashboard/settings/form.tsx +++ b/app/(app)/dashboard/settings/form.tsx @@ -3,7 +3,7 @@ import { useState, useTransition } from "react"; import { saveSettings } from "@/app/actions/settings"; -type Cadence = "off" | "weekly" | "monthly"; +type Cadence = "off" | "daily" | "weekly" | "monthly"; export function SettingsForm({ displayName, @@ -69,8 +69,9 @@ export function SettingsForm({ Email reports

A combined digest of your audit scores and Autoblog activity. - Sent on Monday 09:00 (weekly) or the 1st of the month at 09:00 - (monthly), in the timezone below. + Sent at 09:00 in the timezone below — every day (nightly), on + Monday (weekly), or on the 1st (monthly). The nightly one leads + with traffic over the last 24 hours, busiest property first.

@@ -82,6 +83,7 @@ export function SettingsForm({ onChange={(e) => setCadence(e.target.value as Cadence)} > + diff --git a/app/(app)/dashboard/settings/page.tsx b/app/(app)/dashboard/settings/page.tsx index 24dc38b5..4a34b9df 100644 --- a/app/(app)/dashboard/settings/page.tsx +++ b/app/(app)/dashboard/settings/page.tsx @@ -61,8 +61,9 @@ export default async function SettingsPage() { displayName={profile?.display_name ?? ""} retainRawHtml={!!profile?.retain_raw_html} perfReportCadence={ - (profile?.perf_report_cadence ?? "weekly") as + (profile?.perf_report_cadence ?? "daily") as | "off" + | "daily" | "weekly" | "monthly" } diff --git a/app/actions/settings.ts b/app/actions/settings.ts index bf26f4f6..f80bac69 100644 --- a/app/actions/settings.ts +++ b/app/actions/settings.ts @@ -3,7 +3,7 @@ import { createClient } from "@/lib/supabase/server"; import { isValidTimezone } from "@/lib/timezones"; -const ALLOWED_CADENCES = ["off", "weekly", "monthly"] as const; +const ALLOWED_CADENCES = ["off", "daily", "weekly", "monthly"] as const; type Cadence = (typeof ALLOWED_CADENCES)[number]; export async function saveSettings(input: { diff --git a/lib/perfReport.ts b/lib/perfReport.ts index 204772de..0eae77ce 100644 --- a/lib/perfReport.ts +++ b/lib/perfReport.ts @@ -10,7 +10,7 @@ import { env } from "./env"; const DAY_MS = 24 * 60 * 60 * 1000; -export type Cadence = "weekly" | "monthly"; +export type Cadence = "daily" | "weekly" | "monthly"; export type ProjectRow = { id: string; @@ -29,6 +29,27 @@ export type AutoblogSummary = { nextPublishAt: string | null; } | null; +/** + * A property's last-24h traffic, split the way the tracker splits it: a + * bucket starting `bot:` is a crawler, everything else is a human. AI + * referrals count as humans on purpose — someone asked a model and came. + */ +export type TrafficRow = { + projectId: string; + name: string; + humans: number; + bots: number; + total: number; +}; + +export type TrafficSummary = { + humans: number; + bots: number; + total: number; + /** Most to least traffic. */ + properties: TrafficRow[]; +}; + export type PerfReport = { userId: string; userEmail: string; @@ -38,9 +59,12 @@ export type PerfReport = { windowEnd: Date; projects: ProjectRow[]; autoblog: AutoblogSummary; + /** Present on every cadence; it is the point of the nightly one. */ + traffic: TrafficSummary; }; function windowDays(cadence: Cadence): number { + if (cadence === "daily") return 1; return cadence === "weekly" ? 7 : 30; } @@ -96,6 +120,15 @@ export function isReportDue( } if (local.hour !== 9) return false; + if (cadence === "daily") { + // One tick a day is eligible, and 20h of dedupe covers a DST shift + // without ever letting two sends land in the same local day. + if (lastSentAt && now.getTime() - lastSentAt.getTime() < 20 * 60 * 60 * 1000) { + return false; + } + return true; + } + if (cadence === "weekly") { if (local.weekday !== 1) return false; // Mon if ( @@ -121,6 +154,64 @@ export function isReportDue( return false; } +/** + * Traffic per property over the window, most to least. + * + * `tracker_daily_stats` is already bucketed per day, so this is a read of + * whole days rather than a rolling window — a nightly report at 09:00 local + * wants "yesterday and today so far", which is what `days` back from today + * gives. Counting is by bucket because that is the only place the human/bot + * split exists: `tracker_event_daily_stats` has no bucket column and its + * pageview counts include crawlers. + */ +async function aggregateTraffic( + supabase: SupabaseClient, + projects: { id: string; name: string }[], + days: number, + now: Date, +): Promise { + const empty: TrafficSummary = { humans: 0, bots: 0, total: 0, properties: [] }; + if (!projects.length) return empty; + + const since = new Date(now.getTime() - days * DAY_MS).toISOString().slice(0, 10); + const { data } = await supabase + .from("tracker_daily_stats") + .select("project_id, bucket, count") + .in( + "project_id", + projects.map((project) => project.id), + ) + .gte("day", since); + + const byProject = new Map(); + for (const row of (data ?? []) as { project_id: string; bucket: string; count: number }[]) { + const tally = byProject.get(row.project_id) ?? { humans: 0, bots: 0 }; + if (row.bucket?.startsWith("bot:")) tally.bots += row.count ?? 0; + else tally.humans += row.count ?? 0; + byProject.set(row.project_id, tally); + } + + const properties: TrafficRow[] = projects + .map((project) => { + const tally = byProject.get(project.id) ?? { humans: 0, bots: 0 }; + return { + projectId: project.id, + name: project.name, + humans: tally.humans, + bots: tally.bots, + total: tally.humans + tally.bots, + }; + }) + .sort((a, b) => b.total - a.total || a.name.localeCompare(b.name)); + + return { + humans: properties.reduce((sum, row) => sum + row.humans, 0), + bots: properties.reduce((sum, row) => sum + row.bots, 0), + total: properties.reduce((sum, row) => sum + row.total, 0), + properties, + }; +} + export async function aggregatePerfReport( supabase: SupabaseClient, userId: string, @@ -228,6 +319,12 @@ export async function aggregatePerfReport( windowEnd: now, projects: projectRows, autoblog, + traffic: await aggregateTraffic( + supabase, + (projects ?? []).map((project: any) => ({ id: project.id, name: project.name })), + days, + now, + ), }; } @@ -254,11 +351,15 @@ export function renderPerfReportEmail(r: PerfReport): { subject: string; html: string; } { - const window = r.cadence === "weekly" ? "this week" : "this month"; + const window = + r.cadence === "daily" ? "today" : r.cadence === "weekly" ? "this week" : "this month"; + // The nightly one leads with the number the reader actually wants. const subject = - r.projects.length === 0 && !r.autoblog - ? `Your CrawlProof ${r.cadence} digest` - : `CrawlProof ${r.cadence} digest — ${r.projects.length} project${r.projects.length === 1 ? "" : "s"}${r.autoblog ? " + Autoblog" : ""}`; + r.cadence === "daily" + ? `CrawlProof nightly — ${fmt(r.traffic.humans)} human visit${r.traffic.humans === 1 ? "" : "s"}, ${fmt(r.traffic.bots)} bot` + : r.projects.length === 0 && !r.autoblog + ? `Your CrawlProof ${r.cadence} digest` + : `CrawlProof ${r.cadence} digest — ${r.projects.length} project${r.projects.length === 1 ? "" : "s"}${r.autoblog ? " + Autoblog" : ""}`; const projectRowsHtml = r.projects.length ? r.projects @@ -310,10 +411,52 @@ export function renderPerfReportEmail(r: PerfReport): { ` : ""; + const trafficRowsHtml = r.traffic.properties.length + ? r.traffic.properties + .map((row) => { + const share = row.total ? Math.round((row.bots / row.total) * 100) : 0; + return ` + + + + + + +
+
${escapeHtml(row.name)}
+
${fmt(row.humans)} human · ${fmt(row.bots)} bot (${share}%)
+ 		+ ${fmt(row.total)} +
+ + `; + }) + .join("") + : `No traffic recorded. Is the tracker tag on the site?`; + + const trafficBlock = ` + + + + + + + +
Traffic · last ${r.cadence === "daily" ? "24 hours" : window.replace(/^this /, "")}
+ + + ${statCell("Humans", r.traffic.humans, "#6ee7b7")} + ${statCell("Bots", r.traffic.bots)} + ${statCell("All hits", r.traffic.total)} + +
+
${trafficRowsHtml}
`; + const innerHtml = `

Your ${r.cadence} digest

-

${window.replace(/^this /, "")} of audit + Autoblog activity.

+

Traffic, audits and Autoblog for ${window.replace(/^this /, "")}, busiest property first.

+ ${trafficBlock} ${projectRowsHtml}
@@ -330,6 +473,10 @@ export function renderPerfReportEmail(r: PerfReport): { return { subject, html }; } +function fmt(n: number): string { + return n.toLocaleString("en-US"); +} + function statCell(label: string, value: number, accent?: string): string { return `
${escapeHtml(label)}
diff --git a/supabase/migrations/20260906100000_perf_report_daily.sql b/supabase/migrations/20260906100000_perf_report_daily.sql new file mode 100644 index 00000000..038ec6be --- /dev/null +++ b/supabase/migrations/20260906100000_perf_report_daily.sql @@ -0,0 +1,26 @@ +-- Nightly performance reports. +-- +-- The cadence column already gates who gets a digest and how often; this +-- adds 'daily' to it and makes it the default, so a new account gets the +-- nightly traffic summary without having to find the setting. +-- +-- Existing rows keep whatever their owner chose. A default only applies to +-- inserts, and silently moving somebody from weekly to nightly would be +-- changing a preference they set, not honouring one. +-- +-- Apply one file at a time via the Supabase MCP's apply_migration — prod's +-- migration history has diverged from this directory, so `supabase db push` +-- would try to replay files prod already has. + +alter table public.profiles + drop constraint if exists profiles_perf_report_cadence_check; + +alter table public.profiles + add constraint profiles_perf_report_cadence_check + check (perf_report_cadence in ('off', 'daily', 'weekly', 'monthly')); + +alter table public.profiles + alter column perf_report_cadence set default 'daily'; + +-- The hourly cron reads `where perf_report_cadence <> 'off'`; the existing +-- index still covers it, and daily simply makes more rows eligible per tick. diff --git a/tests/perf-report-daily.test.ts b/tests/perf-report-daily.test.ts new file mode 100644 index 00000000..dfd40470 --- /dev/null +++ b/tests/perf-report-daily.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import { isReportDue, renderPerfReportEmail, type PerfReport } from "@/lib/perfReport"; + +// 09:00 in the timezone under test is the only eligible hour. +const at = (iso: string) => new Date(iso); + +describe("isReportDue — daily", () => { + it("fires at 09:00 local on any day of the week", () => { + // A Wednesday and a Sunday; neither is special for the daily cadence. + expect(isReportDue("daily", "UTC", at("2026-09-02T09:00:00Z"), null)).toBe(true); + expect(isReportDue("daily", "UTC", at("2026-09-06T09:30:00Z"), null)).toBe(true); + }); + + it("ignores every other hour", () => { + expect(isReportDue("daily", "UTC", at("2026-09-06T08:59:00Z"), null)).toBe(false); + expect(isReportDue("daily", "UTC", at("2026-09-06T10:00:00Z"), null)).toBe(false); + }); + + it("is local, not UTC", () => { + // 09:00 Los Angeles is 16:00 UTC. + const utcMorning = at("2026-09-06T09:00:00Z"); + expect(isReportDue("daily", "America/Los_Angeles", utcMorning, null)).toBe(false); + expect(isReportDue("daily", "America/Los_Angeles", at("2026-09-06T16:00:00Z"), null)).toBe(true); + }); + + it("does not send twice in the same local day", () => { + const now = at("2026-09-06T09:00:00Z"); + const anHourAgo = at("2026-09-06T08:00:00Z"); + expect(isReportDue("daily", "UTC", now, anHourAgo)).toBe(false); + }); + + it("sends again the next day, even when the clock shifted", () => { + // 23h later is a different local day but under the 24h a naive gate + // would use; the 20h window is what makes a DST day still send. + const now = at("2026-09-07T09:00:00Z"); + const yesterday = at("2026-09-06T09:00:00Z"); + expect(isReportDue("daily", "UTC", now, yesterday)).toBe(true); + }); + + it("leaves the other cadences alone", () => { + // 2026-09-06 is a Sunday, so weekly (Monday) must not fire. + expect(isReportDue("weekly", "UTC", at("2026-09-06T09:00:00Z"), null)).toBe(false); + expect(isReportDue("monthly", "UTC", at("2026-09-06T09:00:00Z"), null)).toBe(false); + expect(isReportDue("weekly", "UTC", at("2026-09-07T09:00:00Z"), null)).toBe(true); + }); +}); + +const report = (overrides: Partial = {}): PerfReport => ({ + userId: "u1", + userEmail: "a@b.com", + userDisplayName: "A", + cadence: "daily", + windowStart: at("2026-09-05T09:00:00Z"), + windowEnd: at("2026-09-06T09:00:00Z"), + projects: [], + autoblog: null, + traffic: { + humans: 1234, + bots: 91011, + total: 92245, + properties: [ + { projectId: "p1", name: "genrewatch.com", humans: 27, bots: 90000, total: 90027 }, + { projectId: "p2", name: "chovy blog", humans: 1207, bots: 1011, total: 2218 }, + ], + }, + ...overrides, +}); + +describe("the nightly email", () => { + it("leads its subject with humans, and formats big numbers", () => { + const { subject, html } = renderPerfReportEmail(report()); + expect(subject).toBe("CrawlProof nightly — 1,234 human visits, 91,011 bot"); + expect(html).toContain("genrewatch.com"); + expect(html).toContain("90,027"); + }); + + it("lists properties in the order the aggregator gave them", () => { + const { html } = renderPerfReportEmail(report()); + expect(html.indexOf("genrewatch.com")).toBeLessThan(html.indexOf("chovy blog")); + }); + + it("shows each property's bot share", () => { + const { html } = renderPerfReportEmail(report()); + expect(html).toContain("27 human · 90,000 bot (100%)"); + }); + + it("says so plainly when nothing was recorded", () => { + const { html } = renderPerfReportEmail( + report({ traffic: { humans: 0, bots: 0, total: 0, properties: [] } }), + ); + expect(html).toContain("No traffic recorded"); + }); + + it("keeps the weekly subject unchanged", () => { + const { subject } = renderPerfReportEmail(report({ cadence: "weekly" })); + expect(subject).toContain("weekly digest"); + }); +});