From 25e1109f5b550809d3c889dd8ddba4913e67daa9 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 02:51:36 +0000 Subject: [PATCH] crawlproof stats: who arrived and from where, from the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard's stats route is session-authed, so a CLI holding an API token could read ads and slots but nothing about traffic — and "did that post do anything" is the question a terminal is best placed to answer. GET /api/tracker/v1/stats?site&range&who serves the same panels the dashboard renders, with the bearer auth the ads API already uses. The project is resolved by hostname, id or name and always scoped by owner_id, so a token cannot read a site it does not own by guessing an id. `crawlproof stats [site]` prints sources, referrers and top pages. Defaults are the last day and humans only, because over a month with bots counted a launch is invisible inside the crawler traffic. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013GX6xEcjGFxcvcMCFLhQng --- app/api/tracker/v1/stats/route.ts | 42 ++++++++++ cli/index.ts | 60 +++++++++++++++ lib/tracker/apiStats.ts | 124 ++++++++++++++++++++++++++++++ tests/tracker-api-stats.test.ts | 99 ++++++++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 app/api/tracker/v1/stats/route.ts create mode 100644 lib/tracker/apiStats.ts create mode 100644 tests/tracker-api-stats.test.ts diff --git a/app/api/tracker/v1/stats/route.ts b/app/api/tracker/v1/stats/route.ts new file mode 100644 index 0000000..d862a5a --- /dev/null +++ b/app/api/tracker/v1/stats/route.ts @@ -0,0 +1,42 @@ +// /api/tracker/v1/stats — a project's traffic for a bearer-token caller. +// +// GET ?site=&range=1h&who=humans +// +// The dashboard reads the same panels through a session; this is the same data +// for something holding an API token, which is what `crawlproof stats` calls. +// Auth matches /api/ads/v1/*: `Authorization: Bearer crp_…`, and the project is +// always scoped to the token's owner. + +import { NextResponse, type NextRequest } from "next/server"; + +import { serviceClient } from "@/lib/supabase/service"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { projectStats, resolveProject } from "@/lib/tracker/apiStats"; +import { trackerRange } from "@/lib/tracker/ranges"; +import { DEFAULT_WHO, parseWho, WHO_PARAM, whoToKind } from "@/lib/tracker/who"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +export async function GET(req: NextRequest) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + + const sp = req.nextUrl.searchParams; + + // An unknown `who` is a 400 rather than a quiet fall-back, so a typo cannot + // answer a different question than the one asked. + const whoParam = sp.get(WHO_PARAM); + const who = whoParam === null ? DEFAULT_WHO : parseWho(whoParam); + if (!who) { + return NextResponse.json({ error: "Unknown who. Expected humans, bots or all." }, { status: 400 }); + } + + const sb = serviceClient(); + const resolved = await resolveProject(sb, auth.userId, sp.get("site")); + if (!resolved.ok) return NextResponse.json({ error: resolved.error }, { status: resolved.status }); + + const range = trackerRange(sp.get("range")); + const stats = await projectStats(sb, resolved.project, range, whoToKind(who), who); + return NextResponse.json(stats); +} diff --git a/cli/index.ts b/cli/index.ts index 81ac2d1..105420f 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -6,6 +6,7 @@ // crawlproof report // crawlproof sweep // crawlproof track --project= --event= +// crawlproof stats [site] [--range=1d] [--who=humans] // crawlproof help // // Currently `audit` runs the rule-based engine locally with no DB/credit @@ -336,6 +337,57 @@ async function cmdAds(args: Args): Promise { return 2; } +/** + * `crawlproof stats [site]` — who arrived, and from where. + * + * The question this exists for is "did that post do anything", so the default + * window is short and the default audience is humans: over a month, with bots + * counted, a launch is invisible inside the crawler traffic. + */ +async function cmdStats(args: Args): Promise { + const site = args.positional[0] ?? (args.flags.site as string | undefined) ?? process.env.CRAWLPROOF_PROJECT; + const range = (args.flags.range as string | undefined) ?? "1d"; + const who = (args.flags.who as string | undefined) ?? "humans"; + + const query = new URLSearchParams({ range, who }); + if (site) query.set("site", site); + + const { status, json } = await apiCall(args, "GET", `/api/tracker/v1/stats?${query.toString()}`); + if (status >= 400) { + console.error(`error: ${String(json.error ?? status)}`); + return 1; + } + if (args.flags.json) { + console.log(JSON.stringify(json, null, 2)); + return 0; + } + + const project = json.project as { name?: string; url?: string } | undefined; + const totals = json.totals as { visitors?: number; pageviews?: number } | undefined; + const list = (key: string) => (Array.isArray(json[key]) ? (json[key] as { label: string; value: number }[]) : []); + + process.stdout.write(`${project?.name ?? "project"} ${range} ${who}\n`); + process.stdout.write(`${totals?.visitors ?? 0} visitors, ${totals?.pageviews ?? 0} pageviews\n`); + + const section = (title: string, items: { label: string; value: number }[]) => { + if (!items.length) return; + process.stdout.write(`\n${title}\n`); + const width = Math.min(46, Math.max(...items.map((i) => i.label.length))); + for (const item of items.slice(0, 10)) { + process.stdout.write(` ${item.label.slice(0, width).padEnd(width)} ${item.value}\n`); + } + }; + section("Sources", list("sources")); + section("Referrers", list("referrers")); + section("Pages", list("pages")); + + // Nothing at all is a real answer, and the likeliest cause is worth naming. + if (!(totals?.pageviews ?? 0) && !list("sources").length) { + process.stdout.write(`\nNothing in this window. Check the tag is on the page, or widen --range.\n`); + } + return 0; +} + async function cmdSlots(args: Args): Promise { const sub = args.positional[0]; if (sub === "create") { @@ -434,6 +486,12 @@ COMMANDS slots list [--json] Your slots. + stats [site] [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all] [--json] + Who arrived and from where: sources, referrers and top pages. Defaults + to the last day and humans only, because a launch is invisible inside a + month of crawler traffic. The site is a hostname, a project id or a + project name; with one project it can be left out. Needs an API token. + help Print this message. @@ -472,6 +530,8 @@ async function main() { return await cmdAds(args); case "slots": return await cmdSlots(args); + case "stats": + return await cmdStats(args); case "help": case "--help": case "-h": diff --git a/lib/tracker/apiStats.ts b/lib/tracker/apiStats.ts new file mode 100644 index 0000000..84afe43 --- /dev/null +++ b/lib/tracker/apiStats.ts @@ -0,0 +1,124 @@ +// Reading a project's tracker numbers as an API caller rather than as a +// signed-in dashboard user. +// +// The dashboard's own stats route is session-authed (`requireProjectAccess`), +// which a CLI holding a bearer token cannot satisfy. The panels themselves are +// already pure reads over the service client, so this is only the two things +// the route needs on top: finding which of the caller's projects was meant, +// and choosing what a "did my post land" answer consists of. + +import type { SupabaseClient } from "@supabase/supabase-js"; + +import { hostOf } from "@/lib/ads/slots"; +import { fetchPanels, type ListItem, type PanelKey, type PanelPayload } from "@/lib/tracker/panels"; +import type { TrackerRange } from "@/lib/tracker/ranges"; +import type { TrackerKind } from "@/lib/tracker/who"; + +type Sb = SupabaseClient; + +export type ProjectRow = { id: string; name: string; url: string; tracker_enabled?: boolean | null }; + +/** + * The panels that answer "is anybody arriving, and from where". + * + * Deliberately not every panel: a CLI answer people read in a terminal is the + * sources, the pages and the shape over time. Devices and browsers are a + * different question and cost another query each. + */ +export const STATS_PANELS: PanelKey[] = ["series", "sources", "pages", "referrers"]; + +export type ResolveResult = + | { ok: true; project: ProjectRow } + | { ok: false; status: number; error: string }; + +/** + * Which project the caller means: a uuid, a hostname, or their only one. + * + * Always scoped by `owner_id`, so a token cannot read a project it does not + * own even if it guesses the id. A hostname is matched the way the ads API + * matches it, so `crawlproof stats nichedb.dev` and `crawlproof slots create + * nichedb.dev` mean the same site. + */ +export async function resolveProject(sb: Sb, userId: string, site: string | null): Promise { + const { data, error } = await sb + .from("projects") + .select("id, name, url, tracker_enabled") + .eq("owner_id", userId) + .is("archived_at", null); + if (error) return { ok: false, status: 500, error: error.message }; + + const projects = (data ?? []) as ProjectRow[]; + if (!projects.length) return { ok: false, status: 404, error: "No projects on this account yet." }; + + if (!site) { + if (projects.length === 1) return { ok: true, project: projects[0] as ProjectRow }; + return { + ok: false, + status: 400, + error: `Which site? ${projects.map((p) => p.name).join(", ")}`, + }; + } + + const wanted = site.trim(); + const byId = projects.find((p) => p.id === wanted); + if (byId) return { ok: true, project: byId }; + + // A bare hostname is not a URL, so try it as one before giving up on it. + const host = hostOf(wanted) ?? hostOf(`https://${wanted}`); + if (host) { + const byHost = projects.find((p) => { + const projectHost = hostOf(p.url ?? ""); + return projectHost !== null && projectHost === host; + }); + if (byHost) return { ok: true, project: byHost }; + } + + const byName = projects.find((p) => p.name.toLowerCase() === wanted.toLowerCase()); + if (byName) return { ok: true, project: byName }; + + return { ok: false, status: 404, error: `No project for "${site}". Yours: ${projects.map((p) => p.name).join(", ")}` }; +} + +export type StatsAnswer = { + project: { id: string; name: string; url: string }; + range: string; + who: string; + totals: { visitors: number; pageviews: number }; + sources: ListItem[]; + referrers: ListItem[]; + pages: ListItem[]; +}; + +const asList = (payload: PanelPayload | undefined): ListItem[] => (Array.isArray(payload) ? payload : []); + +/** Sum a series payload's points into the two numbers a summary line needs. */ +export function totalsFromSeries(payload: PanelPayload | undefined): { visitors: number; pageviews: number } { + if (!payload || Array.isArray(payload)) return { visitors: 0, pageviews: 0 }; + const points = (payload as { points?: Record[] }).points ?? []; + let visitors = 0; + let pageviews = 0; + for (const point of points) { + visitors += Number(point.visitors ?? point.humans ?? 0) || 0; + pageviews += Number(point.pageviews ?? 0) || 0; + } + return { visitors, pageviews }; +} + +export async function projectStats( + sb: Sb, + project: ProjectRow, + range: TrackerRange, + kind: TrackerKind | null, + who: string, +): Promise { + const panels = await fetchPanels(sb, project.id, STATS_PANELS, range, kind); + return { + project: { id: project.id, name: project.name, url: project.url }, + range: range.key, + who, + totals: totalsFromSeries(panels.series), + sources: asList(panels.sources), + referrers: asList(panels.referrers), + pages: asList(panels.pages), + }; +} diff --git a/tests/tracker-api-stats.test.ts b/tests/tracker-api-stats.test.ts new file mode 100644 index 0000000..7099958 --- /dev/null +++ b/tests/tracker-api-stats.test.ts @@ -0,0 +1,99 @@ +/** + * `crawlproof stats` reads a project the caller owns. + * + * The two things worth pinning are the ones that decide whether an answer is + * the right answer: which project a name resolves to, and that the resolution + * is always scoped by owner so a token cannot read somebody else's site by + * guessing its id. + */ +import { describe, expect, it } from "vitest"; + +import { resolveProject, totalsFromSeries } from "@/lib/tracker/apiStats"; + +type Row = { id: string; name: string; url: string }; + +/** A stand-in for the one query resolveProject makes. */ +function client(rows: Row[], captured: Record = {}) { + const builder = { + select: () => builder, + eq: (column: string, value: unknown) => { + captured[column] = value; + return builder; + }, + is: () => Promise.resolve({ data: rows, error: null }), + }; + return { from: () => builder, captured } as never; +} + +const rows: Row[] = [ + { id: "11111111-1111-1111-1111-111111111111", name: "dev.profullstack.com", url: "https://dev.profullstack.com/~anthony/blog/" }, + { id: "22222222-2222-2222-2222-222222222222", name: "nichedb.dev", url: "https://nichedb.dev/" }, +]; + +describe("resolveProject", () => { + it("finds a project by id, bare hostname and name", async () => { + for (const site of ["22222222-2222-2222-2222-222222222222", "nichedb.dev", "https://nichedb.dev/pricing", "NicheDB.dev"]) { + const result = await resolveProject(client(rows), "user-1", site); + expect(result.ok, site).toBe(true); + if (result.ok) expect(result.project.name, site).toBe("nichedb.dev"); + } + }); + + it("matches a hostname that carries a path and a www prefix", async () => { + const result = await resolveProject(client(rows), "user-1", "www.dev.profullstack.com"); + expect(result.ok).toBe(true); + if (result.ok) expect(result.project.name).toBe("dev.profullstack.com"); + }); + + // The whole point of scoping: the query is always filtered to the caller. + it("always filters by owner_id", async () => { + const captured: Record = {}; + await resolveProject(client(rows, captured), "user-1", "nichedb.dev"); + expect(captured.owner_id).toBe("user-1"); + }); + + it("asks which site when there are several and none was named", async () => { + const result = await resolveProject(client(rows), "user-1", null); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(400); + expect(result.error).toMatch(/nichedb\.dev/); + } + }); + + it("needs no name when the account has exactly one project", async () => { + const result = await resolveProject(client([rows[0] as Row]), "user-1", null); + expect(result.ok).toBe(true); + if (result.ok) expect(result.project.name).toBe("dev.profullstack.com"); + }); + + it("says so when the name matches nothing, and lists what does", async () => { + const result = await resolveProject(client(rows), "user-1", "someone-elses-site.com"); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(404); + expect(result.error).toMatch(/nichedb\.dev/); + } + }); + + it("reports an account with no projects rather than pretending", async () => { + const result = await resolveProject(client([]), "user-1", null); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.status).toBe(404); + }); +}); + +describe("totalsFromSeries", () => { + it("sums the points, counting humans when visitors is absent", () => { + expect( + totalsFromSeries({ points: [{ visitors: 3, pageviews: 9 }, { visitors: 1, pageviews: 2 }] } as never), + ).toEqual({ visitors: 4, pageviews: 11 }); + expect(totalsFromSeries({ points: [{ humans: 2, pageviews: 5 }] } as never)).toEqual({ visitors: 2, pageviews: 5 }); + }); + + it("is zero for a list payload or nothing at all, rather than NaN", () => { + expect(totalsFromSeries(undefined)).toEqual({ visitors: 0, pageviews: 0 }); + expect(totalsFromSeries([] as never)).toEqual({ visitors: 0, pageviews: 0 }); + expect(totalsFromSeries({ points: [{ pageviews: "4" }] } as never)).toEqual({ visitors: 0, pageviews: 4 }); + }); +});