crawlproof stats: who arrived and from where, from the CLI

app/api/tracker/v1/stats/route.ts

@@ -0,0 +1,42 @@
+// /api/tracker/v1/stats — a project's traffic for a bearer-token caller.
+//
+//   GET ?site=<hostname|uuid|name>&range=1h&who=humans
+//
+// The dashboard reads the same panels through a session; this is the same data
+// for something holding an API token, which is what `crawlproof stats` calls.
+// Auth matches /api/ads/v1/*: `Authorization: Bearer crp_…`, and the project is
+// always scoped to the token's owner.
+
+import { NextResponse, type NextRequest } from "next/server";
+
+import { serviceClient } from "@/lib/supabase/service";
+import { authenticateBearer } from "@/lib/sp/apiAuth";
+import { projectStats, resolveProject } from "@/lib/tracker/apiStats";
+import { trackerRange } from "@/lib/tracker/ranges";
+import { DEFAULT_WHO, parseWho, WHO_PARAM, whoToKind } from "@/lib/tracker/who";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function GET(req: NextRequest) {
+  const auth = await authenticateBearer(req);
+  if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status });
+
+  const sp = req.nextUrl.searchParams;
+
+  // An unknown `who` is a 400 rather than a quiet fall-back, so a typo cannot
+  // answer a different question than the one asked.
+  const whoParam = sp.get(WHO_PARAM);
+  const who = whoParam === null ? DEFAULT_WHO : parseWho(whoParam);
+  if (!who) {
+    return NextResponse.json({ error: "Unknown who. Expected humans, bots or all." }, { status: 400 });
+  }
+
+  const sb = serviceClient();
+  const resolved = await resolveProject(sb, auth.userId, sp.get("site"));
+  if (!resolved.ok) return NextResponse.json({ error: resolved.error }, { status: resolved.status });
+
+  const range = trackerRange(sp.get("range"));
+  const stats = await projectStats(sb, resolved.project, range, whoToKind(who), who);
+  return NextResponse.json(stats);
+}

cli/index.ts

@@ -6,6 +6,7 @@
 //   crawlproof report <token>
 //   crawlproof sweep
 //   crawlproof track --project=<uuid> --event=<name>
+//   crawlproof stats [site] [--range=1d] [--who=humans]
 //   crawlproof help
 //
 // Currently `audit` runs the rule-based engine locally with no DB/credit
@@ -336,6 +337,57 @@ async function cmdAds(args: Args): Promise<number> {
   return 2;
 }
 
+/**
+ * `crawlproof stats [site]` — who arrived, and from where.
+ *
+ * The question this exists for is "did that post do anything", so the default
+ * window is short and the default audience is humans: over a month, with bots
+ * counted, a launch is invisible inside the crawler traffic.
+ */
+async function cmdStats(args: Args): Promise<number> {
+  const site = args.positional[0] ?? (args.flags.site as string | undefined) ?? process.env.CRAWLPROOF_PROJECT;
+  const range = (args.flags.range as string | undefined) ?? "1d";
+  const who = (args.flags.who as string | undefined) ?? "humans";
+
+  const query = new URLSearchParams({ range, who });
+  if (site) query.set("site", site);
+
+  const { status, json } = await apiCall(args, "GET", `/api/tracker/v1/stats?${query.toString()}`);
+  if (status >= 400) {
+    console.error(`error: ${String(json.error ?? status)}`);
+    return 1;
+  }
+  if (args.flags.json) {
+    console.log(JSON.stringify(json, null, 2));
+    return 0;
+  }
+
+  const project = json.project as { name?: string; url?: string } | undefined;
+  const totals = json.totals as { visitors?: number; pageviews?: number } | undefined;
+  const list = (key: string) => (Array.isArray(json[key]) ? (json[key] as { label: string; value: number }[]) : []);
+
+  process.stdout.write(`${project?.name ?? "project"}  ${range}  ${who}\n`);
+  process.stdout.write(`${totals?.visitors ?? 0} visitors, ${totals?.pageviews ?? 0} pageviews\n`);
+
+  const section = (title: string, items: { label: string; value: number }[]) => {
+    if (!items.length) return;
+    process.stdout.write(`\n${title}\n`);
+    const width = Math.min(46, Math.max(...items.map((i) => i.label.length)));
+    for (const item of items.slice(0, 10)) {
+      process.stdout.write(`  ${item.label.slice(0, width).padEnd(width)}  ${item.value}\n`);
+    }
+  };
+  section("Sources", list("sources"));
+  section("Referrers", list("referrers"));
+  section("Pages", list("pages"));
+
+  // Nothing at all is a real answer, and the likeliest cause is worth naming.
+  if (!(totals?.pageviews ?? 0) && !list("sources").length) {
+    process.stdout.write(`\nNothing in this window. Check the tag is on the page, or widen --range.\n`);
+  }
+  return 0;
+}
+
 async function cmdSlots(args: Args): Promise<number> {
   const sub = args.positional[0];
   if (sub === "create") {
@@ -434,6 +486,12 @@ COMMANDS
   slots list [--json]
       Your slots.
 
+  stats [site] [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all] [--json]
+      Who arrived and from where: sources, referrers and top pages. Defaults
+      to the last day and humans only, because a launch is invisible inside a
+      month of crawler traffic. The site is a hostname, a project id or a
+      project name; with one project it can be left out. Needs an API token.
+
   help
       Print this message.
 
@@ -472,6 +530,8 @@ async function main() {
         return await cmdAds(args);
       case "slots":
         return await cmdSlots(args);
+      case "stats":
+        return await cmdStats(args);
       case "help":
       case "--help":
       case "-h":

lib/tracker/apiStats.ts

@@ -0,0 +1,124 @@
+// Reading a project's tracker numbers as an API caller rather than as a
+// signed-in dashboard user.
+//
+// The dashboard's own stats route is session-authed (`requireProjectAccess`),
+// which a CLI holding a bearer token cannot satisfy. The panels themselves are
+// already pure reads over the service client, so this is only the two things
+// the route needs on top: finding which of the caller's projects was meant,
+// and choosing what a "did my post land" answer consists of.
+
+import type { SupabaseClient } from "@supabase/supabase-js";
+
+import { hostOf } from "@/lib/ads/slots";
+import { fetchPanels, type ListItem, type PanelKey, type PanelPayload } from "@/lib/tracker/panels";
+import type { TrackerRange } from "@/lib/tracker/ranges";
+import type { TrackerKind } from "@/lib/tracker/who";
+
+type Sb = SupabaseClient;
+
+export type ProjectRow = { id: string; name: string; url: string; tracker_enabled?: boolean | null };
+
+/**
+ * The panels that answer "is anybody arriving, and from where".
+ *
+ * Deliberately not every panel: a CLI answer people read in a terminal is the
+ * sources, the pages and the shape over time. Devices and browsers are a
+ * different question and cost another query each.
+ */
+export const STATS_PANELS: PanelKey[] = ["series", "sources", "pages", "referrers"];
+
+export type ResolveResult =
+  | { ok: true; project: ProjectRow }
+  | { ok: false; status: number; error: string };
+
+/**
+ * Which project the caller means: a uuid, a hostname, or their only one.
+ *
+ * Always scoped by `owner_id`, so a token cannot read a project it does not
+ * own even if it guesses the id. A hostname is matched the way the ads API
+ * matches it, so `crawlproof stats nichedb.dev` and `crawlproof slots create
+ * nichedb.dev` mean the same site.
+ */
+export async function resolveProject(sb: Sb, userId: string, site: string | null): Promise<ResolveResult> {
+  const { data, error } = await sb
+    .from("projects")
+    .select("id, name, url, tracker_enabled")
+    .eq("owner_id", userId)
+    .is("archived_at", null);
+  if (error) return { ok: false, status: 500, error: error.message };
+
+  const projects = (data ?? []) as ProjectRow[];
+  if (!projects.length) return { ok: false, status: 404, error: "No projects on this account yet." };
+
+  if (!site) {
+    if (projects.length === 1) return { ok: true, project: projects[0] as ProjectRow };
+    return {
+      ok: false,
+      status: 400,
+      error: `Which site? ${projects.map((p) => p.name).join(", ")}`,
+    };
+  }
+
+  const wanted = site.trim();
+  const byId = projects.find((p) => p.id === wanted);
+  if (byId) return { ok: true, project: byId };
+
+  // A bare hostname is not a URL, so try it as one before giving up on it.
+  const host = hostOf(wanted) ?? hostOf(`https://${wanted}`);
+  if (host) {
+    const byHost = projects.find((p) => {
+      const projectHost = hostOf(p.url ?? "");
+      return projectHost !== null && projectHost === host;
+    });
+    if (byHost) return { ok: true, project: byHost };
+  }
+
+  const byName = projects.find((p) => p.name.toLowerCase() === wanted.toLowerCase());
+  if (byName) return { ok: true, project: byName };
+
+  return { ok: false, status: 404, error: `No project for "${site}". Yours: ${projects.map((p) => p.name).join(", ")}` };
+}
+
+export type StatsAnswer = {
+  project: { id: string; name: string; url: string };
+  range: string;
+  who: string;
+  totals: { visitors: number; pageviews: number };
+  sources: ListItem[];
+  referrers: ListItem[];
+  pages: ListItem[];
+};
+
+const asList = (payload: PanelPayload | undefined): ListItem[] => (Array.isArray(payload) ? payload : []);
+
+/** Sum a series payload's points into the two numbers a summary line needs. */
+export function totalsFromSeries(payload: PanelPayload | undefined): { visitors: number; pageviews: number } {
+  if (!payload || Array.isArray(payload)) return { visitors: 0, pageviews: 0 };
+  const points = (payload as { points?: Record<string, unknown>[] }).points ?? [];
+  let visitors = 0;
+  let pageviews = 0;
+  for (const point of points) {
+    visitors += Number(point.visitors ?? point.humans ?? 0) || 0;
+    pageviews += Number(point.pageviews ?? 0) || 0;
+  }
+  return { visitors, pageviews };
+}
+
+export async function projectStats(
+  sb: Sb,
+  project: ProjectRow,
+  range: TrackerRange,
+  kind: TrackerKind | null,
+  who: string,
+): Promise<StatsAnswer> {
+  const panels = await fetchPanels(sb, project.id, STATS_PANELS, range, kind);
+  return {
+    project: { id: project.id, name: project.name, url: project.url },
+    range: range.key,
+    who,
+    totals: totalsFromSeries(panels.series),
+    sources: asList(panels.sources),
+    referrers: asList(panels.referrers),
+    pages: asList(panels.pages),
+  };
+}

tests/tracker-api-stats.test.ts

@@ -0,0 +1,99 @@
+/**
+ * `crawlproof stats` reads a project the caller owns.
+ *
+ * The two things worth pinning are the ones that decide whether an answer is
+ * the right answer: which project a name resolves to, and that the resolution
+ * is always scoped by owner so a token cannot read somebody else's site by
+ * guessing its id.
+ */
+import { describe, expect, it } from "vitest";
+
+import { resolveProject, totalsFromSeries } from "@/lib/tracker/apiStats";
+
+type Row = { id: string; name: string; url: string };
+
+/** A stand-in for the one query resolveProject makes. */
+function client(rows: Row[], captured: Record<string, unknown> = {}) {
+  const builder = {
+    select: () => builder,
+    eq: (column: string, value: unknown) => {
+      captured[column] = value;
+      return builder;
+    },
+    is: () => Promise.resolve({ data: rows, error: null }),
+  };
+  return { from: () => builder, captured } as never;
+}
+
+const rows: Row[] = [
+  { id: "11111111-1111-1111-1111-111111111111", name: "dev.profullstack.com", url: "https://dev.profullstack.com/~anthony/blog/" },
+  { id: "22222222-2222-2222-2222-222222222222", name: "nichedb.dev", url: "https://nichedb.dev/" },
+];
+
+describe("resolveProject", () => {
+  it("finds a project by id, bare hostname and name", async () => {
+    for (const site of ["22222222-2222-2222-2222-222222222222", "nichedb.dev", "https://nichedb.dev/pricing", "NicheDB.dev"]) {
+      const result = await resolveProject(client(rows), "user-1", site);
+      expect(result.ok, site).toBe(true);
+      if (result.ok) expect(result.project.name, site).toBe("nichedb.dev");
+    }
+  });
+
+  it("matches a hostname that carries a path and a www prefix", async () => {
+    const result = await resolveProject(client(rows), "user-1", "www.dev.profullstack.com");
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.project.name).toBe("dev.profullstack.com");
+  });
+
+  // The whole point of scoping: the query is always filtered to the caller.
+  it("always filters by owner_id", async () => {
+    const captured: Record<string, unknown> = {};
+    await resolveProject(client(rows, captured), "user-1", "nichedb.dev");
+    expect(captured.owner_id).toBe("user-1");
+  });
+
+  it("asks which site when there are several and none was named", async () => {
+    const result = await resolveProject(client(rows), "user-1", null);
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.status).toBe(400);
+      expect(result.error).toMatch(/nichedb\.dev/);
+    }
+  });
+
+  it("needs no name when the account has exactly one project", async () => {
+    const result = await resolveProject(client([rows[0] as Row]), "user-1", null);
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.project.name).toBe("dev.profullstack.com");
+  });
+
+  it("says so when the name matches nothing, and lists what does", async () => {
+    const result = await resolveProject(client(rows), "user-1", "someone-elses-site.com");
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.status).toBe(404);
+      expect(result.error).toMatch(/nichedb\.dev/);
+    }
+  });
+
+  it("reports an account with no projects rather than pretending", async () => {
+    const result = await resolveProject(client([]), "user-1", null);
+    expect(result.ok).toBe(false);
+    if (!result.ok) expect(result.status).toBe(404);
+  });
+});
+
+describe("totalsFromSeries", () => {
+  it("sums the points, counting humans when visitors is absent", () => {
+    expect(
+      totalsFromSeries({ points: [{ visitors: 3, pageviews: 9 }, { visitors: 1, pageviews: 2 }] } as never),
+    ).toEqual({ visitors: 4, pageviews: 11 });
+    expect(totalsFromSeries({ points: [{ humans: 2, pageviews: 5 }] } as never)).toEqual({ visitors: 2, pageviews: 5 });
+  });
+
+  it("is zero for a list payload or nothing at all, rather than NaN", () => {
+    expect(totalsFromSeries(undefined)).toEqual({ visitors: 0, pageviews: 0 });
+    expect(totalsFromSeries([] as never)).toEqual({ visitors: 0, pageviews: 0 });
+    expect(totalsFromSeries({ points: [{ pageviews: "4" }] } as never)).toEqual({ visitors: 0, pageviews: 4 });
+  });
+});