diff --git a/app/api/ads/v1/campaigns/route.ts b/app/api/ads/v1/campaigns/route.ts new file mode 100644 index 00000000..3ed13408 --- /dev/null +++ b/app/api/ads/v1/campaigns/route.ts @@ -0,0 +1,59 @@ +// /api/ads/v1/campaigns — campaigns for a bearer-token caller. +// +// POST { url, name?, daily_budget_cents?, bid_credits?, status? } +// Read the page, write the creatives, save the campaign. Active unless +// status is "draft". A live campaign for the same URL is returned +// instead of a twin, with `existing: true`. +// GET ?limit=20 +// The caller's campaigns, newest first. +// +// Same auth as /api/sp/v1/* and the MCP server: `Authorization: Bearer crp_…` +// from Social → API tokens. This is what `crawlproof ads` and the myna +// crawlproof plugin call. + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { createCampaignForUrl, listCampaigns, parseCampaignRequest } from "@/lib/ads/campaigns"; +import { env } from "@/lib/env"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; +// Reading the page and writing four creatives takes a while. +export const maxDuration = 120; + +const siteUrl = () => (env.siteUrl || "https://crawlproof.com").replace(/\/$/, ""); + +export async function POST(req: NextRequest) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + + let body: Record; + try { + body = (await req.json()) as Record; + } catch { + return NextResponse.json({ error: "Body must be JSON." }, { status: 400 }); + } + const parsed = parseCampaignRequest(body ?? {}); + if (!parsed.ok) return NextResponse.json({ error: parsed.error }, { status: 400 }); + + const sb = serviceClient(); + const { data: profile } = await sb.from("profiles").select("email").eq("id", auth.userId).maybeSingle(); + const result = await createCampaignForUrl({ + sb, + userId: auth.userId, + email: (profile as { email?: string | null } | null)?.email ?? null, + request: parsed.request, + siteUrl: siteUrl(), + }); + if (!result.ok) return NextResponse.json({ error: result.error }, { status: result.status }); + return NextResponse.json(result.campaign, { status: result.campaign.existing ? 200 : 201 }); +} + +export async function GET(req: NextRequest) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + const limit = Number(new URL(req.url).searchParams.get("limit")) || 20; + const campaigns = await listCampaigns({ sb: serviceClient(), userId: auth.userId, limit, siteUrl: siteUrl() }); + return NextResponse.json({ campaigns }); +} diff --git a/app/api/ads/v1/slots/route.ts b/app/api/ads/v1/slots/route.ts new file mode 100644 index 00000000..32e21318 --- /dev/null +++ b/app/api/ads/v1/slots/route.ts @@ -0,0 +1,46 @@ +// /api/ads/v1/slots — publisher slots for a bearer-token caller. +// +// POST { site, placement?, formats?, format?, status?, enable_tracking? } +// A slot on the site named by hostname or URL. The site's project is +// found or created with the tracker on. Returns the slot with the two +// tags to paste (`embed`, `tracker`). A site that already has a slot +// gets that slot back, with `existing: true`. +// GET The caller's slots, newest first. +// +// Same auth as /api/ads/v1/campaigns. This is what `crawlproof slots` calls. + +import { NextResponse, type NextRequest } from "next/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { createSlotForSite, listSlots, parseSlotRequest } from "@/lib/ads/slots"; +import { env } from "@/lib/env"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +const siteUrl = () => (env.siteUrl || "https://crawlproof.com").replace(/\/$/, ""); + +export async function POST(req: NextRequest) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + + let body: Record; + try { + body = (await req.json()) as Record; + } catch { + return NextResponse.json({ error: "Body must be JSON." }, { status: 400 }); + } + const parsed = parseSlotRequest(body ?? {}); + if (!parsed.ok) return NextResponse.json({ error: parsed.error }, { status: 400 }); + + const result = await createSlotForSite({ sb: serviceClient(), userId: auth.userId, request: parsed.request, siteUrl: siteUrl() }); + if (!result.ok) return NextResponse.json({ error: result.error }, { status: result.status }); + return NextResponse.json(result.slot, { status: result.slot.existing ? 200 : 201 }); +} + +export async function GET(req: NextRequest) { + const auth = await authenticateBearer(req); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + const slots = await listSlots({ sb: serviceClient(), userId: auth.userId, siteUrl: siteUrl() }); + return NextResponse.json({ slots }); +} diff --git a/app/api/track/route.ts b/app/api/track/route.ts index d43c5665..0b104a92 100644 --- a/app/api/track/route.ts +++ b/app/api/track/route.ts @@ -182,7 +182,7 @@ async function ingest(request: NextRequest, parseBody: boolean) { const gate = await gateAgent(sb, site, userAgent); if (gate.action !== "allow") return refuse(gate); - const { bucket, isAi } = categorize({ referrer, userAgent }); + const { bucket, isAi } = categorize({ referrer, userAgent, url: pageUrl }); // Which side of the human / bot line this hit counts on. The bucket table // carries the whole bucket; the other rollups record only this, so the // stats page can split every breakdown, not just the headline. diff --git a/cli/index.ts b/cli/index.ts index 2b3dcfec..508621f4 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -177,6 +177,157 @@ async function cmdSweep(args: Args): Promise { return 0; } +// ---------------------------------------------------------------- ads / slots +// +// Both talk to /api/ads/v1/* with a CrawlProof API token (Social → API +// tokens), the same token the MCP server and the myna plugin use. + +function apiToken(args: Args): string | null { + const token = (args.flags.token as string | undefined) ?? process.env.CRAWLPROOF_TOKEN ?? null; + return token && token.trim() ? token.trim() : null; +} + +function apiBase(args: Args): string { + const base = + (args.flags.base as string | undefined) ?? + process.env.CRAWLPROOF_SITE_URL ?? + "https://crawlproof.com"; + return base.replace(/\/$/, ""); +} + +async function apiCall( + args: Args, + method: "GET" | "POST", + path: string, + body?: Record, +): Promise<{ status: number; json: Record }> { + const token = apiToken(args); + if (!token) { + throw new Error("No API token. Set CRAWLPROOF_TOKEN or pass --token (Social → API tokens in the app)."); + } + const res = await fetch(`${apiBase(args)}${path}`, { + method, + headers: { + authorization: `Bearer ${token}`, + accept: "application/json", + ...(body ? { "content-type": "application/json" } : {}), + }, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + let json: Record = {}; + try { + json = text ? (JSON.parse(text) as Record) : {}; + } catch { + json = { error: text.slice(0, 200) }; + } + return { status: res.status, json }; +} + +/** The request body `crawlproof ads create` sends, from its flags. Pure, for tests. */ +export function campaignBodyFromArgs(args: Args): Record { + const body: Record = { url: args.positional[1] }; + if (typeof args.flags.name === "string") body.name = args.flags.name; + if (typeof args.flags.budget === "string") body.daily_budget_cents = Number(args.flags.budget); + if (typeof args.flags.bid === "string") body.bid_credits = Number(args.flags.bid); + body.status = args.flags.draft ? "draft" : "active"; + return body; +} + +/** The request body `crawlproof slots create` sends, from its flags. Pure, for tests. */ +export function slotBodyFromArgs(args: Args): Record { + const body: Record = { site: args.positional[1] }; + if (typeof args.flags.placement === "string") body.placement = args.flags.placement; + if (typeof args.flags.format === "string") body.format = args.flags.format; + if (typeof args.flags.formats === "string") body.formats = args.flags.formats.split(",").map((f) => f.trim()); + if (args.flags.inactive) body.status = "inactive"; + if (args.flags["no-tracking"]) body.enable_tracking = false; + return body; +} + +async function cmdAds(args: Args): Promise { + const sub = args.positional[0]; + if (sub === "create") { + if (!args.positional[1]) { + console.error("usage: crawlproof ads create [--name=N] [--budget=CENTS] [--bid=CREDITS] [--draft] [--json]"); + return 2; + } + const { status, json } = await apiCall(args, "POST", "/api/ads/v1/campaigns", campaignBodyFromArgs(args)); + if (status >= 400) { + console.error(`ads create failed: ${status} ${json.error ?? ""}`); + return 1; + } + if (args.flags.json) { + process.stdout.write(`${JSON.stringify(json, null, 2)}\n`); + } else { + const existing = json.existing ? " (already existed)" : ""; + process.stdout.write(`${json.status} ${json.ref_slug} ${json.name}${existing}\n ${json.destination_url}\n ${json.dashboard_url ?? ""}\n`); + } + return 0; + } + if (sub === "list" || sub === undefined) { + const limit = (args.flags.limit as string | undefined) ?? "20"; + const { status, json } = await apiCall(args, "GET", `/api/ads/v1/campaigns?limit=${encodeURIComponent(limit)}`); + if (status >= 400) { + console.error(`ads list failed: ${status} ${json.error ?? ""}`); + return 1; + } + const campaigns = (json.campaigns as Record[]) ?? []; + if (args.flags.json) { + process.stdout.write(`${JSON.stringify(campaigns, null, 2)}\n`); + return 0; + } + if (!campaigns.length) process.stdout.write("No campaigns yet.\n"); + for (const c of campaigns) { + process.stdout.write(`${String(c.status).padEnd(8)} ${String(c.ref_slug).padEnd(20)} ${c.name} ${c.destination_url}\n`); + } + return 0; + } + console.error(`unknown: crawlproof ads ${sub} (expected: create | list)`); + return 2; +} + +async function cmdSlots(args: Args): Promise { + const sub = args.positional[0]; + if (sub === "create") { + if (!args.positional[1]) { + console.error("usage: crawlproof slots create [--placement=inline] [--format=text_link] [--formats=a,b] [--inactive] [--no-tracking] [--json]"); + return 2; + } + const { status, json } = await apiCall(args, "POST", "/api/ads/v1/slots", slotBodyFromArgs(args)); + if (status >= 400) { + console.error(`slots create failed: ${status} ${json.error ?? ""}`); + return 1; + } + if (args.flags.json) { + process.stdout.write(`${JSON.stringify(json, null, 2)}\n`); + } else { + const existing = json.existing ? " (already existed)" : ""; + process.stdout.write(`${json.status} slot ${json.id} on ${json.site}${existing}\n\nPaste before :\n\n${json.embed}\n`); + } + return 0; + } + if (sub === "list" || sub === undefined) { + const { status, json } = await apiCall(args, "GET", "/api/ads/v1/slots"); + if (status >= 400) { + console.error(`slots list failed: ${status} ${json.error ?? ""}`); + return 1; + } + const slots = (json.slots as Record[]) ?? []; + if (args.flags.json) { + process.stdout.write(`${JSON.stringify(slots, null, 2)}\n`); + return 0; + } + if (!slots.length) process.stdout.write("No slots yet.\n"); + for (const s of slots) { + process.stdout.write(`${String(s.status).padEnd(9)} ${s.id} ${s.site} ${s.placement}\n`); + } + return 0; + } + console.error(`unknown: crawlproof slots ${sub} (expected: create | list)`); + return 2; +} + function help() { console.log(`crawlproof — AEO audit CLI (stub) @@ -207,13 +358,31 @@ COMMANDS Defaults --event to "pageview". Project id can also come from CRAWLPROOF_PROJECT. Override host with CRAWLPROOF_SITE_URL. + ads create [--name=N] [--budget=CENTS] [--bid=CREDITS] [--draft] [--json] + Run an ad campaign for a URL: CrawlProof reads the page, writes the + creatives and starts serving (active unless --draft). A URL that + already has a live campaign gets that campaign back. Needs an API + token (CRAWLPROOF_TOKEN, from Social → API tokens). + + ads list [--limit=20] [--json] + Your campaigns, newest first. + + slots create [--placement=inline] [--format=text_link] [--formats=a,b] [--inactive] [--no-tracking] [--json] + A publisher slot on a site you own, named by hostname or URL. The + site's project is found or created with the stats tracker on, and + the output is the two tags to paste before . + + slots list [--json] + Your slots. + help Print this message. ENV ANTHROPIC_API_KEY Required for --engine=claude. - CRAWLPROOF_SITE_URL Override the API base URL for 'report', 'sweep', and 'track'. + CRAWLPROOF_SITE_URL Override the API base URL for 'report', 'sweep', 'track', 'ads' and 'slots'. CRAWLPROOF_PROJECT Default project UUID for 'track'. + CRAWLPROOF_TOKEN API token (crp_…) for 'ads' and 'slots'; --token overrides. CRON_SECRET Required for 'sweep'. EXAMPLES @@ -223,6 +392,8 @@ EXAMPLES CRAWLPROOF_SITE_URL=http://localhost:3000 crawlproof sweep CRAWLPROOF_SITE_URL=http://localhost:3000 crawlproof sweep --target=autoblog crawlproof track --project=ac4e0a7d-... --event=signup --target=hero_cta + CRAWLPROOF_TOKEN=crp_... crawlproof ads create https://nichedb.dev --name "NicheDB" + CRAWLPROOF_TOKEN=crp_... crawlproof slots create nichedb.dev `); } @@ -238,6 +409,10 @@ async function main() { return await cmdSweep(args); case "track": return await cmdTrack(args); + case "ads": + return await cmdAds(args); + case "slots": + return await cmdSlots(args); case "help": case "--help": case "-h": diff --git a/lib/ads/campaign-request.ts b/lib/ads/campaign-request.ts new file mode 100644 index 00000000..e1363b0e --- /dev/null +++ b/lib/ads/campaign-request.ts @@ -0,0 +1,57 @@ +// The shape of a campaign request, parsed without touching a database. +// +// Kept apart from lib/ads/campaigns.ts because that module reaches the +// generator and the org helper, which are server-only; this one is imported +// by tests and could be by a client. + +import { isAllowedTargetUrl } from "@/lib/rateLimit"; + +export type CampaignStatus = "active" | "draft"; + +export type CampaignRequest = { + url: string; + name?: string; + dailyBudgetCents?: number; + bidCredits?: number; + status?: CampaignStatus; +}; + +export function domainOf(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, ""); + } catch { + return url; + } +} + +/** + * Normalise what a caller sent. Pure, so it is testable without a database: + * the clamps here are the same ones saveCampaign applies. + */ +export function parseCampaignRequest(body: Record): { ok: true; request: CampaignRequest; url: string } | { ok: false; error: string } { + const rawUrl = typeof body.url === "string" ? body.url : ""; + const check = isAllowedTargetUrl(rawUrl); + if (!check.ok) return { ok: false, error: check.reason }; + + const budgetRaw = body.daily_budget_cents ?? body.dailyBudgetCents; + const bidRaw = body.bid_credits ?? body.bidCredits; + const statusRaw = body.status; + if (statusRaw !== undefined && statusRaw !== "active" && statusRaw !== "draft") { + return { ok: false, error: 'status must be "active" or "draft".' }; + } + + const request: CampaignRequest = { url: check.url }; + if (typeof body.name === "string" && body.name.trim()) request.name = body.name.trim().slice(0, 120); + if (budgetRaw !== undefined && budgetRaw !== null) { + const n = Number(budgetRaw); + if (!Number.isFinite(n) || n < 0) return { ok: false, error: "daily_budget_cents must be a non-negative number." }; + request.dailyBudgetCents = Math.round(n); + } + if (bidRaw !== undefined && bidRaw !== null) { + const n = Number(bidRaw); + if (!Number.isFinite(n) || n < 1) return { ok: false, error: "bid_credits must be at least 1." }; + request.bidCredits = Math.min(200, Math.round(n)); + } + if (statusRaw === "active" || statusRaw === "draft") request.status = statusRaw; + return { ok: true, request, url: check.url }; +} diff --git a/lib/ads/campaigns.ts b/lib/ads/campaigns.ts new file mode 100644 index 00000000..04eea188 --- /dev/null +++ b/lib/ads/campaigns.ts @@ -0,0 +1,181 @@ +// Campaigns created from outside the dashboard. +// +// The dashboard's saveCampaign (app/actions/ads.ts) takes creatives the person +// already previewed and edited. The API and the CLI have no preview step: a +// caller hands over a URL and expects a running campaign back, which is what +// myna does the moment it publishes a blog post. So this reads the page, +// writes the creatives, and saves — one call, service-role client, scoped by +// the owner id the bearer token resolved to. +// +// Idempotent on the destination: a second call for a URL that already has a +// live campaign returns that campaign rather than minting a twin. A blog post +// announced twice should not be paying for two campaigns. + +import type { SupabaseClient } from "@supabase/supabase-js"; +import { parseCampaignRequest, domainOf, type CampaignRequest, type CampaignStatus } from "@/lib/ads/campaign-request"; + +export { parseCampaignRequest, domainOf, type CampaignRequest, type CampaignStatus }; +import { getOrCreateDefaultOrg } from "@/lib/orgs"; +import { generateAdCreatives, cleanSummary, type AdCreative, type AdSummary } from "@/lib/ads/creative"; +import { DEFAULT_BID_CREDITS } from "@/lib/ads/pricing"; + +export type CampaignSummary = { + id: string; + ref_slug: string; + name: string; + status: string; + destination_url: string; + daily_budget_cents: number; + bid_credits: number | null; + created_at?: string; + creatives?: number; + dashboard_url?: string; + /** True when a live campaign for this URL already existed and was returned instead. */ + existing?: boolean; +}; + +export type CampaignResult = + | { ok: true; campaign: CampaignSummary } + | { ok: false; status: number; error: string }; + +/** Statuses under which a second campaign for the same URL would be a twin. */ +const LIVE_STATUSES = ["active", "draft", "paused", "pending_review"]; + +function creativeRow(campaignId: string, ownerId: string, c: AdCreative) { + return { + campaign_id: campaignId, + owner_id: ownerId, + format: c.format, + headline: (c.headline ?? "").slice(0, 80), + body: (c.body ?? "").slice(0, 140), + cta_text: (c.ctaText ?? "Learn more").slice(0, 24) || "Learn more", + image_url: c.imageUrl ?? null, + logo_url: c.logoUrl ?? null, + bg_color: c.bgColor, + fg_color: c.fgColor, + accent_color: c.accentColor, + light_bg_color: c.lightBgColor ?? null, + light_fg_color: c.lightFgColor ?? null, + light_accent_color: c.lightAccentColor ?? null, + font_family: (c.fontFamily ?? "system-ui, sans-serif").slice(0, 200), + }; +} + +function summaryColumns(summary: AdSummary | null | undefined, domain: string): Record { + if (!summary) return {}; + const short = cleanSummary(summary.short, 400); + const long = cleanSummary(summary.long, 1600); + if (!short && !long) return {}; + if (String(summary.domain ?? "").toLowerCase() !== domain.toLowerCase()) return {}; + return { + summary_short: short || null, + summary_long: long || null, + summary_domain: domain.toLowerCase(), + summary_generated_at: new Date().toISOString(), + }; +} + +const schemaLag = (message: string | undefined) => /organization_id|summary_|schema cache|column/i.test(message ?? ""); + +export async function createCampaignForUrl(input: { + sb: SupabaseClient; + userId: string; + email?: string | null; + request: CampaignRequest; + siteUrl: string; +}): Promise { + const { sb, userId, request } = input; + const status: CampaignStatus = request.status ?? "active"; + const domain = domainOf(request.url); + + // A live twin wins over a new campaign. If the caller wants it running and + // it is only a draft, running it is what they asked for. + const { data: twin } = await sb + .from("ad_campaigns") + .select("id, ref_slug, name, status, destination_url, daily_budget_cents, bid_credits, created_at") + .eq("owner_id", userId) + .eq("destination_url", request.url) + .in("status", LIVE_STATUSES) + .order("created_at", { ascending: false }) + .limit(1) + .maybeSingle(); + if (twin) { + let current = twin.status as string; + if (status === "active" && current !== "active") { + const { error } = await sb.from("ad_campaigns").update({ status: "active" }).eq("id", twin.id).eq("owner_id", userId); + if (!error) current = "active"; + } + return { + ok: true, + campaign: { + ...(twin as CampaignSummary), + status: current, + existing: true, + dashboard_url: `${input.siteUrl}/dashboard/ads/${twin.id}`, + }, + }; + } + + let generated: Awaited>; + try { + generated = await generateAdCreatives(request.url, { supabase: sb }); + } catch (err) { + return { ok: false, status: 502, error: err instanceof Error ? `Could not write ads for that URL: ${err.message}` : "Could not write ads for that URL." }; + } + if (!generated.creatives.length) return { ok: false, status: 502, error: "No creatives could be written for that URL." }; + + const org = await getOrCreateDefaultOrg({ userId, email: input.email }).catch(() => ({ id: null as string | null })); + const payload: Record = { + owner_id: userId, + name: (request.name || generated.brand.title?.slice(0, 60) || domain).slice(0, 120), + destination_url: request.url, + destination_domain: domain, + daily_budget_cents: request.dailyBudgetCents ?? 500, + bid_credits: request.bidCredits ?? DEFAULT_BID_CREDITS, + status, + brand: generated.brand ?? {}, + ...summaryColumns(generated.summary, domain), + }; + if (org.id) payload.organization_id = org.id; + + const select = "id, ref_slug, name, status, destination_url, daily_budget_cents, bid_credits, created_at"; + let inserted = await sb.from("ad_campaigns").insert(payload).select(select).single(); + // Migrations here are applied by hand, so a deploy can run ahead of the + // schema; the optional columns are dropped rather than refusing the campaign. + if (inserted.error && schemaLag(inserted.error.message)) { + for (const key of Object.keys(payload)) if (key === "organization_id" || key.startsWith("summary_")) delete payload[key]; + inserted = await sb.from("ad_campaigns").insert(payload).select(select).single(); + } + if (inserted.error || !inserted.data) { + return { ok: false, status: 500, error: inserted.error?.message ?? "Failed to save the campaign." }; + } + const campaign = inserted.data as CampaignSummary; + + const { error: creativeError } = await sb + .from("ad_creatives") + .insert(generated.creatives.map((c) => creativeRow(campaign.id, userId, c))); + if (creativeError) { + // A campaign with no creatives serves nothing; do not leave it behind. + await sb.from("ad_campaigns").delete().eq("id", campaign.id).eq("owner_id", userId); + return { ok: false, status: 500, error: creativeError.message }; + } + + return { + ok: true, + campaign: { + ...campaign, + creatives: generated.creatives.length, + dashboard_url: `${input.siteUrl}/dashboard/ads/${campaign.id}`, + }, + }; +} + +export async function listCampaigns(input: { sb: SupabaseClient; userId: string; limit: number; siteUrl: string }): Promise { + const { data } = await input.sb + .from("ad_campaigns") + .select("id, ref_slug, name, status, destination_url, daily_budget_cents, bid_credits, created_at") + .eq("owner_id", input.userId) + .order("created_at", { ascending: false }) + .limit(Math.min(200, Math.max(1, input.limit))); + return ((data as CampaignSummary[]) ?? []).map((c) => ({ ...c, dashboard_url: `${input.siteUrl}/dashboard/ads/${c.id}` })); +} diff --git a/lib/ads/slots.ts b/lib/ads/slots.ts new file mode 100644 index 00000000..e6884da2 --- /dev/null +++ b/lib/ads/slots.ts @@ -0,0 +1,215 @@ +// Publisher slots created from outside the dashboard. +// +// A slot is where ads render on a site the caller owns. Creating one from the +// CLI or the API means naming a site — "nichedb.dev" — and getting back an id +// and the two tags to paste: the ad unit and the stats tracker. The site's +// project is found by hostname, or created with the tracker on, because a +// blog that carries ads should be counting its readers too. +// +// Idempotent on the site: a second call for a site that already has a slot +// returns that slot. Every publisher unit on a page can share one slot id. + +import type { SupabaseClient } from "@supabase/supabase-js"; +import { isAllowedTargetUrl } from "@/lib/rateLimit"; + +export type SlotPlacement = "inline" | "sidebar" | "footer" | "sticky"; +export type SlotStatus = "active" | "inactive"; + +export type SlotRequest = { + /** The site, as a hostname or URL. */ + site: string; + placement?: SlotPlacement; + formats?: string[]; + /** The format the pasted unit renders. */ + format?: string; + status?: SlotStatus; + /** Turn the site's stats tracker on (default true). */ + enableTracking?: boolean; +}; + +export type SlotSummary = { + id: string; + status: string; + placement: string; + formats: string[]; + project_id: string; + site: string; + created_at?: string; + /** True when the site already had a slot and it was returned instead. */ + existing?: boolean; + /** The ad unit plus the tracker, ready to paste before . */ + embed: string; + tracker: string; + dashboard_url: string; +}; + +export type SlotResult = { ok: true; slot: SlotSummary } | { ok: false; status: number; error: string }; + +const PLACEMENTS = new Set(["inline", "sidebar", "footer", "sticky"]); +/** What a unit written into a blog page renders by default: the text strip. */ +export const DEFAULT_UNIT_FORMAT = "text_link"; + +export function hostOf(input: string): string | null { + const check = isAllowedTargetUrl(input); + if (!check.ok) return null; + try { + return new URL(check.url).hostname.toLowerCase().replace(/^www\./, ""); + } catch { + return null; + } +} + +/** Pure: what a caller sent, normalised. */ +export function parseSlotRequest(body: Record): { ok: true; request: SlotRequest; host: string } | { ok: false; error: string } { + const site = typeof body.site === "string" ? body.site : typeof body.url === "string" ? body.url : ""; + const host = hostOf(site); + if (!host) return { ok: false, error: "site must be a hostname or URL you own, like nichedb.dev." }; + + const request: SlotRequest = { site: host }; + if (body.placement !== undefined) { + if (!PLACEMENTS.has(body.placement as SlotPlacement)) return { ok: false, error: "placement must be inline, sidebar, footer or sticky." }; + request.placement = body.placement as SlotPlacement; + } + if (body.formats !== undefined) { + const raw = Array.isArray(body.formats) ? body.formats : String(body.formats).split(","); + const formats = raw.map((f) => String(f).trim()).filter((f) => /^[a-z0-9_]+$/.test(f)); + if (!formats.length) return { ok: false, error: "formats must name at least one ad format." }; + request.formats = formats; + } + if (body.format !== undefined) { + const format = String(body.format).trim(); + if (!/^[a-z0-9_]+$/.test(format)) return { ok: false, error: "format must be an ad format id, like text_link." }; + request.format = format; + } + if (body.status !== undefined) { + if (body.status !== "active" && body.status !== "inactive") return { ok: false, error: 'status must be "active" or "inactive".' }; + request.status = body.status; + } + if (body.enable_tracking !== undefined || body.enableTracking !== undefined) { + request.enableTracking = (body.enable_tracking ?? body.enableTracking) !== false; + } + return { ok: true, request, host }; +} + +/** The tags a publisher pastes: the unit, then the tracker and the renderer. */ +export function embedFor(siteUrl: string, slotId: string, projectId: string, format: string): { embed: string; tracker: string } { + const origin = siteUrl.replace(/\/$/, ""); + const tracker = ``; + const unit = ``; + const renderer = ``; + return { embed: `${unit}\n\n${tracker}\n${renderer}`, tracker }; +} + +function sameHost(projectUrl: string | null | undefined, host: string): boolean { + if (!projectUrl) return false; + const candidate = hostOf(projectUrl); + return candidate === host; +} + +export async function createSlotForSite(input: { + sb: SupabaseClient; + userId: string; + request: SlotRequest; + siteUrl: string; +}): Promise { + const { sb, userId, request } = input; + const host = request.site; + const enableTracking = request.enableTracking !== false; + const unitFormat = request.format ?? DEFAULT_UNIT_FORMAT; + + // The site's project: the one whose URL is this host, else a new one. + const { data: projects, error: projectsError } = await sb + .from("projects") + .select("id, name, url, organization_id, tracker_enabled") + .eq("owner_id", userId) + .order("created_at", { ascending: true }); + if (projectsError) return { ok: false, status: 500, error: projectsError.message }; + type ProjectRow = { id: string; name: string; url: string; organization_id?: string | null; tracker_enabled?: boolean }; + let project = (projects ?? []).find((p) => sameHost((p as { url?: string }).url, host)) as ProjectRow | undefined; + + if (!project) { + const { data: created, error } = await sb + .from("projects") + .insert({ owner_id: userId, name: host, url: `https://${host}`, tracker_enabled: enableTracking }) + .select("id, name, url, organization_id, tracker_enabled") + .single(); + if (error || !created) return { ok: false, status: 500, error: error?.message ?? "Failed to create the site." }; + project = created as ProjectRow; + } else if (enableTracking && project.tracker_enabled === false) { + await sb.from("projects").update({ tracker_enabled: true }).eq("id", project.id).eq("owner_id", userId); + } + if (!project) return { ok: false, status: 500, error: "Failed to resolve the site." }; + + const select = "id, status, placement, formats, project_id, created_at"; + const { data: existing } = await sb + .from("ad_slots") + .select(select) + .eq("project_id", project.id) + .eq("owner_id", userId) + .order("created_at", { ascending: true }) + .limit(1) + .maybeSingle(); + + type SlotRow = { id: string; status: string; placement: string; formats: string[]; project_id: string; created_at?: string }; + let slot = existing as SlotRow | null; + let wasExisting = false; + if (slot) { + wasExisting = true; + const wanted = request.status ?? "active"; + if (wanted === "active" && slot.status !== "active") { + const { error } = await sb.from("ad_slots").update({ status: "active" }).eq("id", slot.id).eq("owner_id", userId); + if (!error) slot = { ...slot, status: "active" }; + } + } else { + const payload: Record = { + project_id: project.id, + owner_id: userId, + status: request.status ?? "active", + placement: request.placement ?? "inline", + }; + if (request.formats) payload.formats = request.formats; + if (project.organization_id) payload.organization_id = project.organization_id; + let inserted = await sb.from("ad_slots").insert(payload).select(select).single(); + if (inserted.error && /organization_id|schema cache|column/i.test(inserted.error.message ?? "")) { + delete payload.organization_id; + inserted = await sb.from("ad_slots").insert(payload).select(select).single(); + } + if (inserted.error || !inserted.data) return { ok: false, status: 500, error: inserted.error?.message ?? "Failed to create the slot." }; + slot = inserted.data as SlotRow; + } + if (!slot) return { ok: false, status: 500, error: "Failed to create the slot." }; + + const tags = embedFor(input.siteUrl, slot.id, project.id, unitFormat); + return { + ok: true, + slot: { + ...slot, + site: host, + existing: wasExisting || undefined, + ...tags, + dashboard_url: `${input.siteUrl.replace(/\/$/, "")}/dashboard/ads/slots`, + }, + }; +} + +export async function listSlots(input: { sb: SupabaseClient; userId: string; siteUrl: string }): Promise[]> { + const { data } = await input.sb + .from("ad_slots") + .select("id, status, placement, formats, project_id, created_at, projects(url)") + .eq("owner_id", input.userId) + .order("created_at", { ascending: false }) + .limit(200); + return ((data as Record[]) ?? []).map((row) => { + const project = row.projects as { url?: string } | { url?: string }[] | null; + const url = Array.isArray(project) ? project[0]?.url : project?.url; + return { + id: String(row.id), + status: String(row.status), + placement: String(row.placement), + formats: (row.formats as string[]) ?? [], + project_id: String(row.project_id), + site: hostOf(url ?? "") ?? url ?? "", + created_at: row.created_at as string | undefined, + }; + }); +} diff --git a/lib/tracker/categorize.ts b/lib/tracker/categorize.ts index 6f86672e..6210d22e 100644 --- a/lib/tracker/categorize.ts +++ b/lib/tracker/categorize.ts @@ -6,6 +6,13 @@ interface CategorizeInput { referrer: string | null; userAgent: string | null; + /** + * The page the hit landed on, query string included. A visit that arrived + * through an ad announces itself there, not in the referrer: our click + * redirect appends `?ref=crawlproof-ad-NNN`, and paid links elsewhere carry + * utm_medium=cpc and friends. + */ + url?: string | null; } interface CategorizeResult { @@ -120,9 +127,43 @@ function matchHost( return null; } +/** utm_medium values that mean "somebody paid for this click". */ +const PAID_MEDIUMS = new Set(["cpc", "ppc", "cpm", "paid", "paidsocial", "paid_social", "display", "ad", "ads", "banner", "sponsored"]); + +/** A CrawlProof campaign ref, as appendRef() writes it onto the destination. */ +const CRAWLPROOF_REF = /^crawlproof-ad-\d+$/; + +/** + * The ad this hit came from, if it came from one. `ad:crawlproof-ad-072` for + * our own campaigns — the value is the campaign's ref slug, so a campaign's + * visits can be counted on the site it points at — and `ad:` for + * paid links tagged the conventional way. + */ +export function adFromUrl(url: string | null | undefined): string | null { + if (!url) return null; + let params: URLSearchParams; + try { + params = new URL(url, "https://placeholder.invalid").searchParams; + } catch { + return null; + } + const ref = params.get("ref")?.trim().toLowerCase() ?? ""; + if (CRAWLPROOF_REF.test(ref)) return ref; + + const medium = params.get("utm_medium")?.trim().toLowerCase() ?? ""; + if (PAID_MEDIUMS.has(medium)) { + const source = (params.get("utm_source") ?? "").trim().toLowerCase().replace(/[^a-z0-9_.-]+/g, "_").slice(0, 40); + return source || "unknown"; + } + if (params.has("gclid")) return "google"; + if (params.has("msclkid")) return "bing"; + return null; +} + export function categorize({ referrer, userAgent, + url, }: CategorizeInput): CategorizeResult { const ua = userAgent || ""; @@ -143,6 +184,11 @@ export function categorize({ return { bucket: "bot:other", isAi: false }; } + // An ad click beats the referrer: the referrer of a paid visit is whatever + // page carried the unit, which says where the ad ran, not why they came. + const ad = adFromUrl(url); + if (ad) return { bucket: `ad:${ad}`, isAi: false }; + const host = hostnameFromReferrer(referrer); if (host) { @@ -175,6 +221,8 @@ export function bucketLabel(bucket: string): string { return `Social · ${value}`; case "referral": return `Referral · ${value}`; + case "ad": + return `Ad · ${value}`; case "human": return value === "direct" ? "Direct" : value; default: diff --git a/tests/ads-api-requests.test.ts b/tests/ads-api-requests.test.ts new file mode 100644 index 00000000..17dfa582 --- /dev/null +++ b/tests/ads-api-requests.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { parseCampaignRequest, domainOf } from "@/lib/ads/campaign-request"; +import { parseSlotRequest, embedFor, hostOf, DEFAULT_UNIT_FORMAT } from "@/lib/ads/slots"; +import { campaignBodyFromArgs, slotBodyFromArgs, parseArgs } from "@/cli/index"; + +describe("POST /api/ads/v1/campaigns body", () => { + it("accepts a URL and applies the dashboard's clamps", () => { + const parsed = parseCampaignRequest({ url: "https://nichedb.dev/i/17", name: " NicheDB ", daily_budget_cents: 250.4, bid_credits: 999 }); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.url).toBe("https://nichedb.dev/i/17"); + expect(parsed.request).toEqual({ url: "https://nichedb.dev/i/17", name: "NicheDB", dailyBudgetCents: 250, bidCredits: 200 }); + }); + + it("refuses what the audit target guard refuses, and a made-up status", () => { + expect(parseCampaignRequest({ url: "http://localhost:3000/x" })).toMatchObject({ ok: false }); + expect(parseCampaignRequest({ url: "ftp://example.com" })).toMatchObject({ ok: false }); + expect(parseCampaignRequest({})).toMatchObject({ ok: false }); + expect(parseCampaignRequest({ url: "https://example.com", status: "paused" })).toMatchObject({ ok: false, error: expect.stringContaining("status") }); + expect(parseCampaignRequest({ url: "https://example.com", daily_budget_cents: -1 })).toMatchObject({ ok: false }); + }); + + it("takes camelCase too, and a draft status", () => { + const parsed = parseCampaignRequest({ url: "example.com", dailyBudgetCents: 100, status: "draft" }); + expect(parsed).toMatchObject({ ok: true, url: "https://example.com/", request: { dailyBudgetCents: 100, status: "draft" } }); + }); + + it("domainOf strips www", () => { + expect(domainOf("https://www.nichedb.dev/a")).toBe("nichedb.dev"); + }); +}); + +describe("POST /api/ads/v1/slots body", () => { + it("names the site by host, whether given a host or a URL", () => { + expect(parseSlotRequest({ site: "nichedb.dev" })).toMatchObject({ ok: true, host: "nichedb.dev", request: { site: "nichedb.dev" } }); + expect(parseSlotRequest({ url: "https://www.nichedb.dev/blog/" })).toMatchObject({ ok: true, host: "nichedb.dev" }); + expect(hostOf("localhost")).toBeNull(); + expect(parseSlotRequest({ site: "" })).toMatchObject({ ok: false }); + }); + + it("validates placement, formats and status", () => { + expect(parseSlotRequest({ site: "x.dev", placement: "roof" })).toMatchObject({ ok: false, error: expect.stringContaining("placement") }); + expect(parseSlotRequest({ site: "x.dev", formats: "text_link, banner_728x90" })).toMatchObject({ ok: true, request: { formats: ["text_link", "banner_728x90"] } }); + expect(parseSlotRequest({ site: "x.dev", formats: ["'); + expect(tags.embed.split("\n")).toEqual([ + '', + "", + tags.tracker, + '', + ]); + }); +}); + +describe("crawlproof ads / slots CLI", () => { + it("builds the campaign body from its flags", () => { + expect(campaignBodyFromArgs(parseArgs(["ads", "create", "https://nichedb.dev", "--name=NicheDB", "--budget=300", "--bid=5"]))).toEqual({ + url: "https://nichedb.dev", + name: "NicheDB", + daily_budget_cents: 300, + bid_credits: 5, + status: "active", + }); + expect(campaignBodyFromArgs(parseArgs(["ads", "create", "https://x.dev", "--draft"]))).toEqual({ url: "https://x.dev", status: "draft" }); + }); + + it("builds the slot body from its flags", () => { + expect(slotBodyFromArgs(parseArgs(["slots", "create", "nichedb.dev", "--placement=footer", "--format=text_link", "--no-tracking"]))).toEqual({ + site: "nichedb.dev", + placement: "footer", + format: "text_link", + enable_tracking: false, + }); + expect(slotBodyFromArgs(parseArgs(["slots", "create", "x.dev", "--formats=a,b", "--inactive"]))).toEqual({ site: "x.dev", formats: ["a", "b"], status: "inactive" }); + }); +}); diff --git a/tests/tracker-ad-bucket.test.ts b/tests/tracker-ad-bucket.test.ts new file mode 100644 index 00000000..f5a1d9ba --- /dev/null +++ b/tests/tracker-ad-bucket.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { adFromUrl, bucketLabel, categorize } from "@/lib/tracker/categorize"; +import { kindFromBucket } from "@/lib/tracker/humans"; + +const CHROME = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/128.0 Safari/537.36"; + +describe("ad attribution", () => { + it("our own click redirect is the campaign's ref slug", () => { + expect(adFromUrl("https://nichedb.dev/?ref=crawlproof-ad-072")).toBe("crawlproof-ad-072"); + expect(adFromUrl("/blog/043-post.html?ref=CrawlProof-Ad-3")).toBe("crawlproof-ad-3"); + // A ref that is not ours is not an ad. + expect(adFromUrl("https://x.dev/?ref=producthunt")).toBeNull(); + }); + + it("paid utm tags name the source; organic tags do not count", () => { + expect(adFromUrl("https://x.dev/?utm_source=reddit&utm_medium=cpc")).toBe("reddit"); + expect(adFromUrl("https://x.dev/?utm_medium=paid")).toBe("unknown"); + expect(adFromUrl("https://x.dev/?utm_source=newsletter&utm_medium=email")).toBeNull(); + expect(adFromUrl("https://x.dev/?gclid=abc")).toBe("google"); + expect(adFromUrl(null)).toBeNull(); + expect(adFromUrl("::not a url::")).toBeNull(); + }); + + it("an ad visit is bucketed as an ad, ahead of the referrer, and is human", () => { + const hit = categorize({ referrer: "https://dev.profullstack.com/~anthony/blog/042-post.html", userAgent: CHROME, url: "https://nichedb.dev/?ref=crawlproof-ad-072" }); + expect(hit).toEqual({ bucket: "ad:crawlproof-ad-072", isAi: false }); + expect(kindFromBucket(hit.bucket)).toBe("human"); + expect(bucketLabel(hit.bucket)).toBe("Ad · crawlproof-ad-072"); + }); + + it("a bot clicking an ad is still a bot", () => { + expect(categorize({ referrer: null, userAgent: "Mozilla/5.0 (compatible; GPTBot/1.0)", url: "https://x.dev/?ref=crawlproof-ad-1" }).bucket).toBe("bot:gptbot"); + }); + + it("nothing changes for a hit without a URL", () => { + expect(categorize({ referrer: "https://t.co/abc", userAgent: CHROME }).bucket).toBe("social:twitter"); + expect(categorize({ referrer: null, userAgent: CHROME }).bucket).toBe("human:direct"); + expect(categorize({ referrer: "https://someblog.example/post", userAgent: CHROME, url: "https://x.dev/" }).bucket).toBe("referral:someblog.example"); + }); +});