From 32ea0421919b63c60ed38b4d26b20a89a08310dc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:30:58 +0000 Subject: [PATCH] Charge AI training crawlers for access (@profullstack/x402-gateway) Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended) get 402 Payment Required with an x402 offer, or the sales page at /crawl, and a paid pass opens the site for a day. People, search engines and retrieval crawlers pass through untouched. robots.txt is now generated from the same lists. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- apps/web/app/robots.ts | 37 -------------------------------- apps/web/app/robots.txt/route.ts | 9 ++++++++ apps/web/lib/crawl-gateway.ts | 27 +++++++++++++++++++++++ apps/web/middleware.ts | 20 +++++++++++++++++ apps/web/package.json | 1 + pnpm-lock.yaml | 9 ++++++++ pnpm-workspace.yaml | 3 +++ 7 files changed, 69 insertions(+), 37 deletions(-) delete mode 100644 apps/web/app/robots.ts create mode 100644 apps/web/app/robots.txt/route.ts create mode 100644 apps/web/lib/crawl-gateway.ts create mode 100644 apps/web/middleware.ts diff --git a/apps/web/app/robots.ts b/apps/web/app/robots.ts deleted file mode 100644 index a4c15ee..0000000 --- a/apps/web/app/robots.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { MetadataRoute } from 'next' -import { SITE } from '@/lib/site' - -/** - * The wildcard already allows everything, so these named rules grant no access - * the crawlers did not have. They are here because the policy is worth stating - * rather than inferring: DiskPush is MIT-licensed software whose documentation - * exists to be read, and an answer engine that summarises it accurately is - * doing the thing the docs are for. - * - * Named explicitly so the position is legible to a person reading the file, and - * so that narrowing it later is an edit rather than a decision nobody recorded. - */ -const AI_CRAWLERS = [ - 'GPTBot', - 'OAI-SearchBot', - 'ChatGPT-User', - 'ClaudeBot', - 'Claude-User', - 'PerplexityBot', - 'Google-Extended', - 'Applebot-Extended', - 'CCBot', - 'meta-externalagent', - 'Bytespider', -] - -export default function robots(): MetadataRoute.Robots { - return { - rules: [ - { userAgent: '*', allow: '/' }, - ...AI_CRAWLERS.map((userAgent) => ({ userAgent, allow: '/' })), - ], - sitemap: `${SITE.url}/sitemap.xml`, - host: SITE.url, - } -} diff --git a/apps/web/app/robots.txt/route.ts b/apps/web/app/robots.txt/route.ts new file mode 100644 index 0000000..319d762 --- /dev/null +++ b/apps/web/app/robots.txt/route.ts @@ -0,0 +1,9 @@ +import { robotsRoute } from "@profullstack/x402-gateway/next"; +import { gateway } from "@/lib/crawl-gateway"; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ["/api/"], +}); diff --git a/apps/web/lib/crawl-gateway.ts b/apps/web/lib/crawl-gateway.ts new file mode 100644 index 0000000..cffadf0 --- /dev/null +++ b/apps/web/lib/crawl-gateway.ts @@ -0,0 +1,27 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://diskpush.com", + siteName: "DiskPush", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:support@diskpush.com", +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/middleware.ts b/apps/web/middleware.ts new file mode 100644 index 0000000..ef2ea76 --- /dev/null +++ b/apps/web/middleware.ts @@ -0,0 +1,20 @@ +import { gate } from "@/lib/crawl-gateway"; +import { NextResponse, type NextRequest } from "next/server"; + +export async function middleware(request: NextRequest) { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + + return NextResponse.next(); +} + +export const config = { + // Everything but Next's own assets and static files. API routes stay + // covered on purpose: a training crawler hitting the API gets 402 too. + matcher: [ + "/((?!_next/static|_next/image|favicon.ico|.*\.(?:svg|png|jpg|jpeg|gif|webp|ico|css|js|map|woff|woff2|ttf|otf|mp3|mp4|webmanifest)$).*)", + ], +}; diff --git a/apps/web/package.json b/apps/web/package.json index 2668190..c822dbf 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -10,6 +10,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@profullstack/x402-gateway": "^0.1.0", "marked": "^15.0.7", "next": "^15.5.4", "react": "^19.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c6ff06..f4bafdc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -126,6 +126,9 @@ importers: apps/web: dependencies: + '@profullstack/x402-gateway': + specifier: ^0.1.0 + version: 0.1.0 marked: specifier: ^15.0.7 version: 15.0.12 @@ -1110,6 +1113,10 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@profullstack/x402-gateway@0.1.0': + resolution: {integrity: sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==} + engines: {node: '>=20.11'} + '@radix-ui/primitive@1.1.7': resolution: {integrity: sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==} @@ -4981,6 +4988,8 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@profullstack/x402-gateway@0.1.0': {} + '@radix-ui/primitive@1.1.7': {} '@radix-ui/react-compose-refs@1.1.5(@types/react@19.2.18)(react@19.2.8)': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1ed2858..f862126 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -12,3 +12,6 @@ allowBuilds: # falls back to its pure-JS implementation. ssh2: false cpu-features: false + +minimumReleaseAgeExclude: + - '@profullstack/x402-gateway@0.1.0'