From 16fdb7e4ea65085ddb25dadd902ce0412ae33c49 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:35:07 +0000 Subject: [PATCH 1/2] Charge AI training crawlers for access (@profullstack/x402-gateway) Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended) get 402 Payment Required with an x402 offer, or the sales page at /crawl, and a paid pass opens the site for a day. People, search engines and retrieval crawlers pass through untouched. robots.txt is now generated from the same lists. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- .env.example | 7 ++++++- apps/web/package.json | 1 + apps/web/public/robots.txt | 25 ------------------------- apps/web/src/app/robots.txt/route.ts | 9 +++++++++ apps/web/src/lib/crawl-gateway.ts | 27 +++++++++++++++++++++++++++ apps/web/src/middleware.ts | 7 +++++++ pnpm-lock.yaml | 16 +++++++++++++++- 7 files changed, 65 insertions(+), 27 deletions(-) delete mode 100644 apps/web/public/robots.txt create mode 100644 apps/web/src/app/robots.txt/route.ts create mode 100644 apps/web/src/lib/crawl-gateway.ts diff --git a/.env.example b/.env.example index 9dc588d2..e6fd8a16 100644 --- a/.env.example +++ b/.env.example @@ -159,4 +159,9 @@ VAPID_PRIVATE_KEY=your-vapid-private-key APPLE_ID=your-developer-email APPLE_TEAM_ID=your-apple-team-id -APPLE_APP_SPECIFIC_PASSWORD=your-app-specific-password \ No newline at end of file +APPLE_APP_SPECIFIC_PASSWORD=your-app-specific-password +# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day over +# x402. A SCOPED CoinPay key (payments:create) and the EVM address that receives +# the USDC. Unset = crawlers still get 402, nothing sold. +COINPAY_X402_KEY= +CRAWL_PAY_TO= diff --git a/apps/web/package.json b/apps/web/package.json index 1194e74d..21f93a41 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -19,6 +19,7 @@ "@profullstack/autoblog": "github:profullstack/autoblog#75e54af", "@profullstack/emailer": "^1.0.3", "@profullstack/stack": "^0.1.3", + "@profullstack/x402-gateway": "^0.1.0", "@serwist/next": "^9.5.0", "@supabase/ssr": "^0.5.0", "@supabase/supabase-js": "^2.47.0", diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt deleted file mode 100644 index 854392e1..00000000 --- a/apps/web/public/robots.txt +++ /dev/null @@ -1,25 +0,0 @@ -User-agent: * -Allow: / - -User-agent: GPTBot -Allow: / - -User-agent: ClaudeBot -Allow: / - -User-agent: PerplexityBot -Allow: / - -User-agent: Google-Extended -Allow: / - -User-agent: OAI-SearchBot -Allow: / - -User-agent: Applebot-Extended -Allow: / - -User-agent: CCBot -Allow: / - -Sitemap: https://pairux.com/sitemap.xml diff --git a/apps/web/src/app/robots.txt/route.ts b/apps/web/src/app/robots.txt/route.ts new file mode 100644 index 00000000..2d8b719e --- /dev/null +++ b/apps/web/src/app/robots.txt/route.ts @@ -0,0 +1,9 @@ +import { robotsRoute } from '@profullstack/x402-gateway/next'; +import { gateway } from '@/lib/crawl-gateway'; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ['/api/'], +}); diff --git a/apps/web/src/lib/crawl-gateway.ts b/apps/web/src/lib/crawl-gateway.ts new file mode 100644 index 00000000..e92124d3 --- /dev/null +++ b/apps/web/src/lib/crawl-gateway.ts @@ -0,0 +1,27 @@ +import { createGateway } from '@profullstack/x402-gateway'; +import { x402Proxy } from '@profullstack/x402-gateway/next'; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env('SITE_URL') || env('NEXT_PUBLIC_SITE_URL') || 'https://pairux.com', + siteName: 'PairUX', + coinpay: { apiKey: env('COINPAY_X402_KEY') }, + payTo: env('CRAWL_PAY_TO'), + contact: 'mailto:support@pairux.com', +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/src/middleware.ts b/apps/web/src/middleware.ts index 54b29e27..421434cd 100644 --- a/apps/web/src/middleware.ts +++ b/apps/web/src/middleware.ts @@ -1,3 +1,4 @@ +import { gate } from '@/lib/crawl-gateway'; import { NextResponse, type NextRequest } from 'next/server'; import { updateSession } from '@/lib/supabase/middleware'; import { CORS_HEADERS } from '@/lib/cors'; @@ -43,6 +44,12 @@ function buildCsp(nonce: string, embeddable: boolean): string { } export async function middleware(request: NextRequest) { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + const { pathname } = request.nextUrl; const isEmbed = pathname === '/embed' || pathname.startsWith('/embed/'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 37267084..ef8ca4f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -350,6 +350,9 @@ importers: '@profullstack/stack': specifier: ^0.1.3 version: 0.1.3(@supabase/ssr@0.5.2(@supabase/supabase-js@2.91.0))(next@16.2.9(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) + '@profullstack/x402-gateway': + specifier: ^0.1.0 + version: 0.1.0 '@serwist/next': specifier: ^9.5.0 version: 9.5.0(next@16.2.9(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(typescript@5.9.3) @@ -1906,7 +1909,7 @@ packages: '@expo/bunyan@4.0.1': resolution: {integrity: sha512-+Lla7nYSiHZirgK+U/uYzsLv/X+HaJienbD5AKX1UQZHYfWaP+9uuQluRB4GrEVWF0GZ7vEVp/jzaOT9k/SQlg==} - engines: {'0': node >=0.10.0} + engines: {node: '>=0.10.0'} '@expo/cli@0.22.28': resolution: {integrity: sha512-lvt72KNitGuixYD2l3SZmRKVu2G4zJpmg5V7WfUBNpmUU5oODBw/6qmiJ6kSLAlfDozscUk+BBGknBBzxUrwrA==} @@ -2564,16 +2567,19 @@ packages: '@nut-tree-fork/libnut-darwin@2.7.5': resolution: {integrity: sha512-LbqtPtMPTJUcg4XoPP2jsU1wc8flBcGyKTerKsIfK9cD7nBHROnO0QksbrsbSWEpLym8T8fRtuU7XEY83l6Z2Q==} engines: {node: '>=10.15.3'} + cpu: [x64, arm64] os: [darwin, linux, win32] '@nut-tree-fork/libnut-linux@2.7.5': resolution: {integrity: sha512-uxaXEcRKnFObAljsoR6tLOBUU1dJ2sctloG6gFgCBGN7+k6Jdv6jZfOuNjd/fpdq2C5WPMm0rtn9EE7h5J3Jcg==} engines: {node: '>=10.15.3'} + cpu: [x64, arm64] os: [darwin, linux, win32] '@nut-tree-fork/libnut-win32@2.7.5': resolution: {integrity: sha512-yqC87zvmFcDPwFrRU40DYhN0xmEVM3aSkOuyF0IX+y1x+HWSu/i0PNklATpPBhGid3QVb/TOHuVoaraMrUFCNw==} engines: {node: '>=10.15.3'} + cpu: [x64, arm64] os: [darwin, linux, win32] '@nut-tree-fork/libnut@4.2.6': @@ -2587,6 +2593,7 @@ packages: '@nut-tree-fork/nut-js@4.2.6': resolution: {integrity: sha512-aI/WCX7gE1HFGPH3EZP/UWqpNMM1NMoM/EkXqp7pKMgXFCi8e5+o5p+jd/QOYpmALv9bQg7+s69nI7FONbMqDg==} engines: {node: '>=16'} + cpu: [x64, arm64] os: [linux, darwin, win32] '@nut-tree-fork/provider-interfaces@4.2.6': @@ -2637,6 +2644,10 @@ packages: react: optional: true + '@profullstack/x402-gateway@0.1.0': + resolution: {integrity: sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==} + engines: {node: '>=20.11'} + '@radix-ui/react-compose-refs@1.0.0': resolution: {integrity: sha512-0KaSv6sx787/hK3eF53iOkiSLwAGlFMx5lotrqD2pTjB18KbybKoEIgkNZTKC60YECDQTKGTRcDBILwZVqVKvA==} peerDependencies: @@ -4706,6 +4717,7 @@ packages: eslint@9.39.2: resolution: {integrity: sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -11425,6 +11437,8 @@ snapshots: next: 16.2.9(react-dom@19.2.3(react@19.2.3))(react@19.2.3) react: 19.2.3 + '@profullstack/x402-gateway@0.1.0': {} + '@radix-ui/react-compose-refs@1.0.0(react@18.3.1)': dependencies: '@babel/runtime': 7.29.7 From f739c8e2a388804448c022371707d8ad915cae76 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:41:07 +0000 Subject: [PATCH 2/2] Satisfy exactOptionalPropertyTypes and prefer-nullish-coalescing in the crawl gateway Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- apps/web/src/lib/crawl-gateway.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/web/src/lib/crawl-gateway.ts b/apps/web/src/lib/crawl-gateway.ts index e92124d3..1df4a7e8 100644 --- a/apps/web/src/lib/crawl-gateway.ts +++ b/apps/web/src/lib/crawl-gateway.ts @@ -16,10 +16,10 @@ import { x402Proxy } from '@profullstack/x402-gateway/next'; const env = (name: string) => process.env[name]; export const gateway = createGateway({ - siteUrl: env('SITE_URL') || env('NEXT_PUBLIC_SITE_URL') || 'https://pairux.com', + siteUrl: env('SITE_URL') ?? env('NEXT_PUBLIC_SITE_URL') ?? 'https://pairux.com', siteName: 'PairUX', - coinpay: { apiKey: env('COINPAY_X402_KEY') }, - payTo: env('CRAWL_PAY_TO'), + coinpay: { apiKey: env('COINPAY_X402_KEY') ?? '' }, + payTo: env('CRAWL_PAY_TO') ?? '', contact: 'mailto:support@pairux.com', });