Charge AI training crawlers for access (x402 gateway)

apps/web/app/robots.txt/route.ts

@@ -0,0 +1,9 @@
+import { robotsRoute } from "@profullstack/x402-gateway/next";
+import { gateway } from "@/lib/crawl-gateway";
+
+// Generated from the same crawler lists the gateway enforces: training
+// crawlers are refused everywhere but /crawl (where they can buy a pass),
+// retrieval crawlers are named as welcome, everyone else gets the rules below.
+export const GET = robotsRoute(gateway, {
+  disallow: ["/api/"],
+});

apps/web/lib/crawl-gateway.ts

@@ -0,0 +1,27 @@
+import { createGateway } from "@profullstack/x402-gateway";
+import { x402Proxy } from "@profullstack/x402-gateway/next";
+
+/**
+ * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot,
+ * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over
+ * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval
+ * crawlers behind AI search pass through untouched.
+ *
+ * Runs inside the middleware, so nothing here may import Node-only modules.
+ * The env is read through a non-literal key on purpose: Next inlines
+ * `process.env.NAME` at build time, and these are runtime secrets. Without
+ * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training
+ * crawlers with 402, just with an empty offer.
+ */
+const env = (name: string) => process.env[name];
+
+export const gateway = createGateway({
+  siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://diskpush.com",
+  siteName: "DiskPush",
+  coinpay: { apiKey: env("COINPAY_X402_KEY") },
+  payTo: env("CRAWL_PAY_TO"),
+  contact: "mailto:support@diskpush.com",
+});
+
+/** Resolves to a Response for a refused crawler, or undefined to carry on. */
+export const gate = x402Proxy(gateway);

apps/web/middleware.ts

@@ -0,0 +1,20 @@
+import { gate } from "@/lib/crawl-gateway";
+import { NextResponse, type NextRequest } from "next/server";
+
+export async function middleware(request: NextRequest) {
+  // Crawl gateway first: AI training crawlers get 402 Payment Required (or the
+  // sales page at /crawl) unless they present a paid pass. People, Googlebot
+  // and retrieval crawlers fall through to everything below.
+  const answer = await gate(request);
+  if (answer) return answer;
+
+  return NextResponse.next();
+}
+
+export const config = {
+  // Everything but Next's own assets and static files. API routes stay
+  // covered on purpose: a training crawler hitting the API gets 402 too.
+  matcher: [
+    "/((?!_next/static|_next/image|favicon.ico|.*\.(?:svg|png|jpg|jpeg|gif|webp|ico|css|js|map|woff|woff2|ttf|otf|mp3|mp4|webmanifest)$).*)",
+  ],
+};

apps/web/package.json

@@ -10,6 +10,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@profullstack/x402-gateway": "^0.1.0",
     "marked": "^15.0.7",
     "next": "^15.5.4",
     "react": "^19.1.0",

pnpm-workspace.yaml

@@ -12,3 +12,6 @@ allowBuilds:
   # falls back to its pure-JS implementation.
   ssh2: false
   cpu-features: false
+
+minimumReleaseAgeExclude:
+  - '@profullstack/x402-gateway@0.1.0'