From 69bf52addc693ac86e5959d25f774dc63232553e Mon Sep 17 00:00:00 2001 From: Dhyey Desai Date: Fri, 2 Oct 2026 11:56:29 -0400 Subject: [PATCH 1/3] Add guest identity: signed cs_guest cookie, /api/guest, guest_sessions Guests get an httpOnly cookie guest_.. (24 h) signed with GUEST_COOKIE_SECRET; auth() returns the guest_ id as the user id, so it is the team_id and existing tenant queries need no change. Clerk users always win over a guest cookie. The middleware accepts a valid guest cookie on protected routes (Clerk and test mode). Web Crypto only, so it runs on edge. POST /api/guest records the session in guest_sessions (migration 003) with an HMAC of the client IP, capped at 5 per IP and 200 overall per 24 h in one INSERT ... SELECT so concurrent requests cannot overshoot. Optional Turnstile check when TURNSTILE_SECRET_KEY is set (fails closed on network errors). Everything is off while GUEST_COOKIE_SECRET is unset. Rejected: a JWT library (one HMAC is enough and keeps the edge bundle small) and storing sessions only in the cookie (no way to cap per IP). --- .env.example | 11 ++ src/app/api/guest/route.test.ts | 111 +++++++++++++++++++ src/app/api/guest/route.ts | 104 ++++++++++++++++++ src/lib/auth.test.ts | 45 +++++++- src/lib/auth.ts | 24 ++++- src/lib/env.ts | 11 ++ src/lib/guest-limits.ts | 33 ++++++ src/lib/guest.test.ts | 68 ++++++++++++ src/lib/guest.ts | 125 ++++++++++++++++++++++ src/lib/migrations/003_guest_sessions.sql | 22 ++++ src/lib/schema.sql | 16 +++ src/lib/schema.test.ts | 7 +- src/lib/turnstile.ts | 30 ++++++ src/middleware.guest.test.ts | 42 ++++++++ src/middleware.ts | 18 +++- tests/e2e/test-mode-guard.spec.ts | 8 ++ tests/support/test-env.mjs | 2 + 17 files changed, 666 insertions(+), 11 deletions(-) create mode 100644 src/app/api/guest/route.test.ts create mode 100644 src/app/api/guest/route.ts create mode 100644 src/lib/guest-limits.ts create mode 100644 src/lib/guest.test.ts create mode 100644 src/lib/guest.ts create mode 100644 src/lib/migrations/003_guest_sessions.sql create mode 100644 src/lib/turnstile.ts create mode 100644 src/middleware.guest.test.ts diff --git a/.env.example b/.env.example index cd5342b..bbcfa5a 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,17 @@ CLERK_SECRET_KEY= # UPSTASH_REDIS_REST_URL=https://your-instance.upstash.io # UPSTASH_REDIS_REST_TOKEN= +# --- Guest access (optional: off unless GUEST_COOKIE_SECRET is set) ------- +# Generate with: openssl rand -base64 48 (>= 32 chars) +# GUEST_COOKIE_SECRET= +# Optional Cloudflare Turnstile on "Try as guest" (set both or neither) +# TURNSTILE_SECRET_KEY= +# NEXT_PUBLIC_TURNSTILE_SITE_KEY= + +# Per-file upload limit for signed-in users, MB (default 100; guests are fixed at 2). +# Keep in sync with MAX_UPLOAD_MB on the profiler Lambda. +# MAX_UPLOAD_MB=100 + # Structured JSON logs: debug | info | warn | error (default info) # LOG_LEVEL=info diff --git a/src/app/api/guest/route.test.ts b/src/app/api/guest/route.test.ts new file mode 100644 index 0000000..0fd9f40 --- /dev/null +++ b/src/app/api/guest/route.test.ts @@ -0,0 +1,111 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +const h = vi.hoisted(() => ({ auth: vi.fn(), queryOne: vi.fn(), turnstile: vi.fn() })); +vi.mock("@/lib/auth", () => ({ auth: () => h.auth() })); +vi.mock("@/lib/db", () => ({ queryOne: (...a: unknown[]) => h.queryOne(...a) })); +vi.mock("@/lib/turnstile", () => ({ verifyTurnstile: (...a: unknown[]) => h.turnstile(...a) })); + +import { DELETE, GET, POST } from "./route"; +import { GUEST_COOKIE, signGuestToken, verifyGuestToken } from "@/lib/guest"; +import { GUEST_LIMITS } from "@/lib/guest-limits"; + +const SECRET = "unit-test-guest-secret-0123456789abcdef"; +const req = (method: string, init: { cookie?: string; body?: unknown; ip?: string } = {}) => + new NextRequest("http://localhost/api/guest", { + method, + headers: { + "content-type": "application/json", + "x-forwarded-for": init.ip ?? "203.0.113.7", + ...(init.cookie ? { cookie: `${GUEST_COOKIE}=${init.cookie}` } : {}), + }, + body: init.body === undefined ? undefined : JSON.stringify(init.body), + }); + +beforeEach(() => { + vi.clearAllMocks(); + process.env.GUEST_COOKIE_SECRET = SECRET; + h.auth.mockResolvedValue({ userId: null }); + h.turnstile.mockResolvedValue(true); + h.queryOne.mockImplementation(async (sql: string, params: unknown[]) => + sql.startsWith("INSERT INTO guest_sessions") ? { id: params[0] } : { n: 0 }); +}); +afterEach(() => { delete process.env.GUEST_COOKIE_SECRET; }); + +describe("POST /api/guest", () => { + it("404s when guest access is off", async () => { + delete process.env.GUEST_COOKIE_SECRET; + expect((await POST(req("POST"))).status).toBe(404); + expect((await DELETE()).status).toBe(404); + expect(await (await GET(req("GET"))).json()).toEqual({ enabled: false, guest: null }); + }); + + it("issues a signed httpOnly 24 h cookie and records the session with a hashed IP and the caps", async () => { + const res = await POST(req("POST")); + expect(res.status).toBe(201); + const cookie = res.cookies.get(GUEST_COOKIE)!; + expect(cookie.httpOnly).toBe(true); + expect(cookie.sameSite).toBe("lax"); + expect(cookie.secure).toBe(true); + const session = await verifyGuestToken(cookie.value, SECRET); + expect(session).not.toBeNull(); + expect(session!.expiresAt - Math.floor(Date.now() / 1000)).toBeGreaterThan(24 * 3600 - 10); + + const [sql, params] = h.queryOne.mock.calls[0]; + expect(sql).toContain("INSERT INTO guest_sessions"); + expect(params[0]).toBe(session!.guestId); + expect(params[1]).not.toContain("203.0.113.7"); + expect(params.slice(2)).toEqual([24 * 3600, GUEST_LIMITS.guestsPerIpPerDay, GUEST_LIMITS.guestsPerDay]); + }); + + it("reuses a valid guest cookie instead of creating another session", async () => { + const token = await signGuestToken({ guestId: "guest_AAAAAAAAAAAAAAAAAAAAAA", expiresAt: Math.floor(Date.now() / 1000) + 60 }, SECRET); + const res = await POST(req("POST", { cookie: token })); + expect(res.status).toBe(200); + expect((await res.json()).reused).toBe(true); + expect(h.queryOne).not.toHaveBeenCalled(); + }); + + it("409s for a signed-in account", async () => { + h.auth.mockResolvedValue({ userId: "user_2abc" }); + expect((await POST(req("POST"))).status).toBe(409); + }); + + it("403s when Turnstile rejects the token", async () => { + h.turnstile.mockResolvedValue(false); + const res = await POST(req("POST", { body: { turnstile_token: "bad" } })); + expect(res.status).toBe(403); + expect(h.turnstile).toHaveBeenCalledWith("bad", "203.0.113.7"); + expect(h.queryOne).not.toHaveBeenCalled(); + }); + + it("429s at the per-IP cap and 503s at the global cap", async () => { + h.queryOne.mockImplementation(async (sql: string) => (sql.startsWith("INSERT") ? null : { n: GUEST_LIMITS.guestsPerIpPerDay })); + const perIp = await POST(req("POST")); + expect(perIp.status).toBe(429); + expect(perIp.cookies.get(GUEST_COOKIE)).toBeUndefined(); + + h.queryOne.mockImplementation(async (sql: string) => (sql.startsWith("INSERT") ? null : { n: 1 })); + expect((await POST(req("POST"))).status).toBe(503); + }); + + it("answers JSON 500 when the database fails", async () => { + h.queryOne.mockRejectedValue(new Error("db down")); + const res = await POST(req("POST")); + expect(res.status).toBe(500); + expect((await res.json()).error).toMatch(/Could not start a guest session/); + }); +}); + +describe("GET/DELETE /api/guest", () => { + it("reports the session and clears it", async () => { + const expiresAt = Math.floor(Date.now() / 1000) + 60; + const token = await signGuestToken({ guestId: "guest_AAAAAAAAAAAAAAAAAAAAAA", expiresAt }, SECRET); + expect(await (await GET(req("GET", { cookie: token }))).json()).toEqual({ + enabled: true, guest: { expires_at: new Date(expiresAt * 1000).toISOString() }, + }); + expect(await (await GET(req("GET", { cookie: "forged" }))).json()).toEqual({ enabled: true, guest: null }); + const del = await DELETE(); + expect(del.headers.get("set-cookie")).toMatch(new RegExp(`${GUEST_COOKIE}=;`)); + }); +}); diff --git a/src/app/api/guest/route.ts b/src/app/api/guest/route.ts new file mode 100644 index 0000000..1b7d094 --- /dev/null +++ b/src/app/api/guest/route.ts @@ -0,0 +1,104 @@ +/** + * Guest sessions (src/lib/guest.ts). Answers 404 unless GUEST_COOKIE_SECRET is set. + * + * POST { turnstile_token? } -> issues the signed `cs_guest` cookie (24 h). + * Reuses a still-valid guest cookie. 409 when signed in with an account, + * 403 when Turnstile is enabled and the token fails, 429 when this IP + * already started GUEST_LIMITS.guestsPerIpPerDay sessions in 24 h, 503 + * when GUEST_LIMITS.guestsPerDay sessions were started in 24 h overall. + * GET -> { enabled, guest: { expires_at } | null } + * DELETE -> ends the session (clears the cookie; data is purged within 24 h). + */ +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/lib/auth"; +import { queryOne } from "@/lib/db"; +import { + GUEST_COOKIE, GUEST_TTL_SECONDS, clientIp, guestAccessEnabled, guestCookieOptions, guestFromCookie, + guestSecret, hashIp, isGuestId, newGuestId, signGuestToken, +} from "@/lib/guest"; +import { GUEST_LIMITS } from "@/lib/guest-limits"; +import { verifyTurnstile } from "@/lib/turnstile"; +import { isTestMode } from "@/lib/test-mode"; +import { logger } from "@/lib/logger"; + +const log = logger.child({ route: "/api/guest" }); + +export const dynamic = "force-dynamic"; + +const notFound = () => new NextResponse("Not Found", { status: 404 }); +const iso = (unixSeconds: number) => new Date(unixSeconds * 1000).toISOString(); + +export async function GET(req: NextRequest) { + if (!guestAccessEnabled()) return NextResponse.json({ enabled: false, guest: null }); + const session = await guestFromCookie(req.cookies.get(GUEST_COOKIE)?.value); + return NextResponse.json({ enabled: true, guest: session ? { expires_at: iso(session.expiresAt) } : null }); +} + +export async function POST(req: NextRequest) { + const secret = guestSecret(); + if (!secret) return notFound(); + + try { + const { userId } = await auth(); + if (userId && !isGuestId(userId)) { + return NextResponse.json({ error: "You are already signed in." }, { status: 409 }); + } + const existing = await guestFromCookie(req.cookies.get(GUEST_COOKIE)?.value); + if (existing) return NextResponse.json({ ok: true, expires_at: iso(existing.expiresAt), reused: true }); + + const body = (await req.json().catch(() => ({}))) as { turnstile_token?: unknown }; + const ip = clientIp(req.headers); + if (!(await verifyTurnstile(body?.turnstile_token, ip))) { + return NextResponse.json({ error: "Verification failed. Please retry." }, { status: 403 }); + } + + const ipHash = await hashIp(ip, secret); + const guestId = newGuestId(); + const expiresAt = Math.floor(Date.now() / 1000) + GUEST_TTL_SECONDS; + + // Caps checked and the row inserted in one statement. + const inserted = await queryOne<{ id: string }>( + `INSERT INTO guest_sessions (id, ip_hash, expires_at) + SELECT $1, $2, now() + make_interval(secs => $3) + WHERE (SELECT count(*) FROM guest_sessions + WHERE ip_hash = $2 AND created_at > now() - interval '24 hours') < $4 + AND (SELECT count(*) FROM guest_sessions + WHERE created_at > now() - interval '24 hours') < $5 + RETURNING id`, + [guestId, ipHash, GUEST_TTL_SECONDS, GUEST_LIMITS.guestsPerIpPerDay, GUEST_LIMITS.guestsPerDay] + ); + if (!inserted) { + const perIp = await queryOne<{ n: number }>( + "SELECT count(*)::int AS n FROM guest_sessions WHERE ip_hash = $1 AND created_at > now() - interval '24 hours'", + [ipHash] + ); + if (Number(perIp?.n ?? 0) >= GUEST_LIMITS.guestsPerIpPerDay) { + log.warn("guest session refused: per-IP cap", { cap: GUEST_LIMITS.guestsPerIpPerDay }); + return NextResponse.json( + { error: `Guest limit reached for your network (${GUEST_LIMITS.guestsPerIpPerDay} guest sessions per day). Sign up for a free account to continue.` }, + { status: 429 } + ); + } + log.warn("guest session refused: global cap", { cap: GUEST_LIMITS.guestsPerDay }); + return NextResponse.json( + { error: "Guest access is at capacity today. Please sign up for a free account or try again tomorrow." }, + { status: 503 } + ); + } + + const res = NextResponse.json({ ok: true, expires_at: iso(expiresAt) }, { status: 201 }); + res.cookies.set(GUEST_COOKIE, await signGuestToken({ guestId, expiresAt }, secret), guestCookieOptions(expiresAt, !isTestMode())); + log.info("guest session started"); + return res; + } catch (err) { + log.error("guest session failed", { err }); + return NextResponse.json({ error: "Could not start a guest session. Please retry." }, { status: 500 }); + } +} + +export async function DELETE() { + if (!guestAccessEnabled()) return notFound(); + const res = NextResponse.json({ ok: true }); + res.cookies.delete(GUEST_COOKIE); + return res; +} diff --git a/src/lib/auth.test.ts b/src/lib/auth.test.ts index aaf8b5d..f5a2262 100644 --- a/src/lib/auth.test.ts +++ b/src/lib/auth.test.ts @@ -16,8 +16,13 @@ vi.mock("@clerk/nextjs/server", () => ({ })); import { auth, currentUserEmail, isValidTestUserId, TEST_USER_COOKIE, userEmailById } from "@/lib/auth"; +import { GUEST_COOKIE, signGuestToken } from "@/lib/guest"; -const ENV_KEYS = ["CLEANSTACK_TEST_MODE", "VERCEL", "AWS_LAMBDA_FUNCTION_NAME"] as const; +const ENV_KEYS = ["CLEANSTACK_TEST_MODE", "VERCEL", "AWS_LAMBDA_FUNCTION_NAME", "GUEST_COOKIE_SECRET"] as const; +const GUEST_SECRET = "unit-test-guest-secret-0123456789abcdef"; +const GUEST = "guest_AAAAAAAAAAAAAAAAAAAAAA"; +const guestCookie = (secret = GUEST_SECRET, expiresAt = Math.floor(Date.now() / 1000) + 3600) => + signGuestToken({ guestId: GUEST, expiresAt }, secret); const saved: Record = {}; beforeEach(() => { @@ -78,6 +83,44 @@ describe("auth facade, test mode", () => { }); }); +describe("auth facade, guests", () => { + beforeEach(() => { clerk.auth.mockResolvedValue({ userId: null } as never); }); + afterEach(() => { clerk.auth.mockResolvedValue({ userId: "user_clerk_1" }); }); + + it("a valid signed guest cookie is the session when there is no Clerk user", async () => { + process.env.GUEST_COOKIE_SECRET = GUEST_SECRET; + cookieStore.set(GUEST_COOKIE, await guestCookie()); + expect(await auth()).toEqual({ userId: GUEST }); + expect(await userEmailById(GUEST)).toBeNull(); + expect(clerk.getUser).not.toHaveBeenCalled(); + }); + + it("a Clerk user wins over a guest cookie", async () => { + process.env.GUEST_COOKIE_SECRET = GUEST_SECRET; + clerk.auth.mockResolvedValue({ userId: "user_clerk_1" }); + cookieStore.set(GUEST_COOKIE, await guestCookie()); + expect(await auth()).toEqual({ userId: "user_clerk_1" }); + }); + + it("ignores guest cookies when guest access is off, forged, or expired", async () => { + cookieStore.set(GUEST_COOKIE, await guestCookie()); + expect(await auth()).toEqual({ userId: null }); // GUEST_COOKIE_SECRET unset + process.env.GUEST_COOKIE_SECRET = GUEST_SECRET; + cookieStore.set(GUEST_COOKIE, await guestCookie("another-secret-0123456789abcdef0123456")); + expect(await auth()).toEqual({ userId: null }); + cookieStore.set(GUEST_COOKIE, await guestCookie(GUEST_SECRET, Math.floor(Date.now() / 1000) - 1)); + expect(await auth()).toEqual({ userId: null }); + }); + + it("test mode accepts a guest cookie too", async () => { + process.env.CLEANSTACK_TEST_MODE = "1"; + process.env.GUEST_COOKIE_SECRET = GUEST_SECRET; + cookieStore.set(GUEST_COOKIE, await guestCookie()); + expect(await auth()).toEqual({ userId: GUEST }); + expect(await currentUserEmail()).toBeNull(); + }); +}); + describe("isValidTestUserId", () => { it.each([["user_test_a", true], ["user_test_", false], ["user_test_a;b", false], ["user_test_" + "x".repeat(65), false], [42, false]])( "%s -> %s", (v, ok) => expect(isValidTestUserId(v)).toBe(ok)); diff --git a/src/lib/auth.ts b/src/lib/auth.ts index 2199b98..ce6cfe4 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -11,10 +11,18 @@ * outside test mode, and this module ignores the cookie outside test mode, so * the bypass cannot be used in a deployment even if the cookie is forged: * isTestMode() is false whenever VERCEL / AWS Lambda markers are present. + * + * Guests (src/lib/guest.ts): when there is no Clerk (or test) user, a valid + * signed `cs_guest` cookie makes `auth()` return the guest id (`guest_...`), + * which is then used as team_id like any user id. Guests have no email. + * Without GUEST_COOKIE_SECRET guest cookies are ignored. */ import { auth as clerkAuth, currentUser as clerkCurrentUser, clerkClient } from "@clerk/nextjs/server"; import { cookies } from "next/headers"; import { isTestMode } from "@/lib/test-mode"; +import { GUEST_COOKIE, guestFromCookie, isGuestId } from "@/lib/guest"; + +export { isGuestId } from "@/lib/guest"; export const TEST_USER_COOKIE = "cs_test_user"; @@ -35,14 +43,19 @@ async function testUserId(): Promise { return isValidTestUserId(value) ? value : null; } -/** `{ userId }` of the signed-in user, or `{ userId: null }`. */ +async function guestUserId(): Promise { + const session = await guestFromCookie((await cookies()).get(GUEST_COOKIE)?.value); + return session?.guestId ?? null; +} + +/** `{ userId }` of the signed-in user or guest (`guest_...`), or `{ userId: null }`. */ export async function auth(): Promise<{ userId: string | null }> { - if (isTestMode()) return { userId: await testUserId() }; + if (isTestMode()) return { userId: (await testUserId()) ?? (await guestUserId()) }; const { userId } = await clerkAuth(); - return { userId }; + return { userId: userId ?? (await guestUserId()) }; } -/** Primary email of the signed-in user, or null. */ +/** Primary email of the signed-in user, or null (always null for guests). */ export async function currentUserEmail(): Promise { if (isTestMode()) { const id = await testUserId(); @@ -52,8 +65,9 @@ export async function currentUserEmail(): Promise { return user?.primaryEmailAddress?.emailAddress ?? null; } -/** First email address of any user by id (used by webhook-driven routes), or null. */ +/** First email address of any user by id (used by webhook-driven routes), or null. Guests have none. */ export async function userEmailById(userId: string): Promise { + if (isGuestId(userId)) return null; if (isTestMode()) return isValidTestUserId(userId) ? testUserEmail(userId) : null; const clerk = await clerkClient(); const user = await clerk.users.getUser(userId).catch(() => null); diff --git a/src/lib/env.ts b/src/lib/env.ts index 10a4dbc..e7ccad0 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -57,6 +57,10 @@ export const ENV_SPEC = { UPSTASH_REDIS_REST_TOKEN: { validate: nonEmpty, required: false, description: "Upstash Redis token" }, NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: { validate: nonEmpty, required: true, description: "Clerk publishable key (read by @clerk/nextjs)" }, CLERK_SECRET_KEY: { validate: nonEmpty, required: true, description: "Clerk secret key (read by @clerk/nextjs)" }, + GUEST_COOKIE_SECRET: { validate: nonEmpty, required: false, description: "Signs guest session cookies (src/lib/guest.ts); guest access is OFF when unset or shorter than 32 chars" }, + TURNSTILE_SECRET_KEY: { validate: nonEmpty, required: false, description: "Cloudflare Turnstile secret; when set, starting a guest session requires a Turnstile token" }, + NEXT_PUBLIC_TURNSTILE_SITE_KEY: { validate: nonEmpty, required: false, description: "Cloudflare Turnstile site key for the guest button widget (set with TURNSTILE_SECRET_KEY)" }, + MAX_UPLOAD_MB: { validate: (v) => (Number(v) > 0 ? null : "must be a positive number"), required: false, description: "Per-file upload limit for signed-in users in MB (default 100; guests 2). Keep in sync with the profiler Lambda's MAX_UPLOAD_MB" }, LOG_LEVEL: { validate: (v) => (["debug", "info", "warn", "error"].includes(v) ? null : "must be debug|info|warn|error"), required: false, description: "Structured logger threshold (default info)" }, // Test-only (honoured only when src/lib/test-mode.ts isTestMode() is true) CLEANSTACK_TEST_MODE: { validate: (v) => (v === "1" || v === "0" ? null : 'must be "1" or "0"'), required: false, description: "TEST ONLY: enables the test-only switches; ignored on Vercel/Lambda" }, @@ -141,6 +145,13 @@ export function checkEnv(source: EnvSource = process.env): EnvReport { if (read("AI_QUEUE_ENABLED", source) === "true" && read("AI_JOBS_QUEUE_URL", source) === undefined) { errors.push("AI_JOBS_QUEUE_URL is not set but AI_QUEUE_ENABLED=true"); } + const guestSecretValue = read("GUEST_COOKIE_SECRET", source); + if (guestSecretValue !== undefined && guestSecretValue.length < 32) { + errors.push("GUEST_COOKIE_SECRET is shorter than 32 characters; guest access stays OFF"); + } + if ((read("TURNSTILE_SECRET_KEY", source) === undefined) !== (read("NEXT_PUBLIC_TURNSTILE_SITE_KEY", source) === undefined)) { + warnings.push("TURNSTILE_SECRET_KEY and NEXT_PUBLIC_TURNSTILE_SITE_KEY must be set together"); + } const hasRedisUrl = read("UPSTASH_REDIS_REST_URL", source) !== undefined; const hasRedisToken = read("UPSTASH_REDIS_REST_TOKEN", source) !== undefined; if (hasRedisUrl !== hasRedisToken) { diff --git a/src/lib/guest-limits.ts b/src/lib/guest-limits.ts new file mode 100644 index 0000000..ac8204c --- /dev/null +++ b/src/lib/guest-limits.ts @@ -0,0 +1,33 @@ +/** + * Guest tier. Derived from the Free plan (src/lib/billing.ts: 50,000 rows/month, + * src/lib/rate-limit.ts: 20 uploads/h, 50 AI calls/h, src/lib/bedrock-meter.ts: + * $50 soft / $200 hard AI spend per month) and scaled down for an anonymous + * 24-hour trial. Caps that protect money are counted in Postgres, so they hold + * even when Upstash is unavailable (the rate limiter fails open). + */ +import { GUEST_MAX_UPLOAD_BYTES } from "@/lib/upload-limits"; + +export const GUEST_LIMITS = { + /** Per file (also enforced by the S3 POST policy and the profiler). */ + maxUploadBytes: GUEST_MAX_UPLOAD_BYTES, + /** Rows in one uploaded file; checked after profiling, before any AI call. */ + rowsPerRun: 5_000, + /** Rows across all of a guest's uploads (first passes). */ + rowsPerGuest: 10_000, + /** First-pass uploads per guest (a guest lives 24 h). Sample data counts. */ + uploadsPerGuest: 3, + /** First-pass uploads per IP per 24 h, across all guests from that IP. */ + uploadsPerIpPerDay: 10, + /** Pipelines a guest can create. */ + pipelinesPerGuest: 5, + /** Bedrock calls per guest per rolling hour (bedrock_usage rows). */ + aiCallsPerHour: 10, + /** Estimated Bedrock spend per guest, lifetime. */ + aiSpendPerGuestUsd: 0.25, + /** Estimated Bedrock spend of ALL guests together since 00:00 UTC. */ + aiSpendAllGuestsPerDayUsd: 5, + /** New guest sessions per IP per 24 h. */ + guestsPerIpPerDay: 5, + /** New guest sessions per 24 h, all IPs. */ + guestsPerDay: 200, +} as const; diff --git a/src/lib/guest.test.ts b/src/lib/guest.test.ts new file mode 100644 index 0000000..3ac2404 --- /dev/null +++ b/src/lib/guest.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + GUEST_TTL_SECONDS, clientIp, guestAccessEnabled, guestFromCookie, hashIp, isGuestId, newGuestId, signGuestToken, verifyGuestToken, +} from "@/lib/guest"; + +const SECRET = "unit-test-guest-secret-0123456789abcdef"; +const now = 1_800_000_000; + +describe("guest ids", () => { + it("are guest_ + 22 base64url chars (128 random bits) and unique", () => { + const ids = new Set(Array.from({ length: 200 }, newGuestId)); + expect(ids.size).toBe(200); + for (const id of ids) expect(isGuestId(id)).toBe(true); + }); + + it.each([["guest_", false], ["guest_" + "a".repeat(21), false], ["guest_" + "a".repeat(23), false], + ["user_" + "a".repeat(22), false], ["guest_" + "a".repeat(21) + "/", false], [null, false]])("%s -> %s", (v, ok) => { + expect(isGuestId(v)).toBe(ok); + }); +}); + +describe("guest tokens", () => { + const session = { guestId: "guest_AAAAAAAAAAAAAAAAAAAAAA", expiresAt: now + 3600 }; + + it("round-trip", async () => { + const token = await signGuestToken(session, SECRET); + expect(await verifyGuestToken(token, SECRET, now)).toEqual(session); + }); + + it("reject another secret, tampering, expiry and over-long lifetimes", async () => { + const token = await signGuestToken(session, SECRET); + expect(await verifyGuestToken(token, SECRET + "x", now)).toBeNull(); + const [id, exp, sig] = token.split("."); + expect(await verifyGuestToken(`guest_BAAAAAAAAAAAAAAAAAAAAA.${exp}.${sig}`, SECRET, now)).toBeNull(); + expect(await verifyGuestToken(`${id}.${Number(exp) + 1}.${sig}`, SECRET, now)).toBeNull(); + expect(await verifyGuestToken(`${id}.${exp}.${sig.slice(0, -2)}AA`, SECRET, now)).toBeNull(); + expect(await verifyGuestToken(token, SECRET, now + 3600)).toBeNull(); + const tooLong = await signGuestToken({ ...session, expiresAt: now + GUEST_TTL_SECONDS + 3600 }, SECRET); + expect(await verifyGuestToken(tooLong, SECRET, now)).toBeNull(); + for (const junk of ["", "a.b", "a.b.c.d", `${id}.${exp}.!!!`, "x".repeat(500)]) { + expect(await verifyGuestToken(junk, SECRET, now)).toBeNull(); + } + }); + + it("guest access is off without a secret of at least 32 chars", async () => { + const token = await signGuestToken({ ...session, expiresAt: Math.floor(Date.now() / 1000) + 60 }, SECRET); + expect(guestAccessEnabled({})).toBe(false); + expect(guestAccessEnabled({ GUEST_COOKIE_SECRET: "short" })).toBe(false); + expect(await guestFromCookie(token, {})).toBeNull(); + expect(guestAccessEnabled({ GUEST_COOKIE_SECRET: SECRET })).toBe(true); + expect(await guestFromCookie(token, { GUEST_COOKIE_SECRET: SECRET })).not.toBeNull(); + }); +}); + +describe("client IP", () => { + it("uses the first X-Forwarded-For hop, then X-Real-IP", () => { + expect(clientIp(new Headers({ "x-forwarded-for": "203.0.113.7, 10.0.0.1" }))).toBe("203.0.113.7"); + expect(clientIp(new Headers({ "x-real-ip": "198.51.100.2" }))).toBe("198.51.100.2"); + expect(clientIp(new Headers())).toBe("unknown"); + }); + + it("is stored only as a keyed hash", async () => { + const h = await hashIp("203.0.113.7", SECRET); + expect(h).toBe(await hashIp("203.0.113.7", SECRET)); + expect(h).not.toBe(await hashIp("203.0.113.8", SECRET)); + expect(h).not.toContain("203"); + }); +}); diff --git a/src/lib/guest.ts b/src/lib/guest.ts new file mode 100644 index 0000000..bf75a93 --- /dev/null +++ b/src/lib/guest.ts @@ -0,0 +1,125 @@ +/** + * Guest access: try CleanStack without an account. + * + * A guest is a random id `guest_<22 base64url chars>` (128 bits) that is used + * as `team_id` everywhere a Clerk user id would be, so every existing + * `team_id = $userId` query and the `${userId}/` S3 prefix scope guest data + * exactly like a user's. The id lives in the `cs_guest` cookie as + * `...")>`: + * httpOnly, SameSite=Lax, Secure outside test mode, 24 h. It cannot be forged + * or extended without the secret. + * + * Guest access is OFF unless GUEST_COOKIE_SECRET is set (>= 32 chars): no + * cookie is issued and any cookie is ignored. Edge-safe (Web Crypto only), so + * src/middleware.ts can verify the cookie too. + */ +import type { EnvSource } from "@/lib/env"; + +export const GUEST_COOKIE = "cs_guest"; +export const GUEST_ID_PREFIX = "guest_"; +export const GUEST_TTL_SECONDS = 24 * 60 * 60; +const GUEST_ID = /^guest_[A-Za-z0-9_-]{22}$/; +const MIN_SECRET_LENGTH = 32; + +export interface GuestSession { + guestId: string; + /** Unix seconds. */ + expiresAt: number; +} + +export function isGuestId(id: unknown): id is string { + return typeof id === "string" && GUEST_ID.test(id); +} + +/** The signing secret, or null when guest access is disabled. */ +export function guestSecret(env: EnvSource = process.env): string | null { + const s = env.GUEST_COOKIE_SECRET?.trim(); + return s && s.length >= MIN_SECRET_LENGTH ? s : null; +} + +export function guestAccessEnabled(env: EnvSource = process.env): boolean { + return guestSecret(env) !== null; +} + +function base64url(bytes: Uint8Array): string { + let bin = ""; + for (const b of bytes) bin += String.fromCharCode(b); + return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +function fromBase64url(s: string): Uint8Array | null { + if (!/^[A-Za-z0-9_-]+$/.test(s)) return null; + try { + const bin = atob(s.replace(/-/g, "+").replace(/_/g, "/") + "===".slice((s.length + 3) % 4)); + return Uint8Array.from(bin, (c) => c.charCodeAt(0)); + } catch { + return null; + } +} + +export function newGuestId(): string { + return GUEST_ID_PREFIX + base64url(crypto.getRandomValues(new Uint8Array(16))); +} + +async function hmacKey(secret: string): Promise { + return crypto.subtle.importKey("raw", new TextEncoder().encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign", "verify"]); +} + +export async function signGuestToken(session: GuestSession, secret: string): Promise { + const payload = `${session.guestId}.${session.expiresAt}`; + const sig = await crypto.subtle.sign("HMAC", await hmacKey(secret), new TextEncoder().encode(payload)); + return `${payload}.${base64url(new Uint8Array(sig))}`; +} + +/** The session in a cookie value, or null if malformed, tampered with, or expired. */ +export async function verifyGuestToken( + token: string | undefined | null, + secret: string, + nowSeconds: number = Math.floor(Date.now() / 1000), +): Promise { + if (!token || token.length > 200) return null; + const parts = token.split("."); + if (parts.length !== 3) return null; + const [guestId, expRaw, sigRaw] = parts; + if (!isGuestId(guestId) || !/^\d{1,12}$/.test(expRaw)) return null; + const expiresAt = Number(expRaw); + if (expiresAt <= nowSeconds || expiresAt > nowSeconds + GUEST_TTL_SECONDS + 60) return null; + const sig = fromBase64url(sigRaw); + if (!sig) return null; + // crypto.subtle.verify compares in constant time. + const ok = await crypto.subtle.verify("HMAC", await hmacKey(secret), sig as BufferSource, new TextEncoder().encode(`${guestId}.${expRaw}`)); + return ok ? { guestId, expiresAt } : null; +} + +/** Verify a cookie value with GUEST_COOKIE_SECRET; null when guest access is off. */ +export async function guestFromCookie(value: string | undefined | null, env: EnvSource = process.env): Promise { + const secret = guestSecret(env); + if (!secret) return null; + return verifyGuestToken(value, secret); +} + +/** Cookie attributes for the guest cookie. `secure` is false only for plain-http test servers. */ +export function guestCookieOptions(expiresAt: number, secure: boolean) { + return { + httpOnly: true, + sameSite: "lax" as const, + secure, + path: "/", + expires: new Date(expiresAt * 1000), + }; +} + +/** + * Client IP for per-IP caps: the first X-Forwarded-For hop (Vercel sets it and + * strips client-supplied values), then X-Real-IP. "unknown" if neither. + */ +export function clientIp(headers: Headers): string { + const xff = headers.get("x-forwarded-for")?.split(",")[0]?.trim(); + return xff || headers.get("x-real-ip")?.trim() || "unknown"; +} + +/** HMAC of the IP so raw addresses are never stored. */ +export async function hashIp(ip: string, secret: string): Promise { + const sig = await crypto.subtle.sign("HMAC", await hmacKey(`ip:${secret}`), new TextEncoder().encode(ip)); + return base64url(new Uint8Array(sig)); +} diff --git a/src/lib/migrations/003_guest_sessions.sql b/src/lib/migrations/003_guest_sessions.sql new file mode 100644 index 0000000..833d90a --- /dev/null +++ b/src/lib/migrations/003_guest_sessions.sql @@ -0,0 +1,22 @@ +-- 003: guest access (src/lib/guest.ts). +-- One row per guest session. Guest data itself lives in the normal tables with +-- team_id = the guest id; this table backs the per-IP and global guest caps +-- and the 24 h purge (/api/cron/purge-guests). ip_hash is an HMAC of the IP, +-- never the address. Idempotent: safe to re-run. + +CREATE TABLE IF NOT EXISTS guest_sessions ( + id TEXT PRIMARY KEY, + ip_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL, + CHECK (left(id, 6) = 'guest_') +); + +CREATE INDEX IF NOT EXISTS idx_guest_sessions_ip_created + ON guest_sessions (ip_hash, created_at); + +CREATE INDEX IF NOT EXISTS idx_guest_sessions_created + ON guest_sessions (created_at); + +CREATE INDEX IF NOT EXISTS idx_guest_sessions_expires + ON guest_sessions (expires_at); diff --git a/src/lib/schema.sql b/src/lib/schema.sql index db3d516..17e991d 100644 --- a/src/lib/schema.sql +++ b/src/lib/schema.sql @@ -151,6 +151,22 @@ CREATE TABLE IF NOT EXISTS ai_spend_limits ( CHECK (soft_cap_usd >= 0 AND hard_cap_usd >= soft_cap_usd) ); +-- Guest sessions (also applied via 003_guest_sessions.sql) +CREATE TABLE IF NOT EXISTS guest_sessions ( + id TEXT PRIMARY KEY, + ip_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL, + CHECK (left(id, 6) = 'guest_') +); + +CREATE INDEX IF NOT EXISTS idx_guest_sessions_ip_created + ON guest_sessions (ip_hash, created_at); +CREATE INDEX IF NOT EXISTS idx_guest_sessions_created + ON guest_sessions (created_at); +CREATE INDEX IF NOT EXISTS idx_guest_sessions_expires + ON guest_sessions (expires_at); + -- Indexes CREATE INDEX IF NOT EXISTS idx_pipelines_team_id ON pipelines(team_id); CREATE INDEX IF NOT EXISTS idx_pipeline_runs_pipeline_id ON pipeline_runs(pipeline_id); diff --git a/src/lib/schema.test.ts b/src/lib/schema.test.ts index dc166d2..370f015 100644 --- a/src/lib/schema.test.ts +++ b/src/lib/schema.test.ts @@ -29,7 +29,7 @@ const migrated = tables(migrationFiles.map((f) => readFileSync(join(MIGRATIONS, const APP_TABLES = [ "pipelines", "pipeline_runs", "data_profiles", "transform_rules", "approval_reviews", "schema_snapshots", "pipeline_templates", "pipeline_destinations", "subscriptions", - "bedrock_usage", "ai_spend_limits", + "bedrock_usage", "ai_spend_limits", "guest_sessions", ]; describe("database schema", () => { @@ -44,6 +44,11 @@ describe("database schema", () => { } }); + it("existing databases get guest_sessions from 003", () => { + expect(migrationFiles).toContain("003_guest_sessions.sql"); + expect(migrated.get("guest_sessions")).toEqual(schema.get("guest_sessions")); + }); + it("bedrock_usage has every column bedrock-meter inserts", () => { const src = readFileSync(join(LIB, "bedrock-meter.ts"), "utf8"); const cols = /INSERT INTO bedrock_usage \(([^)]+)\)/.exec(src)![1].split(",").map((c) => c.trim()); diff --git a/src/lib/turnstile.ts b/src/lib/turnstile.ts new file mode 100644 index 0000000..7bc3245 --- /dev/null +++ b/src/lib/turnstile.ts @@ -0,0 +1,30 @@ +/** + * Optional Cloudflare Turnstile check for guest sign-up. Active only when + * TURNSTILE_SECRET_KEY is set (the widget needs NEXT_PUBLIC_TURNSTILE_SITE_KEY). + */ +import { optionalEnv } from "@/lib/env"; +import { logger } from "@/lib/logger"; + +const log = logger.child({ component: "turnstile" }); +const VERIFY_URL = "https://challenges.cloudflare.com/turnstile/v0/siteverify"; + +export function turnstileEnabled(): boolean { + return optionalEnv("TURNSTILE_SECRET_KEY") !== undefined; +} + +/** true when Turnstile is off, or the token verifies. Fails closed on network errors. */ +export async function verifyTurnstile(token: unknown, ip: string): Promise { + const secret = optionalEnv("TURNSTILE_SECRET_KEY"); + if (!secret) return true; + if (typeof token !== "string" || !token || token.length > 2048) return false; + try { + const body = new URLSearchParams({ secret, response: token }); + if (ip !== "unknown") body.set("remoteip", ip); + const res = await fetch(VERIFY_URL, { method: "POST", body, signal: AbortSignal.timeout(5000) }); + const data = (await res.json().catch(() => ({}))) as { success?: boolean }; + return data.success === true; + } catch (err) { + log.warn("Turnstile verification failed; refusing guest session", { err }); + return false; + } +} diff --git a/src/middleware.guest.test.ts b/src/middleware.guest.test.ts new file mode 100644 index 0000000..f444e3e --- /dev/null +++ b/src/middleware.guest.test.ts @@ -0,0 +1,42 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { NextRequest, type NextFetchEvent } from "next/server"; +import middleware from "@/middleware"; +import { GUEST_COOKIE, signGuestToken } from "@/lib/guest"; + +// Test-mode branch of the middleware (the Clerk branch needs real keys); it applies +// the same protected-route and guest rules as the Clerk branch. +const SECRET = "unit-test-guest-secret-0123456789abcdef"; +const saved = { ...process.env }; +beforeEach(() => { + process.env.CLEANSTACK_TEST_MODE = "1"; + process.env.GUEST_COOKIE_SECRET = SECRET; + for (const k of ["VERCEL", "VERCEL_ENV", "VERCEL_URL", "AWS_LAMBDA_FUNCTION_NAME", "AWS_EXECUTION_ENV"]) delete process.env[k]; +}); +afterEach(() => { process.env = { ...saved }; }); + +const guestToken = (exp = Math.floor(Date.now() / 1000) + 600) => + signGuestToken({ guestId: "guest_AAAAAAAAAAAAAAAAAAAAAA", expiresAt: exp }, SECRET); +const run = (path: string, cookie?: string) => + middleware(new NextRequest(`http://localhost:3000${path}`, { headers: cookie ? { cookie } : {} }), {} as NextFetchEvent) as Promise; + +describe("middleware: guests", () => { + it("a valid guest cookie passes protected pages and APIs", async () => { + const cookie = `${GUEST_COOKIE}=${await guestToken()}`; + for (const path of ["/dashboard", "/pipelines/new", "/api/upload", "/api/pipelines"]) { + const res = await run(path, cookie); + expect(res.headers.get("x-middleware-next"), path).toBe("1"); + } + const root = await run("/", cookie); + expect(root.headers.get("location")).toBe("http://localhost:3000/dashboard"); + }); + + it("forged, expired or disabled guest cookies do not", async () => { + const forged = `${GUEST_COOKIE}=guest_AAAAAAAAAAAAAAAAAAAAAA.9999999999.AAAA`; + expect((await run("/api/upload", forged)).status).toBe(401); + const expired = `${GUEST_COOKIE}=${await guestToken(Math.floor(Date.now() / 1000) - 5)}`; + expect((await run("/dashboard", expired)).headers.get("location")).toContain("/sign-in"); + const valid = `${GUEST_COOKIE}=${await guestToken()}`; + delete process.env.GUEST_COOKIE_SECRET; + expect((await run("/api/pipelines", valid)).status).toBe(401); + }); +}); diff --git a/src/middleware.ts b/src/middleware.ts index b022a61..8dadaf3 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -1,6 +1,7 @@ import { clerkMiddleware, createRouteMatcher } from "@clerk/nextjs/server"; import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server"; import { isTestMode } from "@/lib/test-mode"; +import { GUEST_COOKIE, guestFromCookie } from "@/lib/guest"; const isProtectedRoute = createRouteMatcher([ "/dashboard(.*)", @@ -21,16 +22,25 @@ const isProtectedRoute = createRouteMatcher([ const isRootPath = createRouteMatcher(["/"]); +/** A valid signed guest cookie (src/lib/guest.ts); always false when guest access is off. */ +async function hasGuestSession(req: NextRequest): Promise { + return (await guestFromCookie(req.cookies.get(GUEST_COOKIE)?.value)) !== null; +} + const clerkHandler = clerkMiddleware(async (auth, req) => { if (isProtectedRoute(req)) { + // Clerk users first; otherwise a guest may pass, otherwise Clerk's normal redirect / 401. + const { userId } = await auth(); + if (userId) return; + if (await hasGuestSession(req)) return; await auth.protect(); return; } - // Redirect authenticated users from landing page → dashboard + // Redirect authenticated users (and guests) from landing page → dashboard if (isRootPath(req)) { const { userId } = await auth(); - if (userId) { + if (userId || (await hasGuestSession(req))) { return NextResponse.redirect(new URL("/dashboard", req.url)); } } @@ -45,9 +55,9 @@ const clerkHandler = clerkMiddleware(async (auth, req) => { const TEST_USER_COOKIE = "cs_test_user"; // keep in sync with src/lib/auth.ts const TEST_USER_ID = /^user_test_[A-Za-z0-9_-]{1,64}$/; -function testModeMiddleware(req: NextRequest) { +async function testModeMiddleware(req: NextRequest) { const userId = req.cookies.get(TEST_USER_COOKIE)?.value; - const signedIn = userId !== undefined && TEST_USER_ID.test(userId); + const signedIn = (userId !== undefined && TEST_USER_ID.test(userId)) || (await hasGuestSession(req)); if (isProtectedRoute(req) && !signedIn) { if (req.nextUrl.pathname.startsWith("/api/")) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); diff --git a/tests/e2e/test-mode-guard.spec.ts b/tests/e2e/test-mode-guard.spec.ts index fe573c9..3ea1aee 100644 --- a/tests/e2e/test-mode-guard.spec.ts +++ b/tests/e2e/test-mode-guard.spec.ts @@ -30,5 +30,13 @@ for (const { name, url } of servers) { expect([401, 403, 404], `${path} -> ${res.status()}`).toContain(res.status()); } }); + + test("a forged cs_guest cookie is not a session", async ({ request }) => { + const headers = { cookie: "cs_guest=guest_AAAAAAAAAAAAAAAAAAAAAA.9999999999.forged" }; + for (const path of ["/api/usage", "/api/pipelines", "/dashboard"]) { + const res = await request.get(`${url}${path}`, { headers, maxRedirects: 0 }); + expect([401, 403, 404], `${path} -> ${res.status()}`).toContain(res.status()); + } + }); }); } diff --git a/tests/support/test-env.mjs b/tests/support/test-env.mjs index b674f25..79e50f2 100644 --- a/tests/support/test-env.mjs +++ b/tests/support/test-env.mjs @@ -33,6 +33,8 @@ export const TEST_ENV_DEFAULTS = { APP_URL: "http://localhost:3000", WEBHOOK_SECRET: "test-webhook-secret-0123456789abcdef0123", CRON_SECRET: "test-cron-secret-0123456789abcdef01234567", + // Enables guest access (src/lib/guest.ts) in the suites. + GUEST_COOKIE_SECRET: "test-guest-cookie-secret-0123456789abcdef", // Never used with DB_DRIVER=pg; set so requireEnv-style config checks pass. AURORA_CLUSTER_ARN: "arn:aws:rds:us-east-1:000000000000:cluster:unused-in-tests", AURORA_SECRET_ARN: "arn:aws:secretsmanager:us-east-1:000000000000:secret:unused-in-tests", From c2d0622c508fdf6551b00b4a7b67815ea4ed8623 Mon Sep 17 00:00:00 2001 From: Dhyey Desai Date: Fri, 2 Oct 2026 11:56:29 -0400 Subject: [PATCH 2/3] Cap upload size for everyone: presigned POST with content-length-range A presigned PUT cannot limit the body size, so /api/upload now returns a presigned POST whose policy pins the key and content type and carries content-length-range (100 MB, MAX_UPLOAD_MB; guests 2 MB). The route refuses a declared size over the limit with 413 before creating a run, GET returns the caller's limit so the form refuses a large file before creating a pipeline, and the profiler checks ContentLength before reading the object and fails the run (backstop for clients that bypass the form; LocalStack does not enforce POST policies, so tests cover this path). Deploy note: the raw bucket CORS must allow POST and the profiler trigger must include s3:ObjectCreated:Post. --- lambdas/profiler/handler.py | 47 ++++++++++ .../integration/test_profiler_s3_event.py | 29 ++++++ lambdas/tests/test_profiler_limits.py | 32 +++++++ package-lock.json | 19 ++++ package.json | 1 + src/app/api/upload/presign.test.ts | 39 +++++--- src/app/api/upload/route.test.ts | 56 ++++++++++-- src/app/api/upload/route.ts | 54 +++++++++--- src/app/pipelines/new/page.tsx | 41 ++++++--- src/lib/upload-limits.ts | 29 ++++++ tests/e2e/guest.spec.ts | 57 ++++++++++++ tests/integration/guest-access.test.ts | 88 +++++++++++++++++++ tests/integration/helpers.ts | 8 ++ tests/integration/upload-to-download.test.ts | 18 ++-- tests/support/setup-services.mjs | 4 +- 15 files changed, 466 insertions(+), 56 deletions(-) create mode 100644 lambdas/tests/test_profiler_limits.py create mode 100644 src/lib/upload-limits.ts create mode 100644 tests/e2e/guest.spec.ts create mode 100644 tests/integration/guest-access.test.ts diff --git a/lambdas/profiler/handler.py b/lambdas/profiler/handler.py index e3e189b..b1436f0 100644 --- a/lambdas/profiler/handler.py +++ b/lambdas/profiler/handler.py @@ -569,6 +569,46 @@ def _effective_null_pct(s: pd.Series) -> float: return signals +# Upload size limits; keep in sync with src/lib/upload-limits.ts. +GUEST_MAX_UPLOAD_BYTES = 2 * 1024 * 1024 + + +def max_upload_bytes(key: str) -> int: + """Per-file limit for the uploader: guests (key prefix guest_) 2 MB, others MAX_UPLOAD_MB (default 100).""" + if key.startswith("guest_"): + return GUEST_MAX_UPLOAD_BYTES + try: + mb = float(os.environ.get("MAX_UPLOAD_MB", "100")) + except ValueError: + mb = 100.0 + return int((mb if mb > 0 else 100.0) * 1024 * 1024) + + +def _mb(n: int) -> str: + v = n / (1024 * 1024) + return f"{v:.0f} MB" if v == int(v) else f"{v:.1f} MB" + + +def _reject_too_large(run_id: str, size: int, limit: int) -> dict: + """Fail a pending run whose raw object exceeds the upload limit (no profiling, no AI).""" + message = f"File is {_mb(size)}; the upload limit is {_mb(limit)} per file." + print(f"[profiler] run {run_id}: {message}") + conn = get_db_conn() + cur = conn.cursor() + try: + cur.execute( + "UPDATE pipeline_runs SET status = 'failed', error_message = %s, updated_at = now() " + "WHERE id = %s AND status = 'pending' RETURNING id", + (message, run_id), + ) + claimed = cur.fetchone() + conn.commit() + finally: + cur.close() + conn.close() + return {"statusCode": 200, "run_id": run_id, "rejected": "too_large" if claimed else "already_claimed"} + + def handler(event, context): record = event["Records"][0]["s3"] bucket = record["bucket"]["name"] @@ -580,6 +620,13 @@ def handler(event, context): return {"statusCode": 200, "body": "skipped"} obj = s3.get_object(Bucket=bucket, Key=key) + size = int(obj.get("ContentLength") or 0) + limit = max_upload_bytes(key) + if size > limit: + # Backstop for the presigned POST policy (content-length-range): never read + # an oversized object into memory. Mark the run failed and stop. + obj["Body"].close() + return _reject_too_large(parts[2], size, limit) file_bytes = obj["Body"].read() # Strip UTF-8 BOM if present (prevents invisible char in first column name) if file_bytes[:3] == b'\xef\xbb\xbf': diff --git a/lambdas/tests/integration/test_profiler_s3_event.py b/lambdas/tests/integration/test_profiler_s3_event.py index 8993845..c9d3b94 100644 --- a/lambdas/tests/integration/test_profiler_s3_event.py +++ b/lambdas/tests/integration/test_profiler_s3_event.py @@ -48,3 +48,32 @@ def test_profiler_handles_real_s3_notification(live_profiler, services, db, monk finally: with db.cursor() as cur: cur.execute("DELETE FROM pipelines WHERE id = %s", (pipeline_id,)) + + +def test_profiler_rejects_an_oversized_guest_upload_without_reading_it(live_profiler, services, db, monkeypatch): + """Backstop for the presigned POST size policy (LocalStack does not enforce POST policies).""" + s3 = services["s3"] + team = "guest_" + "A" * 22 + pipeline_id, run_id = str(uuid.uuid4()), str(uuid.uuid4()) + key = f"{team}/{pipeline_id}/{run_id}/raw.csv" + with db.cursor() as cur: + cur.execute("INSERT INTO pipelines (id, name, owner_id, team_id) VALUES (%s, 'it', %s, %s)", (pipeline_id, team, team)) + cur.execute("INSERT INTO pipeline_runs (id, pipeline_id, status, file_format, raw_s3_key) VALUES (%s, %s, 'pending', 'csv', %s)", + (run_id, pipeline_id, key)) + calls = [] + monkeypatch.setattr(live_profiler.requests, "post", lambda url, **kw: calls.append(url)) + try: + body = b"id,v\n" + b"1,x\n" * (2 * 1024 * 1024 // 4 + 10) # just over 2 MB + s3.put_object(Bucket=os.environ["S3_RAW_BUCKET"], Key=key, ContentType="text/csv", Body=body) + event = {"Records": [{"s3": {"bucket": {"name": os.environ["S3_RAW_BUCKET"]}, "object": {"key": key}}}]} + assert live_profiler.handler(event, None) == {"statusCode": 200, "run_id": run_id, "rejected": "too_large"} + with db.cursor() as cur: + cur.execute("SELECT status, error_message FROM pipeline_runs WHERE id = %s", (run_id,)) + status, message = cur.fetchone() + assert status == "failed" + assert message == "File is 2.0 MB; the upload limit is 2 MB per file." + assert calls == [] # no profile-complete webhook, so no AI + finally: + with db.cursor() as cur: + cur.execute("DELETE FROM pipelines WHERE id = %s", (pipeline_id,)) + s3.delete_object(Bucket=os.environ["S3_RAW_BUCKET"], Key=key) diff --git a/lambdas/tests/test_profiler_limits.py b/lambdas/tests/test_profiler_limits.py new file mode 100644 index 0000000..59835a3 --- /dev/null +++ b/lambdas/tests/test_profiler_limits.py @@ -0,0 +1,32 @@ +"""Profiler upload size limits (mirror of src/lib/upload-limits.ts).""" +import pytest + + +def test_guest_keys_get_the_2mb_limit(profiler): + assert profiler.max_upload_bytes("guest_abcdefghijklmnopqrstuv/p/r/raw.csv") == 2 * 1024 * 1024 + + +@pytest.mark.parametrize("env,expected_mb", [(None, 100), ("50", 50), ("0.5", 0.5), ("junk", 100), ("-1", 100)]) +def test_user_limit_defaults_to_100mb_and_honours_max_upload_mb(profiler, monkeypatch, env, expected_mb): + if env is None: + monkeypatch.delenv("MAX_UPLOAD_MB", raising=False) + else: + monkeypatch.setenv("MAX_UPLOAD_MB", env) + assert profiler.max_upload_bytes("user_2abc/p/r/raw.csv") == int(expected_mb * 1024 * 1024) + + +def test_oversized_object_is_rejected_before_reading(profiler, monkeypatch): + class Body: + closed = False + def read(self): + raise AssertionError("must not read an oversized object") + def close(self): + Body.closed = True + + monkeypatch.setattr(profiler.s3, "get_object", lambda **kw: {"Body": Body(), "ContentLength": 3 * 1024 * 1024}) + rejected = [] + monkeypatch.setattr(profiler, "_reject_too_large", lambda run_id, size, limit: rejected.append((run_id, size, limit)) or {"rejected": "too_large"}) + event = {"Records": [{"s3": {"bucket": {"name": "b"}, "object": {"key": "guest_abcdefghijklmnopqrstuv/p/run-1/raw.csv"}}}]} + assert profiler.handler(event, None) == {"rejected": "too_large"} + assert rejected == [("run-1", 3 * 1024 * 1024, 2 * 1024 * 1024)] + assert Body.closed diff --git a/package-lock.json b/package-lock.json index 55ef018..2101771 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@aws-sdk/client-s3": "^3.1068.0", "@aws-sdk/client-sqs": "^3.1068.0", "@aws-sdk/rds-signer": "^3.1070.0", + "@aws-sdk/s3-presigned-post": "^3.1068.0", "@aws-sdk/s3-request-presigner": "^3.1068.0", "@clerk/nextjs": "^7.5.2", "@e965/xlsx": "^0.20.3", @@ -739,6 +740,24 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/s3-presigned-post": { + "version": "3.1068.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/s3-presigned-post/-/s3-presigned-post-3.1068.0.tgz", + "integrity": "sha512-c6lXKsClR1/h8jEtKgkw12O7RXWMxtQ0fasBTV2pPalkHLZfk/VSbQuKbLz3a6iXLZMq8RKhTxf/EM0mmAdA3w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-s3": "3.1068.0", + "@aws-sdk/core": "^3.974.20", + "@aws-sdk/types": "^3.973.12", + "@smithy/core": "^3.24.6", + "@smithy/signature-v4": "^5.4.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/s3-request-presigner": { "version": "3.1068.0", "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.1068.0.tgz", diff --git a/package.json b/package.json index 22cc41b..c5a167b 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "@aws-sdk/client-s3": "^3.1068.0", "@aws-sdk/client-sqs": "^3.1068.0", "@aws-sdk/rds-signer": "^3.1070.0", + "@aws-sdk/s3-presigned-post": "^3.1068.0", "@aws-sdk/s3-request-presigner": "^3.1068.0", "@clerk/nextjs": "^7.5.2", "@e965/xlsx": "^0.20.3", diff --git a/src/app/api/upload/presign.test.ts b/src/app/api/upload/presign.test.ts index ac6bcea..114a1dd 100644 --- a/src/app/api/upload/presign.test.ts +++ b/src/app/api/upload/presign.test.ts @@ -1,26 +1,37 @@ import { describe, expect, it } from "vitest"; -import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; -import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { S3Client } from "@aws-sdk/client-s3"; +import { createPresignedPost } from "@aws-sdk/s3-presigned-post"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; const credentials = { accessKeyId: "AKIDEXAMPLE", secretAccessKey: "secret" }; -const presign = (client: S3Client) => - getSignedUrl(client, new PutObjectCommand({ Bucket: "b", Key: "u/p/r/raw.csv", ContentType: "text/csv" }), { expiresIn: 300 }); -describe("upload presigned PUT URL", () => { - it("the SDK default bakes an empty-body checksum into the URL (why the route opts out)", async () => { - const url = await presign(new S3Client({ region: "us-east-1", credentials })); - expect(url).toContain("x-amz-checksum-crc32=AAAAAA%3D%3D"); +describe("upload presigned POST", () => { + it("signs a policy that S3 enforces: exact key, size range and content type", async () => { + // Real AWS endpoint even when the shell points the SDK at LocalStack. + const client = new S3Client({ region: "us-east-1", credentials, requestChecksumCalculation: "WHEN_REQUIRED", endpoint: "https://s3.us-east-1.amazonaws.com" }); + const { url, fields } = await createPresignedPost(client, { + Bucket: "cleanstack-raw-test", Key: "u/p/r/raw.csv", Expires: 300, + Conditions: [["content-length-range", 1, 2 * 1024 * 1024], ["eq", "$Content-Type", "text/csv"]], + Fields: { "Content-Type": "text/csv" }, + }); + expect(url).toMatch(/^https:\/\/cleanstack-raw-test\.s3\.(us-east-1\.)?amazonaws\.com\/$/); + expect(fields.key).toBe("u/p/r/raw.csv"); + const policy = JSON.parse(Buffer.from(fields.Policy, "base64").toString("utf8")); + expect(policy.conditions).toEqual(expect.arrayContaining([ + ["content-length-range", 1, 2 * 1024 * 1024], + ["eq", "$Content-Type", "text/csv"], + { key: "u/p/r/raw.csv" }, + { bucket: "cleanstack-raw-test" }, + ])); + expect(Object.keys(fields).join(",")).not.toMatch(/checksum/i); }); - it("with WHEN_REQUIRED the URL carries no checksum, so any body can be uploaded", async () => { - const url = await presign(new S3Client({ region: "us-east-1", credentials, requestChecksumCalculation: "WHEN_REQUIRED" })); - expect(url).not.toMatch(/x-amz-checksum|x-amz-sdk-checksum-algorithm/); - }); - - it("the upload route's presigning client uses WHEN_REQUIRED", () => { + it("the upload route presigns a POST (not a PUT) with a WHEN_REQUIRED client", () => { const src = readFileSync(fileURLToPath(new URL("./route.ts", import.meta.url)), "utf8"); expect(src).toMatch(/new S3Client\(\{[^}]*requestChecksumCalculation: "WHEN_REQUIRED"/); + expect(src).toContain("createPresignedPost("); + expect(src).toContain('["content-length-range", 1, maxBytes]'); + expect(src).not.toContain("PutObjectCommand"); }); }); diff --git a/src/app/api/upload/route.test.ts b/src/app/api/upload/route.test.ts index 5d3fce4..8e7c019 100644 --- a/src/app/api/upload/route.test.ts +++ b/src/app/api/upload/route.test.ts @@ -7,7 +7,7 @@ const h = vi.hoisted(() => ({ limit: vi.fn(), queryOneWithTeam: vi.fn(), queryOne: vi.fn(), - getSignedUrl: vi.fn(), + createPresignedPost: vi.fn(), })); vi.mock("@/lib/auth", () => ({ auth: () => h.auth(), currentUserEmail: () => h.email() })); @@ -16,7 +16,7 @@ vi.mock("@/lib/db", () => ({ queryOneWithTeam: (...a: unknown[]) => h.queryOneWithTeam(...a), queryOne: (...a: unknown[]) => h.queryOne(...a), })); -vi.mock("@aws-sdk/s3-request-presigner", () => ({ getSignedUrl: (...a: unknown[]) => h.getSignedUrl(...a) })); +vi.mock("@aws-sdk/s3-presigned-post", () => ({ createPresignedPost: (...a: unknown[]) => h.createPresignedPost(...a) })); // Real checkRateLimit, with a limiter whose Upstash calls we control. vi.mock("@/lib/rate-limit", async (importOriginal) => ({ ...(await importOriginal()), @@ -27,10 +27,11 @@ const savedBucket = process.env.S3_RAW_BUCKET; process.env.S3_RAW_BUCKET = "raw-bucket"; afterAll(() => { if (savedBucket === undefined) delete process.env.S3_RAW_BUCKET; else process.env.S3_RAW_BUCKET = savedBucket; }); -const { POST } = await import("./route"); +const { GET, POST } = await import("./route"); +const POST_UPLOAD = { url: "https://raw-bucket.s3.amazonaws.com/", fields: { key: "k", Policy: "p" } }; const OK_QUOTA = { blocked: false, used: 0, includedRows: 1000, plan: "free" }; -const call = (body: unknown = { pipeline_id: "11111111-2222-3333-4444-555555555555", filename: "Email-Tracking.csv" }) => +const call = (body: unknown = { pipeline_id: "11111111-2222-3333-4444-555555555555", filename: "Email-Tracking.csv", size: 2_500 }) => POST(new Request("http://x/api/upload", { method: "POST", body: typeof body === "string" ? body : JSON.stringify(body) }) as never); beforeEach(() => { @@ -41,14 +42,53 @@ beforeEach(() => { h.limit.mockResolvedValue({ success: true, limit: 20, remaining: 19, reset: 0 }); h.queryOneWithTeam.mockResolvedValue({ id: "p" }); h.queryOne.mockResolvedValue({ id: "run_1" }); - h.getSignedUrl.mockResolvedValue("https://raw-bucket.s3.amazonaws.com/put"); + h.createPresignedPost.mockResolvedValue(POST_UPLOAD); +}); + +describe("GET /api/upload", () => { + it("returns the caller's per-file limit", async () => { + expect(await (await GET()).json()).toEqual({ max_bytes: 100 * 1024 * 1024 }); + h.auth.mockResolvedValue({ userId: "guest_" + "a".repeat(22) }); + expect(await (await GET()).json()).toEqual({ max_bytes: 2 * 1024 * 1024 }); + h.auth.mockResolvedValue({ userId: null }); + expect((await GET()).status).toBe(401); + }); }); describe("POST /api/upload", () => { - it("returns a presigned URL", async () => { + it("returns a presigned POST whose policy caps the size and pins the content type", async () => { + const res = await call(); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ upload: POST_UPLOAD, run_id: "run_1", max_bytes: 100 * 1024 * 1024 }); + const [, params] = h.createPresignedPost.mock.calls[0]; + expect(params).toMatchObject({ + Bucket: "raw-bucket", + Key: expect.stringMatching(/^user_1\/11111111-2222-3333-4444-555555555555\/[0-9a-f-]{36}\/raw\.csv$/), + Conditions: [["content-length-range", 1, 100 * 1024 * 1024], ["eq", "$Content-Type", "text/csv"]], + Fields: { "Content-Type": "text/csv" }, + Expires: 300, + }); + }); + + it("guests get a 2 MB policy", async () => { + h.auth.mockResolvedValue({ userId: "guest_" + "a".repeat(22) }); const res = await call(); expect(res.status).toBe(200); - expect(await res.json()).toMatchObject({ presigned_url: "https://raw-bucket.s3.amazonaws.com/put", run_id: "run_1" }); + expect(h.createPresignedPost.mock.calls[0][1].Conditions[0]).toEqual(["content-length-range", 1, 2 * 1024 * 1024]); + }); + + it("refuses a declared size over the limit with 413 before creating a run", async () => { + h.auth.mockResolvedValue({ userId: "guest_" + "a".repeat(22) }); + const res = await call({ pipeline_id: "p", filename: "big.csv", size: 2 * 1024 * 1024 + 1 }); + expect(res.status).toBe(413); + expect(await res.json()).toEqual({ error: "File is 2.0 MB; the limit is 2 MB per file.", max_bytes: 2 * 1024 * 1024 }); + expect(h.queryOne).not.toHaveBeenCalled(); + }); + + it("requires the file size", async () => { + const res = await call({ pipeline_id: "p", filename: "a.csv" }); + expect(res.status).toBe(400); + expect(h.queryOne).not.toHaveBeenCalled(); }); it("still uploads when Upstash fails DNS (the production 500 with an empty body)", async () => { @@ -82,7 +122,7 @@ describe("POST /api/upload", () => { }); it("answers JSON when presigning fails", async () => { - h.getSignedUrl.mockRejectedValue(new Error("no credentials")); + h.createPresignedPost.mockRejectedValue(new Error("no credentials")); const res = await call(); expect(res.status).toBe(500); expect((await res.json()).stage).toBe("presign"); diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index 39fa365..a5ff8e4 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -1,23 +1,27 @@ import { auth, currentUserEmail } from "@/lib/auth"; import { NextRequest, NextResponse } from "next/server"; import { randomUUID } from "crypto"; -import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; -import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { S3Client } from "@aws-sdk/client-s3"; +import { createPresignedPost } from "@aws-sdk/s3-presigned-post"; import { queryOne, queryOneWithTeam } from "@/lib/db"; import { getCachedQuota } from "@/lib/quota-cache"; import { uploadLimiter, checkRateLimit } from "@/lib/rate-limit"; import type { PipelineRun } from "@/lib/types"; import { requireEnv, awsRegion } from "@/lib/env"; import { logger } from "@/lib/logger"; +import { formatMb, maxUploadBytesFor } from "@/lib/upload-limits"; const log = logger.child({ route: "POST /api/upload" }); -// Presigning only. Since SDK 3.729 the default (WHEN_SUPPORTED) bakes a CRC32 of the -// *empty* body (x-amz-checksum-crc32=AAAAAA==) into presigned PutObject URLs, so the -// browser's real upload does not match it (LocalStack rejects it with 400; see -// aws/aws-sdk-js-v3#6810). WHEN_REQUIRED restores the pre-3.729 URL shape. +// Presigning only. The upload is a presigned POST (not PUT) so the policy can carry +// content-length-range: S3 itself rejects a body over the limit. WHEN_REQUIRED keeps +// the SDK from adding checksum fields the browser's body would not match +// (aws/aws-sdk-js-v3#6810). const s3 = new S3Client({ region: awsRegion(), requestChecksumCalculation: "WHEN_REQUIRED" }); +/** Presigned POST lifetime. */ +const UPLOAD_EXPIRES_SECONDS = 300; + const ALLOWED_EXTENSIONS = new Set([ "csv", "tsv", "txt", "json", "jsonl", "xlsx", "xls", "xml", @@ -51,7 +55,11 @@ const STAGE_MESSAGES: Record = { }; /** - * Creates a pending run and returns a presigned S3 PUT URL for the raw file. + * Creates a pending run and returns a presigned S3 POST (url + form fields) for + * the raw file. Body: { pipeline_id, filename, size } (size in bytes). A file + * over the caller's limit (src/lib/upload-limits.ts: 100 MB, guests 2 MB) is + * refused here with 413; the POST policy's content-length-range makes S3 + * enforce the same limit on the actual bytes, and the profiler checks again. * * Every step runs inside one try/catch. Previously auth, rate limiting, the * Clerk email lookup and the quota query ran outside it, so any throw there @@ -62,6 +70,13 @@ const STAGE_MESSAGES: Record = { * that failed and are logged with the same `stage`, so Vercel runtime logs * show the underlying error. */ +/** The caller's per-file limit, so the upload form can refuse a large file before creating anything. */ +export async function GET() { + const { userId } = await auth(); + if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + return NextResponse.json({ max_bytes: maxUploadBytesFor(userId) }); +} + export async function POST(req: NextRequest) { let stage: Stage = "auth"; try { @@ -95,11 +110,21 @@ export async function POST(req: NextRequest) { stage = "parse_body"; const body = await req.json().catch(() => null); - const { pipeline_id, filename } = (body ?? {}) as { pipeline_id?: string; filename?: string }; + const { pipeline_id, filename, size } = (body ?? {}) as { pipeline_id?: string; filename?: string; size?: unknown }; if (!pipeline_id || !filename) { return NextResponse.json({ error: "pipeline_id and filename required" }, { status: 400 }); } + const maxBytes = maxUploadBytesFor(userId); + if (typeof size !== "number" || !Number.isFinite(size) || size <= 0) { + return NextResponse.json({ error: "size (file size in bytes) required" }, { status: 400 }); + } + if (size > maxBytes) { + return NextResponse.json( + { error: `File is ${formatMb(size)}; the limit is ${formatMb(maxBytes)} per file.`, max_bytes: maxBytes }, + { status: 413 } + ); + } const ext = filename.split(".").pop()?.toLowerCase() ?? ""; if (!ALLOWED_EXTENSIONS.has(ext)) { @@ -131,15 +156,18 @@ export async function POST(req: NextRequest) { if (!run) return NextResponse.json({ error: "Failed to create run" }, { status: 500 }); stage = "presign"; - const command = new PutObjectCommand({ + const upload = await createPresignedPost(s3, { Bucket: requireEnv("S3_RAW_BUCKET"), Key: s3Key, - ContentType: content_type, + Conditions: [ + ["content-length-range", 1, maxBytes], + ["eq", "$Content-Type", content_type], + ], + Fields: { "Content-Type": content_type }, + Expires: UPLOAD_EXPIRES_SECONDS, }); - const presignedUrl = await getSignedUrl(s3, command, { expiresIn: 300 }); - - return NextResponse.json({ presigned_url: presignedUrl, run_id: run.id, s3_key: s3Key }); + return NextResponse.json({ upload, max_bytes: maxBytes, run_id: run.id, s3_key: s3Key }); } catch (err) { log.error("upload failed", { stage, err }); return NextResponse.json( diff --git a/src/app/pipelines/new/page.tsx b/src/app/pipelines/new/page.tsx index 0bfcedf..743d6e5 100644 --- a/src/app/pipelines/new/page.tsx +++ b/src/app/pipelines/new/page.tsx @@ -1,10 +1,11 @@ "use client"; -import { useState, useRef } from "react"; +import { useEffect, useState, useRef } from "react"; import { useRouter } from "next/navigation"; import Nav from "@/components/Nav"; import PipelineChat from "@/components/PipelineChat"; import { readJson } from "@/lib/http"; +import { formatMb } from "@/lib/upload-limits"; import { Upload, Sparkles, @@ -30,6 +31,7 @@ const FORMAT_LABELS: Record = { }; type Tab = "upload" | "chat"; +type PresignedPost = { url: string; fields: Record }; type Step = "form" | "uploading" | "processing"; export default function NewPipelinePage() { @@ -44,6 +46,16 @@ export default function NewPipelinePage() { const [step, setStep] = useState("form"); const [error, setError] = useState(null); const [progress, setProgress] = useState(0); + // Per-file limit from GET /api/upload (100 MB by default, 2 MB for guests). The + // server and S3 enforce it regardless; this only saves a doomed upload. + const [maxBytes, setMaxBytes] = useState(null); + + useEffect(() => { + fetch("/api/upload") + .then((r) => (r.ok ? r.json() : null)) + .then((d: { max_bytes?: number } | null) => { if (d?.max_bytes) setMaxBytes(d.max_bytes); }) + .catch(() => {}); + }, []); function handleFile(f: File) { const ext = f.name.split(".").pop()?.toLowerCase() ?? ""; @@ -51,6 +63,10 @@ export default function NewPipelinePage() { setError(`Unsupported file type: .${ext}`); return; } + if (maxBytes !== null && f.size > maxBytes) { + setError(`File is ${formatMb(f.size)}; the limit is ${formatMb(maxBytes)} per file.`); + return; + } setFile(f); setError(null); } @@ -83,22 +99,25 @@ export default function NewPipelinePage() { body: JSON.stringify({ pipeline_id: pipeline.id, filename: file.name, - content_type: file.type || "application/octet-stream", + size: file.size, }), }); - const { presigned_url, run_id } = await readJson<{ presigned_url: string; run_id: string }>(uploadRes, "Upload"); + const { upload, run_id } = await readJson<{ upload: PresignedPost; run_id: string }>(uploadRes, "Upload"); setProgress(40); const uploadController = new AbortController(); const uploadTimeout = setTimeout(() => uploadController.abort(), 120_000); try { - const s3Res = await fetch(presigned_url, { - method: "PUT", - body: file, - headers: { "Content-Type": file.type || "application/octet-stream" }, - signal: uploadController.signal, - }); - if (!s3Res.ok) throw new Error(`File upload to storage failed (HTTP ${s3Res.status}) — please retry`); + // Presigned POST: policy fields first, the file last (S3 ignores fields after it). + const form = new FormData(); + for (const [k, v] of Object.entries(upload.fields)) form.append(k, v); + form.append("file", file); + const s3Res = await fetch(upload.url, { method: "POST", body: form, signal: uploadController.signal }); + if (!s3Res.ok) { + const detail = await s3Res.text().catch(() => ""); + if (/EntityTooLarge/.test(detail)) throw new Error("File is larger than your upload limit"); + throw new Error(`File upload to storage failed (HTTP ${s3Res.status}) — please retry`); + } } catch (fetchErr) { if (fetchErr instanceof DOMException && fetchErr.name === "AbortError") { throw new Error("Upload timed out — check your connection and retry"); @@ -274,7 +293,7 @@ export default function NewPipelinePage() {