diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ddec8e2..9404d6f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -59,10 +59,42 @@ npx playwright install --with-deps chromium     # once
 npm run e2e:build && PYTHON=python npm run test:e2e
 ```
 
+CI runs 6 required checks: `web (node 20)`, `web (node 22)`, `lambdas (pandas 2.2.3)`,
+`lambdas (pandas 3.0.6)`, `integration (postgres + localstack)` and `e2e (playwright)`. `main`
+also requires the branch to be up to date before merging.
+
+| Suite | Covers |
+|---|---|
+| Integration (TS, `tests/integration`) | upload → S3 notification → profiler → suggest-transforms (fake model) → approve → executor → download; account deletion of every object version; concurrent approve-rules; guest sessions, quotas, AI budget, sample data and the purge cron |
+| Integration (Python, `lambdas/tests/integration`) | executor idempotency under duplicate SQS deliveries, retries, profiler on a real S3 notification |
+| E2E (Playwright, `tests/e2e`) | sign in → upload → approve → download with no `__orig_*` columns; the guest flow, its size limit, blocked features and upload cap; the auth bypass off without the flag and on Vercel; forged guest cookies |
+
+All environment for these suites is in `tests/support/test-env.mjs`. The real Lambda handlers run
+in-process (`tests/support/lambda_harness.py`), and only `get_db_conn()` is swapped for a plain
+connection to `DATABASE_URL`. In e2e, `tests/e2e/lambda_worker.py` stands in for the Lambda
+triggers. Without `CLEANSTACK_INTEGRATION=1`, `pytest lambdas/tests` skips the integration directory.
+
 On a CI e2e failure, download the `playwright-report` artifact. It has the HTML report and the
 traces; open a trace with `npx playwright show-trace test-results/<test>/trace.zip`.
 
-Rules for test-only code (see README, "Test-only switches"):
+### Test-only switches
+
+Everything test-only goes through `isTestMode()` in `src/lib/test-mode.ts`. It is true only when
+`CLEANSTACK_TEST_MODE=1` **and** none of `VERCEL`, `VERCEL_ENV`, `VERCEL_URL`,
+`AWS_LAMBDA_FUNCTION_NAME` or `AWS_EXECUTION_ENV` is set, so a leaked flag is ignored on Vercel and
+`checkEnv()` logs it as an error.
+
+| Switch | Effect in test mode | Guarded by |
+|---|---|---|
+| `DB_DRIVER=pg` + `DATABASE_URL` (`src/lib/db-pg.ts`) | node-postgres instead of the RDS Data API | `db-pg.test.ts` |
+| `cs_test_user` cookie (`src/lib/auth.ts`, `POST /api/test-auth`) | signs in `user_test_*` ids without Clerk | `auth.test.ts`, `middleware.test.ts`, e2e `test-mode-guard.spec.ts` |
+| Fake model (`src/lib/fake-model.ts`) | deterministic model instead of Bedrock | `fake-model.test.ts` |
+| LocalStack CSP (`src/lib/csp.ts`) | allows the LocalStack S3 origin | `csp.test.ts` pins the production policy |
+
+`@clerk/testing` is not used because it needs a real Clerk instance, so the real Clerk sign-in is
+not covered by e2e.
+
+Rules for test-only code:
 
 - Gate it with `isTestMode()` from `src/lib/test-mode.ts`, nothing else, and add a test that it
   is off without `CLEANSTACK_TEST_MODE=1` and when `VERCEL=1`.
@@ -95,7 +127,8 @@ Rules for test-only code (see README, "Test-only switches"):
 - **Secrets:** compare shared secrets with `safeCompare` from `src/lib/secrets.ts`.
 - **AI:** the Bedrock model id and prices live in `src/lib/ai-config.ts`. Do not hardcode them.
 - **Database:** schema changes go in a new idempotent `src/lib/migrations/NNN_<name>.sql`
-  (`IF NOT EXISTS`) **and** in `schema.sql`. Every query on tenant data must filter by `team_id`
+  (`IF NOT EXISTS`, no semicolons inside a statement) **and** in `schema.sql`, and the table in
+  `docs/deployment.md` gets a row. Every query on tenant data must filter by `team_id`
   (there is no RLS).
 - **Executor rules:** a new rule type must be added to the executor's `SUPPORTED_TABULAR_RULES`, the
   zod enum in `suggest-transforms`, and `RISK_THRESHOLDS` in `auto-validate`. `test_contracts.py`
@@ -103,3 +136,6 @@ Rules for test-only code (see README, "Test-only switches"):
 - **Profiler/executor quality code:** the `SHARED QUALITY BLOCK` is copied in both handlers and
   must stay identical (`test_quality.py`).
 - **Python dependencies** are exact-pinned in each `lambdas/*/requirements.txt`.
+- **Docs:** keep the README short. Deployment and AWS settings go in `docs/deployment.md`, the
+  runbook in `docs/operations.md`, security and guest limits in `docs/security-model.md`, and rule
+  types in `docs/transform-rules.md`.
diff --git a/README.md b/README.md
index d4c6cfe..146ad58 100644
--- a/README.md
+++ b/README.md
@@ -1,370 +1,122 @@
 # CleanStack
 
-CleanStack is a web app for cleaning tabular files and documents with an AI-reviewed, human-approved workflow. You upload a file and a Lambda profiles it. Claude (via Amazon Bedrock) then proposes transform rules with reasons, and you approve or reject each one in a "Data PR". Another Lambda applies the approved rules with pandas and returns a cleaned file with a before/after quality score.
+CleanStack cleans tabular files and documents with AI suggestions and human approval. A Lambda profiles each upload. Claude on Amazon Bedrock then proposes transform rules, each with a reason, and you approve or reject them in a "Data PR". A second Lambda applies the approved rules with pandas and returns the cleaned file with a before/after quality score.
 
-[![Demo video](public/demo-thumb.png)](https://youtu.be/rhsA_740Zgs) · Live deployment: [clean-stack-eta.vercel.app](https://clean-stack-eta.vercel.app)
+**Live:** [clean-stack-eta.vercel.app](https://clean-stack-eta.vercel.app) · **Demo video:** [youtu.be/rhsA_740Zgs](https://youtu.be/rhsA_740Zgs)
 
-- [Architecture](#architecture)
-- [Run it locally](#run-it-locally)
-- [Run the tests](#run-the-tests)
-- [Configuration](#configuration)
-- [Deployment](#deployment)
-- [Operations runbook](#operations-runbook)
-- [Guest access](#guest-access)
-- [Security model](#security-model)
-- [Known limitations](#known-limitations)
-- [Transform rules](#transform-rules)
+[![Demo video](public/demo-thumb.png)](https://youtu.be/rhsA_740Zgs)
 
----
+## Features
+
+- **Upload and profile:** CSV, TSV, TXT, JSON, JSONL, XLSX, XLS (delivered as XLSX), XML, PDF and DOCX. Each file gets per-column statistics and a quality score.
+- **AI-suggested rules, human-approved:** Claude proposes rules from a fixed set that the executor implements ([rule reference](docs/transform-rules.md)). You review them in a Data PR. Unsafe rules are skipped with a reason instead of half-applied.
+- **Auto-clean:** a committee of three Claude reviewers votes on each rule. Higher-risk rules need more votes. Runs that are over the AI budget fall back to manual review.
+- **Multi-pass:** up to 3 passes per run. Each pass is re-scored on the bytes that were actually written.
+- **Outputs:** download in the original format, **Export As** another format, or export training JSONL (raw, Alpaca or chat). Original values go to a separate `audit.csv` and are never added to the deliverable.
+- **Pipelines and templates:** save the approved rules of a pipeline as a template and reuse it. A chat builder can draft pipelines and synthetic data.
+- **Schema drift alerts:** optional Slack webhook per pipeline.
+- **Usage controls:** monthly row quotas per plan (Free: 50,000 rows; plans are assigned by an admin, with no payment integration), Upstash rate limits, and an AI spend cap checked in Postgres before every Bedrock call.
+- **Guest access:** **Try as guest** on the landing and sign-in pages starts a 24-hour session with no account. **Try with sample data** loads a seeded demo template, so it makes no Bedrock call. Guests get 2 MB files, 3 uploads, 5,000 rows per run and 10 AI calls per hour, with $0.25 of AI spend per guest. AI-heavy or irreversible features (chat builder, auto-clean, training export, Slack alerts, templates, account deletion) are blocked for guests. A guest's data is erased after 24 hours. [Full limits](docs/security-model.md#guest-access)
 
 ## Architecture
 
 ```
-Browser ── Next.js 16 app on Vercel (Clerk auth, API routes) ── Aurora PostgreSQL (RDS Data API)
-   │                │
-   │ presigned POST │ presigned GET (120 s) for downloads
-   │ (size-capped)  │
-   ▼                ▼
-S3 raw bucket    S3 processed bucket  (output.<ext> + audit.csv per run)
-   │ S3 event
+Browser ── Next.js 16 on Vercel (Clerk auth, API routes) ── Aurora PostgreSQL (RDS Data API)
+   │ presigned POST (size-capped)            presigned GET (downloads) ◄── S3 processed bucket
    ▼
-Lambda: profiler ── stats + quality score → data_profiles ── POST /api/webhooks/profile-complete
-                                                                 │
-               AI_QUEUE_ENABLED=true: SQS ai-jobs → Lambda ai-trigger ┐
-               otherwise: inline call ───────────────────────────────┤
-                                                                     ▼
-                                       /api/suggest-transforms (Bedrock) → transform_rules
-                                                                     │
-                     manual: Data PR review → /api/approve-rules     │ auto: /api/auto-validate
-                                                                     ▼   (3-persona committee)
-                                                         SQS executor queue
-                                                                     │
-Lambda: executor ── claims run, applies rules, writes deliverable + audit file,
-                    re-scores output, deletes raw run objects, optional next pass
-                    └─ schema changed → SNS → Lambda: drift → Slack webhook
+S3 raw bucket ── S3 event ──► Lambda profiler ──► webhook ──► suggest-transforms (Bedrock Claude)
+                                             (inline, or SQS AI jobs queue ► Lambda ai-trigger)
+Data PR approval / auto-clean ──► SQS executor queue ──► Lambda executor ──► S3 processed bucket
+                                                         └─ schema changed ► SNS ► Lambda drift ► Slack
 ```
 
-| Component | Where it lives | Notes |
+| Part | Code | Runs on |
 |---|---|---|
-| Web app + API | `src/` (Next.js App Router) | Deployed on Vercel |
-| Profiler Lambda | `lambdas/profiler/handler.py` | S3 `ObjectCreated` trigger (must include `Post`; uploads are presigned POSTs) |
-| AI trigger Lambda | `lambdas/ai-trigger/handler.py` | SQS trigger, calls `/api/suggest-transforms` |
-| Executor Lambda | `lambdas/executor/handler.py` | SQS trigger |
-| Drift Lambda | `lambdas/drift/handler.py` | SNS trigger |
-| Schema | `src/lib/schema.sql`, `src/lib/migrations/NNN_*.sql` | Applied by `run-migration.mjs` |
-| AWS resources (buckets, queues, IAM, KMS, Lambda config, cron schedule) | **Not in this repo** | Configured by hand in the AWS console; see [Known limitations](#known-limitations) |
-
-### Run lifecycle
-
-`pending` (presigned URL issued) → `profiling` → `awaiting_ai` → `awaiting_approval` → `queued` → `running` → `completed` / `failed`.
+| Web app and API | `src/` (App Router) | Vercel |
+| Profiler | `lambdas/profiler/handler.py` | Lambda, S3 trigger on the raw bucket |
+| AI trigger | `lambdas/ai-trigger/handler.py` | Lambda, SQS AI jobs queue; calls `/api/suggest-transforms` |
+| Executor | `lambdas/executor/handler.py` | Lambda, SQS executor queue |
+| Drift | `lambdas/drift/handler.py` | Lambda, SNS |
+| Schema | `src/lib/schema.sql`, `src/lib/migrations/00N_*.sql` | Aurora PostgreSQL |
 
-- **Execution is idempotent per run.** The executor moves a run from `queued` to `running` with a conditional `UPDATE`. A duplicate SQS delivery is acknowledged and skipped. A `running` lease older than 15 minutes, which is Lambda's hard limit, can be reclaimed.
-- **Rules are all-or-nothing.** The executor snapshots the frame before each rule. If a rule fails or is unsafe (unknown type, missing column, more than 20% row loss, bad regex, cast that would lose data), the frame is restored and the rule is recorded as *not applied* with a reason in `transform_rules.parameters._execution`. The run page shows those rules as "Not applied".
-- **Deliverables never contain audit columns.** Rules that rewrite values keep the original in an `__orig_<column>` sidecar. Sidecars are written only to `processed/{pipeline}/{run}/audit.csv`, next to `output.<ext>`.
-- **The quality score is computed the same way before and after.** The profiler and executor share one loader and scorer (the `SHARED QUALITY BLOCK`, enforced by a test), and the executor scores the bytes it actually wrote.
+Other services: Clerk (auth), Upstash Redis (rate limits and quota cache, which fail open when not configured) and Sentry (optional). AWS resources are configured by hand, and there is no infrastructure as code in this repo.
 
----
+A run moves through `pending → profiling → awaiting_ai → awaiting_approval → queued → running → completed | failed`. Execution is idempotent per run: the executor claims a run with a conditional `UPDATE`, so a duplicate SQS delivery is skipped. A stale run is marked failed after 20 minutes by `/api/cron/reconcile-runs`.
 
-## Run it locally
+## Local development
 
-Prerequisites: Node.js 20 or 22, npm, Python 3.12 (for the Lambda tests), and an AWS account with the resources in [Configuration](#configuration). The UI cannot do anything useful without Aurora, S3, SQS, Bedrock and Clerk. There is no local mock stack.
+Requirements: Node.js 20 or 22, Python 3.12 and Docker (for the test services). The app has no local mock stack. Running it against real data needs Aurora, S3, SQS, Bedrock and Clerk.
 
 ```bash
-git clone https://github.com/DHYEY166/CleanStack.git
-cd CleanStack
 npm ci
 cp .env.example .env.local     # fill in real values
 npm run dev                    # http://localhost:3000
+AURORA_HOST=<cluster endpoint> node src/lib/migrations/run-migration.mjs   # schema + migrations
 ```
 
-When the server starts it logs one `invalid or missing environment` line listing any required variable that is missing or malformed (see `src/lib/env.ts`).
-
-Database (first time, and after pulling new migrations):
-
-```bash
-AURORA_HOST=<cluster endpoint> RDS_CA_BUNDLE=/path/to/global-bundle.pem \
-  node src/lib/migrations/run-migration.mjs
-```
-
-This applies `schema.sql` and then every `src/lib/migrations/NNN_*.sql` in order. Every statement is idempotent, so re-running is safe. It authenticates with an RDS IAM token for `AURORA_USER` (default `postgres`). For a plain Postgres (local or the test containers), set `DATABASE_URL` instead; it takes precedence and IAM auth is not used. Optional seed data is in `src/lib/seed-templates.sql`.
+`run-migration.mjs` applies `schema.sql` and then each migration in order. Every statement is idempotent. It uses RDS IAM auth, or `DATABASE_URL` for a plain Postgres.
 
-## Run the tests
-
-The same commands as CI (`ci/verify.sh` runs all of them):
+## Tests
 
 ```bash
-npm ci
-npx eslint
-npx tsc --noEmit
-npx vitest run          # TypeScript unit tests (src/**/*.test.ts)
-npm run build
+ci/verify.sh        # eslint, tsc, vitest unit tests, next build, Lambda pytest
 
-python3.12 -m venv .venv && . .venv/bin/activate
-pip install -r lambdas/requirements-dev.txt
-python -m pytest -q lambdas/tests
-```
-
-The Lambda tests import the real handlers. AWS clients are created but never called, and DB and S3 are faked inside the tests. CI runs them against both pandas 2.2.3 (pinned) and pandas 3.0.6.
-
-The workflow definition is `.github/workflows/ci.yml`. It runs on every pull request and on pushes to `main`.
-
-### Integration and end-to-end suites
-
-These run against real services in containers: Postgres 16 and LocalStack 3.8 (S3 + SQS). CI starts them as service containers in the `integration` and `e2e` jobs. Nothing calls real AWS or Bedrock.
-
-```bash
-# 1. Services (Docker). Or point DATABASE_URL / LOCALSTACK_URL at your own.
-docker compose -f tests/support/docker-compose.yml up -d --wait
-npm run services:setup        # schema + migrations, versioned buckets + CORS, queues, S3 -> SQS notification
-
-# 2. Integration: route handlers in-process + Lambda handlers in-process (Python)
-pip install -r lambdas/requirements-dev.txt     # the TS suite shells out to Python
-PYTHON=python npm run test:integration           # tests/integration (vitest.integration.config.mts)
+docker compose -f tests/support/docker-compose.yml up -d --wait   # Postgres 16 + LocalStack (S3, SQS)
+npm run services:setup
+PYTHON=python npm run test:integration                            # route + Lambda handlers in-process
 CLEANSTACK_INTEGRATION=1 python -m pytest -q lambdas/tests/integration
-
-# 3. Browser e2e: production build in test mode + Playwright
-npx playwright install --with-deps chromium
-npm run e2e:build                                # next build with the test env
-PYTHON=python npm run test:e2e                   # tests/e2e (playwright.config.ts)
+npm run e2e:build && PYTHON=python npm run test:e2e               # Playwright against a test-mode build
 ```
 
-| Suite | Command | What it covers |
-|---|---|---|
-| Integration (TS) | `npm run test:integration` | upload presigned POST → S3 notification → profiler → suggest-transforms (fake model) → approve → executor SQS message → executor → presigned download fetch; account deletion purging every object version and delete marker; approve-rules claim under 8 concurrent requests; pg driver result shapes; guest sessions (signed cookie, `guest_sessions` row, per-IP cap) and a guest's 2 MB upload refused by the profiler; guest quotas and AI budget (`guest-limits.test.ts`); sample data on the seeded template through the real profiler and executor to a pinned clean file, then `/api/cron/purge-guests` erasing the expired guest (`guest-demo.test.ts`) |
-| Integration (Python) | `CLEANSTACK_INTEGRATION=1 pytest lambdas/tests/integration` | executor idempotency with duplicate SQS deliveries (separate and same batch), retry and last-attempt semantics, profiler on a real S3 notification |
-| E2E | `npm run test:e2e` | sign in → upload a CSV → suggested rules → approve → completion → download with no `__orig_*` columns; guest session → upload → review, a guest's 2.1 MB file refused in the form and with 413, blocked features and the upload cap, and the full guest flow (Try as guest → sample data → approve → completed → download → End session); the auth bypass is off without the flag and on Vercel, and a forged guest cookie is not a session |
-
-All environment for these suites lives in `tests/support/test-env.mjs`. Variables already set in your shell win. `pytest lambdas/tests` without `CLEANSTACK_INTEGRATION=1` skips the integration directory, so the unit suite never needs containers. With the variable set, missing services are an error, not a skip.
-
-**Lambdas in tests.** The real handlers run in-process (`tests/support/lambda_harness.py`). The only substitution is `get_db_conn()`, which in production builds an RDS IAM token and connects over TLS; in tests it is a plain connection to `DATABASE_URL`. The harness refuses to run unless `AWS_ENDPOINT_URL` points at LocalStack. In e2e, `tests/e2e/lambda_worker.py` stands in for the Lambda triggers: it feeds the S3 notification queue to the profiler and the executor queue to the executor, deleting a message only on success, like Lambda.
-
-### Test-only switches
-
-Everything test-only goes through `isTestMode()` in `src/lib/test-mode.ts`. It is true only when `CLEANSTACK_TEST_MODE=1` **and** none of `VERCEL`, `VERCEL_ENV`, `VERCEL_URL`, `AWS_LAMBDA_FUNCTION_NAME`, `AWS_EXECUTION_ENV` is set. Vercel sets `VERCEL=1` in every build and function, so a flag that leaks into a deployment is ignored, and `checkEnv()` logs it as an error at startup. `NODE_ENV=test` is not used because `next build` and `next start` always run with `NODE_ENV=production`.
-
-| Switch | Where | Effect in test mode | Guard / test |
-|---|---|---|---|
-| `DB_DRIVER=pg` + `DATABASE_URL` | `src/lib/db.ts`, `src/lib/db-pg.ts` | node-postgres instead of the RDS Data API, with Data API-shaped results | Throws `ConfigError` before connecting unless test mode (`db-pg.test.ts`) |
-| Cookie auth (`cs_test_user`) | `src/lib/auth.ts`, `src/middleware.ts`, `POST /api/test-auth`, sign-in page, root layout | Signs in `user_test_*` ids without Clerk (Clerk needs real keys) | Route 404s and cookie ignored outside test mode (`auth.test.ts`, `middleware.test.ts`, `test-auth/route.test.ts`); e2e `test-mode-guard.spec.ts` checks a build started without the flag and with flag + `VERCEL=1` |
-| Fake model | `src/lib/ai-model.ts`, `src/lib/fake-model.ts` | Deterministic `MockLanguageModelV3` instead of Bedrock | `fake-model.test.ts` asserts Bedrock without the flag and on Vercel |
-| LocalStack CSP | `src/lib/csp.ts` (build time) | `connect-src` allows `*.s3.localhost.localstack.cloud:4566`; drops `upgrade-insecure-requests` | `csp.test.ts` pins the production policy byte-for-byte without the flag and on Vercel |
-
-`@clerk/testing` was not used because its testing tokens still need a real Clerk development instance (publishable and secret keys), which CI does not have. The real Clerk sign-in is therefore not covered by e2e.
-
----
+No test calls real AWS, Bedrock or Clerk. CI (`.github/workflows/ci.yml`) runs 6 checks on every PR and on every push to `main`: `web (node 20)`, `web (node 22)`, `lambdas (pandas 2.2.3)`, `lambdas (pandas 3.0.6)`, `integration (postgres + localstack)` and `e2e (playwright)`. `main` requires all six, and a branch must be up to date with `main` before it can merge. See [CONTRIBUTING.md](CONTRIBUTING.md) for what each suite covers and for the test-mode switches.
 
 ## Configuration
 
-### Web app (Vercel / `.env.local`)
-
-The full contract is in `src/lib/env.ts`, and `.env.example` has a placeholder for every variable. A test fails if one is missing from `.env.example`.
+`src/lib/env.ts` is the full contract, and `.env.example` lists every variable (a test enforces this). At startup the server logs any variable that is missing or invalid.
 
-| Variable | Required | Purpose |
-|---|---|---|
-| `AWS_REGION` | no (default `us-east-1`) | Region for S3, SQS, RDS Data API, Bedrock |
-| `AURORA_CLUSTER_ARN` | yes | Aurora cluster ARN (RDS Data API) |
-| `AURORA_SECRET_ARN` | yes | Secrets Manager ARN of the DB credentials |
-| `S3_RAW_BUCKET` | yes | Raw uploads |
-| `S3_PROCESSED_BUCKET` | yes | Deliverables and audit files |
-| `SQS_QUEUE_URL` | yes | Executor queue. If unset, approved runs are marked completed **without executing** |
-| `AI_QUEUE_ENABLED` | no | `true` = enqueue AI jobs to `AI_JOBS_QUEUE_URL`; otherwise `profile-complete` calls `suggest-transforms` inline |
-| `AI_JOBS_QUEUE_URL` | when `AI_QUEUE_ENABLED=true` | AI jobs queue |
-| `NEXT_PUBLIC_APP_URL` | yes | Base URL for internal webhook calls |
-| `WEBHOOK_SECRET` | yes | Shared with the profiler and ai-trigger Lambdas (`x-webhook-secret`) |
-| `CRON_SECRET` | yes | `Authorization: Bearer` for `/api/cron/reconcile-runs` |
-| `ADMIN_SECRET` | no | `x-admin-secret` for `/api/admin/*`; those routes return 401 when unset |
-| `ADMIN_EMAILS`, `ADMIN_USER_IDS` | no | Users who bypass billing quotas |
-| `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY`, `CLERK_SECRET_KEY` | yes | Clerk |
-| `UPSTASH_REDIS_REST_URL`, `UPSTASH_REDIS_REST_TOKEN` | no | Rate limiting and quota cache. **Both are off when unset (fail open)** |
-| `SENTRY_DSN`, `NEXT_PUBLIC_SENTRY_DSN` | no | Error reporting |
-| `LOG_LEVEL` | no (default `info`) | Threshold of the JSON logger |
-| `MAX_UPLOAD_MB` | no (default `100`) | Per-file upload limit for signed-in users. Guests are always 2 MB. Set the same value on the profiler Lambda |
-| `GUEST_COOKIE_SECRET` | no | HMAC key (≥ 32 chars) for the signed `cs_guest` cookie and for hashing guest IPs. **Guest access is off while it is unset.** Generate with `openssl rand -base64 48` |
-| `TURNSTILE_SECRET_KEY`, `NEXT_PUBLIC_TURNSTILE_SITE_KEY` | no | Cloudflare Turnstile check on `POST /api/guest`. Off when unset; set both or neither |
-
-Generate shared secrets with `openssl rand -hex 32`. Values shorter than 32 characters produce a startup warning.
-
-The Bedrock model id and the token prices used for metering are in `src/lib/ai-config.ts`. Today that is `us.anthropic.claude-sonnet-4-6` at $3 / $15 per 1M input/output tokens. Update both in the same commit when changing models.
-
-### Lambdas (set on each function)
-
-| Function | Variables read by the code |
+| Variable | Notes |
 |---|---|
-| profiler | `DATABASE_URL`, `APP_URL`, `WEBHOOK_SECRET`, `MAX_UPLOAD_MB` (optional, default 100; keep equal to the web app's), `SENTRY_DSN` (optional), `AWS_REGION` |
-| ai-trigger | `APP_URL`, `WEBHOOK_SECRET` |
-| executor | `DB_SECRET_ARN`, `S3_RAW_BUCKET`, `S3_PROCESSED_BUCKET`, `SNS_DRIFT_TOPIC_ARN` (optional), `EXECUTOR_MAX_ATTEMPTS` (optional, default 3), `SEMANTIC_DEDUP_MAX_ROWS` (optional, default 500000), `EXECUTOR_RESERVE_S` (optional, default 120: seconds of the invocation `semantic_deduplicate` leaves for writing the output, at most 25% of it), `SENTRY_DSN` (optional), `AWS_REGION` |
-| drift | `DB_SECRET_ARN`, `AWS_REGION` |
-
-Python dependencies are pinned in `lambdas/*/requirements.txt`. At runtime pandas/numpy come from the AWS SDK for pandas (`AWSSDKPandas-Python312`) layer. The code is tested on pandas 2.2.3 and 3.0.6 because the deployed layer version is not recorded here.
-
----
+| `AURORA_CLUSTER_ARN`, `AURORA_SECRET_ARN` | Required. RDS Data API |
+| `S3_RAW_BUCKET`, `S3_PROCESSED_BUCKET` | Required |
+| `SQS_QUEUE_URL` | Required. Executor queue |
+| `AI_QUEUE_ENABLED`, `AI_JOBS_QUEUE_URL` | Optional. Queue AI jobs instead of calling suggest-transforms inline |
+| `NEXT_PUBLIC_APP_URL`, `WEBHOOK_SECRET`, `CRON_SECRET` | Required. Secrets need at least 32 chars (`openssl rand -hex 32`) |
+| `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY`, `CLERK_SECRET_KEY` | Required |
+| `GUEST_COOKIE_SECRET` | Optional. Guest access is off while it is unset |
+| `TURNSTILE_SECRET_KEY`, `NEXT_PUBLIC_TURNSTILE_SITE_KEY` | Optional. Cloudflare Turnstile on guest sign-up |
+| `MAX_UPLOAD_MB` | Optional. Default 100 for signed-in users (guests are always 2 MB). Keep it equal on the profiler Lambda |
+| `UPSTASH_REDIS_REST_URL`, `UPSTASH_REDIS_REST_TOKEN`, `SENTRY_DSN`, `NEXT_PUBLIC_SENTRY_DSN`, `ADMIN_SECRET`, `ADMIN_EMAILS`, `ADMIN_USER_IDS`, `AWS_REGION`, `LOG_LEVEL` | Optional |
+
+The Bedrock model id and token prices live in `src/lib/ai-config.ts`. Lambda variables are listed in [docs/deployment.md](docs/deployment.md#lambda-environment).
 
 ## Deployment
 
-**Web app.** Vercel builds `main` (`npm run build`). The build needs no secrets because configuration is validated per request. Set the variables above in the Vercel project.
+Vercel builds and deploys `main`. The Lambdas, migrations and AWS resources are deployed by hand. [docs/deployment.md](docs/deployment.md) covers:
 
-**Migrations.** Run `run-migration.mjs` (see above) **before** deploying code that needs new tables. Migration `002_ai_usage_tables.sql` creates `bedrock_usage` and `ai_spend_limits`. Without them, AI metering and the spend cap do not work. Migration `003_guest_sessions.sql` creates `guest_sessions`; `POST /api/guest` returns 500 without it.
+- applying migrations 001–003 (`run-migration.mjs`, or the RDS Data API from CloudShell);
+- the CloudShell zip-swap deploy for `cleanstack-profiler`, `cleanstack-executor`, `cleanstack-ai-trigger` and `cleanstack-drift`;
+- raw bucket CORS (PUT, GET, POST), the profiler trigger on all object create events, and processed bucket CORS;
+- EventBridge rules `cleanstack-reconciler-5min` (every 5 minutes) and `cleanstack-purge-guests-hourly` (every hour), which call the cron routes through API destinations as the `cleanstack-eventbridge-invoker` role;
+- the S3 lifecycle rule on the `guest_` prefix;
+- least-privilege IAM for the `cleanstack-vercel` user and for the Lambda roles.
 
-**Lambdas.** Each Lambda is deployed as its handler plus its `requirements.txt`. The profiler and executor also attach the AWSSDKPandas layer. There is no build or deploy script in the repo yet, so build the zip from the pinned requirements (excluding pandas/numpy, which the layer provides) and update the function:
-
-```bash
-aws lambda update-function-code --function-name cleanstack-executor \
-  --zip-file fileb://executor.zip --region us-east-1
-```
-
-**AWS settings the code relies on** (configure them in AWS; they cannot be checked from this repo):
-
-- **Executor SQS queue:** visibility timeout ≥ the executor Lambda timeout (AWS recommends 6×), and a dead-letter queue with `maxReceiveCount` ≥ `EXECUTOR_MAX_ATTEMPTS`. The handler does not return partial batch failures, so one failing record retries the whole batch. That is safe because execution is idempotent, but `BatchSize: 1` keeps retries simple.
-- **Raw bucket CORS:** allow `POST` (and `PUT` for older clients) from the app origin. Uploads are presigned POSTs (multipart form) so that the policy's `content-length-range` makes S3 reject an oversized body.
-- **Profiler S3 trigger:** the raw bucket's event notification must include `s3:ObjectCreated:Post` (or `s3:ObjectCreated:*`). A trigger on `s3:ObjectCreated:Put` alone never fires for browser uploads, and runs sit in `pending`.
-- **Processed bucket CORS:** allow `GET` from the app origin. The browser fetches the presigned URL for **Export As**; the plain download is a navigation and needs no CORS.
-- **IAM for the web app role:** `s3:PutObject` (raw), `s3:GetObject` (processed), and for account erasure `s3:ListBucketVersions`, `s3:ListBucket`, `s3:DeleteObject` and `s3:DeleteObjectVersion` on both buckets, plus `sqs:SendMessage` on both queues, `rds-data:ExecuteStatement`/`BeginTransaction`/`CommitTransaction`/`RollbackTransaction` on the cluster, `secretsmanager:GetSecretValue` on the DB secret, and `bedrock:InvokeModel` for the configured model.
-- **IAM for the executor:** the same S3 version permissions on the raw bucket. Without `s3:ListBucketVersions` it falls back to deleting only the two known raw keys.
-- **Guest purge schedule:** call `GET /api/cron/purge-guests` with `Authorization: Bearer $CRON_SECRET` every hour (an EventBridge Scheduler target, like the reconciler). Each call erases up to 20 expired guests. Vercel Hobby crons run at most daily, so `vercel.json` defines none.
-- **Raw bucket lifecycle rule (backstop):** expire objects under prefix `guest_` after 1 day, including noncurrent versions, in case the purge stops running.
-- **Reconciler schedule:** something must call `GET /api/cron/reconcile-runs` with `Authorization: Bearer $CRON_SECRET` every few minutes. `vercel.json` defines no cron. The project's earlier documentation describes an EventBridge rule running every 5 minutes.
-
----
-
-## Operations runbook
-
-**Logs.** Server routes write one JSON object per line (`src/lib/logger.ts`): `{"ts","level","msg","service","route",...}`. Filter by `run_id`, `route` or `level`. Credential-like keys are redacted. Lambdas log plain text to CloudWatch.
-
-| Symptom | What to check / do |
-|---|---|
-| Run stuck in `profiling`/`awaiting_ai`/`queued`/`running` | `reconcile-runs` marks these `failed` after 20 minutes, so confirm the scheduler is calling it. For `queued`: if the log has `SQS send failed; run left queued without a message`, re-send the message: `aws sqs send-message --queue-url "$SQS_QUEUE_URL" --message-body '{"run_id":"<id>"}'`. This is safe because execution is idempotent. |
-| Run `failed` with `Attempt n/3 failed, retrying` history | Transient errors are retried by SQS up to `EXECUTOR_MAX_ATTEMPTS`; the last attempt marks the run failed. Check executor CloudWatch logs for the run id. |
-| Approved rule shows **Not applied** | Expected when a rule is unsafe or invalid (the reason is shown). The rest of the run is unaffected. |
-| User reports the file is missing `__orig_*` original values | By design. They are in `processed/{pipeline}/{run}/audit.csv`. |
-| **Export As** fails but download works | CORS on the processed bucket (see Deployment). |
-| `DELETE /api/account` returns 500 "No account data was removed" | S3 purge failed. Search logs for `S3 purge incomplete` / `S3 purge failed`, fix the IAM permissions, and have the user retry. The database is left intact so the retry is complete. `all_versions_purged: false` in a 200 response means `s3:ListBucketVersions` is missing. |
-| Startup log `invalid or missing environment` | Set the listed variables in Vercel and redeploy. |
-| Rotating `WEBHOOK_SECRET` | Update the Vercel env **and** the profiler and ai-trigger Lambdas together. Requests fail with 401 while they differ. |
-| AI spend | `GET /api/admin/ai-spend` with `x-admin-secret` lists the current month's estimated spend per team. |
-| Guests see "Guest AI capacity is used up for today" | All guests together reached $5 of estimated Bedrock spend since 00:00 UTC. It resets at midnight UTC; the number is `GUEST_LIMITS.aiSpendAllGuestsPerDayUsd`. |
-
----
-
-## Guest access
-
-Off until `GUEST_COOKIE_SECRET` is set. When on, the landing and sign-in pages show **Try as guest**, which calls `POST /api/guest` (with a Turnstile widget when `NEXT_PUBLIC_TURNSTILE_SITE_KEY` is set) and lands on the dashboard. A guest:
-
-- has an id `guest_<22 chars>` that is their `team_id`, so all tenant isolation applies unchanged;
-- sees a banner in the nav with the time left, **Sign up to keep your work** and **End session** (`DELETE /api/guest`);
-- can upload (2 MB, 3 files) or press **Try with sample data**: `POST /api/sample-data` writes `src/lib/sample-data.ts`'s CSV to the raw bucket and attaches the demo template seeded by migration 003, so the rules come from the template and no Bedrock call is made;
-- gets the limits and blocked features listed under [Security model](#security-model).
-
-After 24 h the cookie stops working and `/api/cron/purge-guests` erases the guest's S3 objects (every version) and rows with the same helper as account deletion (`src/lib/erase-team.ts`). `bedrock_usage` rows are kept so the all-guests daily AI cap and cost reporting stay correct. Signing up does not carry guest work over to the new account.
-
----
-
-## Security model
-
-What the code does today. Settings that live only in AWS are listed as such.
-
-- **Authentication.** Clerk. `src/middleware.ts` protects the app pages and user API routes, and each route also calls `auth()` and returns 401 without a user.
-- **Tenant isolation is enforced in application code**, not by the database. Every user query filters on `pipelines.team_id = <Clerk user id>`. `queryWithTeam()` sets `app.team_id` for the transaction, but **no Row-Level Security policies are defined** in `schema.sql`, so that setting currently has no effect. A missing `team_id` predicate in a new query would leak data across tenants.
-- **Service-to-service calls** (Lambdas → webhooks, cron, admin) use shared secrets compared with `safeCompare` (`src/lib/secrets.ts`: SHA-256 then `timingSafeEqual`). It fails closed when the expected secret is not configured.
-- **File access.** Uploads go straight to S3 with a 300 s presigned POST whose policy pins the key and content type and caps the size (`content-length-range`, `src/lib/upload-limits.ts`: 100 MB or `MAX_UPLOAD_MB`, guests 2 MB). `/api/upload` refuses a larger declared size with 413 and the profiler marks a run failed, without reading the object, when the stored object is over the limit; downloads use a 120 s presigned GET for the caller's own completed run. Files never pass through the serverless function.
-- **Data lifecycle.**
-  - After a successful execution, the executor deletes every object version under the run's raw prefix (`{user}/{pipeline}/{run}/`), including `extracted_text.txt` (`auto_delete_raw`, on by default).
-  - Raw files of **failed** runs are not deleted, and there are no S3 lifecycle rules in the repo (the `guest_` rule in Deployment is set by hand).
-  - Guests are erased 24 h after their session starts by `/api/cron/purge-guests` (S3 every version, then rows; `bedrock_usage` kept).
-  - `DELETE /api/account?confirm=true` purges every version and delete marker under the user's raw prefix and their pipelines' processed prefixes, then deletes the DB rows. If S3 fails it aborts before touching the DB.
-- **AI.** Uploaded content is sent to Amazon Bedrock and wrapped in `<user_data>` tags with an instruction to treat it as data. This reduces prompt-injection risk but does not prevent it, which is why rules need approval (or the committee in auto mode). `checkAiBudget` (`src/lib/bedrock-meter.ts`) runs in Postgres before every Bedrock call (suggest-transforms on every pass, the auto-validate committee, chat-builder, generate-data), and every call is metered in `bedrock_usage`.
-- **Guest access** (`src/lib/guest.ts`, off unless `GUEST_COOKIE_SECRET` is set). `POST /api/guest` issues an httpOnly, `SameSite=Lax`, `Secure` cookie `cs_guest` = `guest_<22 chars>.<expiry>.<HMAC-SHA256>`, valid for 24 h; `auth()` returns that `guest_…` id as the user id (it is the `team_id`), and only when no Clerk user is signed in. Sessions are recorded in `guest_sessions` with an HMAC of the client IP (never the raw IP); at most 5 per IP and 200 overall per 24 h. Turnstile is checked when `TURNSTILE_SECRET_KEY` is set. Guest quotas and blocked features are listed in `src/lib/guest-limits.ts`.
-- **Guest limits** (all counted in Postgres, so they hold when Upstash is down):
-
-  | Limit | Value | Where |
-  |---|---|---|
-  | File size | 2 MB | presigned POST policy, `/api/upload` 413, profiler |
-  | Rows per run | 5,000 | `suggest-transforms`, before templates or AI; the run fails |
-  | Rows per guest | 10,000 (all passes; runs refused for size do not count) | `checkQuota` "guest" plan |
-  | Uploads | 3 per guest, 10 per IP hash per 24 h | the capped `INSERT` in `src/lib/guest-quota.ts` (also refuses expired sessions) |
-  | Pipelines | 5 per guest | `POST /api/pipelines` |
-  | AI calls | 10 per rolling hour | `checkAiBudget` |
-  | AI spend | $0.25 per guest; $5 per UTC day for all guests together | `checkAiBudget` |
-  | Sessions | 5 per IP hash and 200 overall per 24 h | `POST /api/guest` |
-
-  Guests cannot use the chat builder or synthetic data, auto-clean, training export, Slack alerts, account deletion, templates or `/api/admin`: the middleware answers 403 (or redirects `/templates` to the dashboard) and each route also refuses guest ids (`forbidGuest`). When auto-validate is over budget, the run falls back to manual review.
-- **Abuse limits.** Upstash sliding-window limits per user: 20 uploads/h, 50 AI calls/h, 30 chat messages/h. They fail open when Redis is not configured.
-- **Headers.** A CSP is set in `next.config.ts`. It allows `'unsafe-inline'` and `'unsafe-eval'` for scripts (needed by Next.js and Clerk without nonces).
-- **Secrets.** Nothing secret is committed. `.env.example` has placeholders only, and `run-migration.mjs` reads the database host from the environment. AWS account ids and ARNs from older commits remain in git history.
-
-Report vulnerabilities as described in [SECURITY.md](SECURITY.md).
-
----
+Operations runbook: [docs/operations.md](docs/operations.md). Security model: [docs/security-model.md](docs/security-model.md). To report a vulnerability, see [SECURITY.md](SECURITY.md).
 
 ## Known limitations
 
-- **No RLS.** Isolation depends on every query including the `team_id` predicate (see above).
-- **No infrastructure as code.** Buckets, queues, IAM, KMS, Lambda timeouts, DLQs, CORS and the cron schedule are configured by hand and cannot be reviewed or reproduced from the repo.
-- **No automatic re-enqueue.** If the SQS send after approval fails, the run stays `queued` until the reconciler marks it failed (see the runbook).
-- **No retention enforcement.** Raw files of failed runs stay in S3; `data_retention_days` is not enforced.
-- **Downloads are served as stored.** Deliverables produced before the sidecar split may still contain `__orig_*` columns. **Export As** and the training export strip them; the native download does not. Re-run those pipelines to regenerate them.
-- **Excel blanks are not counted as nulls** by the profiler (cells are read as empty strings). Before/after scores are still computed the same way.
-- **AI budget is checked before a call and recorded after it.** Concurrent requests can each pass the check and overshoot a cap by the cost of the calls in flight (the committee's 3 calls are reserved together). Costs are estimates from `src/lib/ai-config.ts`, not the AWS bill.
-- **Upload size limit is per file only.** 100 MB (`MAX_UPLOAD_MB`) for users, 2 MB for guests. A file under the limit can still exhaust the executor Lambda's memory or time for expensive rules. `semantic_deduplicate` (MinHash + LSH, a few seconds for 50k short texts) is skipped with a reason instead of running past the Lambda deadline, and above `SEMANTIC_DEDUP_MAX_ROWS` rows; other rules have no such guard.
-- **Guest abuse limits are per IP hash and per cookie.** Clearing cookies starts a new guest; the 5 sessions per IP and 200 per day caps (and Turnstile, if enabled) bound that. Users behind one NAT or VPN share the per-IP caps. A guest's upload counts come from the IP at session start.
-- **Guest work is not migrated** to an account on sign-up.
-- **Schema drift alerts** store their diff inside `column_definitions`, so the next alert reports a phantom `_diff` column.
-- **Templates** copy approved rules from all recent runs of a pipeline, not only the latest run.
-- **`middleware.ts`** uses the file convention that Next.js 16 deprecated in favour of `proxy.ts`. It still works; the rename is pending a test against real Clerk keys.
-- **Test coverage gaps.** The integration and e2e suites do not exercise the Lambdas' RDS IAM-token connection (`get_db_conn`), the RDS Data API itself (the web app uses the test-only pg driver), or the real Clerk sign-in.
-- **Parquet** is neither accepted for upload nor produced as output.
-- **Dev-only advisories.** `npm audit` reports 4 moderate advisories in drizzle-kit's bundled esbuild (dev dependency only).
-
----
-
-## Transform rules
-
-The model can only suggest rules the executor implements (a contract test compares the two lists). Risk tiers decide how many committee votes a rule needs in auto mode: LOW 1/3, MEDIUM 2/3, HIGH 3/3.
-
-### Tabular
-
-| Rule | What it does |
-|---|---|
-| `trim_whitespace` | Strip leading/trailing whitespace in text columns |
-| `deduplicate` | Drop exact duplicate rows (row-loss guard 20%) |
-| `semantic_deduplicate` | Drop near-duplicate rows: MinHash Jaccard estimate ≥ `threshold` (default 0.8) over lowercased word sets, keeping the first; LSH banding, deterministic hashing |
-| `fill_nulls` | Fill nulls (mean / median / mode / constant); original kept in the audit file |
-| `drop_nulls` | With a column: drop rows where that column is null. Without: drop rows with at least `ceil(threshold × columns)` nulls |
-| `type_cast` | Cast to float / int / datetime / str. Not applied if values would be lost (e.g. `2.5` → int) |
-| `normalize` | Text/date columns only: dates → `YYYY-MM-DD`, other text trimmed and lower-cased (or mapped via `value_map`); nulls stay null. Numeric columns are left unchanged |
-| `filter`, `filter_extended` | Remove rows by condition (row-loss guard 20%) |
-| `rename`, `column_header_normalize` | Rename columns / snake_case headers |
-| `ner_redact` | Regex-based redaction (name, organisation, address/ZIP, date patterns). Heuristic, not a trained NER model |
-| `ffill`, `bfill` | Fill nulls from the previous/next row (signal-gated) |
-| `bool_cast` | Normalise yes/no/true/false/1/0 variants |
-| `outlier_cap` | Cap values at the IQR fences |
-| `multi_currency_strip` | Strip currency symbols and convert to numbers |
-| `split_column` | Split a column on a delimiter into new columns |
-
-### Documents (PDF, DOCX, TXT)
-
-`strip_pii`, `ner_redact`, `fix_encoding`, `remove_headers_footers`, `remove_blank_lines`, `normalize_whitespace`, `strip_html`, `redact_pattern` (pattern length capped at 200 characters; no protection against catastrophic backtracking).
-
-Accepted uploads: CSV, TSV, TXT, JSON, JSONL, XLSX, XLS (delivered as XLSX), XML, PDF, DOCX.
-
----
-
-## Project layout
-
-```
-src/app/            pages and API routes (api/*/route.ts)
-src/components/     React components
-src/lib/            db (Data API), env, logger, secrets, ai-config, s3-erase, erase-team,
-                    guest (cookie), guest-limits/-quota/-guard, upload-limits, sample-data,
-                    download/training-export helpers, schema.sql, migrations/
-lambdas/            profiler, executor, ai-trigger, drift handlers + tests/
-ci/                 verify.sh (the workflow is .github/workflows/ci.yml)
-tests/integration/  integration suite (vitest, real Postgres + LocalStack)
-tests/e2e/          Playwright specs + lambda_worker.py
-tests/support/      test env, service setup, docker-compose, Lambda harness
-```
-
-See [CONTRIBUTING.md](CONTRIBUTING.md) for the workflow and [SECURITY.md](SECURITY.md) for reporting issues.
+- **No Row-Level Security.** Tenant isolation depends on every query filtering on `team_id`.
+- **No infrastructure as code.** Buckets, queues, IAM, Lambda settings and schedules cannot be reviewed or reproduced from this repo.
+- **No automatic re-enqueue.** If the SQS send after approval fails, the run stays `queued` until the reconciler marks it failed ([runbook](docs/operations.md)).
+- **Retention:** raw files of failed runs stay in S3, and `data_retention_days` is not enforced. The `guest_` lifecycle rule is the only exception.
+- **The AI budget is checked before a call and recorded after it.** Concurrent requests can overshoot a cap by the cost of the calls in flight. Costs are estimates, not the AWS bill.
+- **Upload limits are per file.** A file under the limit can still exhaust the executor's memory or time on expensive rules. Only `semantic_deduplicate` has a time and row guard.
+- **Guest caps are per IP hash and per cookie.** Users behind one NAT share the caps, and guest work is not carried over on sign-up.
+- **Older deliverables** may still contain `__orig_*` columns in the native download. Export As strips them.
+- **Profiler:** Excel blanks are not counted as nulls, and Parquet is not supported.
+- **Drift alerts** store their diff in `column_definitions`, so the next alert reports a phantom `_diff` column.
+- **`src/middleware.ts`** uses the file convention that Next.js 16 deprecates in favor of `proxy.ts`.
+- **Not covered by tests:** real Clerk sign-in, the RDS Data API, and the Lambdas' IAM-token database connection.
 
 ## License
 
-The project was published as MIT, but no `LICENSE` file has been committed yet.
+No `LICENSE` file has been committed yet.
diff --git a/SECURITY.md b/SECURITY.md
index 571631f..2e7897a 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -19,7 +19,7 @@ Only the `main` branch (what is deployed) receives fixes.
 
 ## Scope notes
 
-The [security model in the README](README.md#security-model) describes what the code enforces today,
+The [security model](docs/security-model.md) describes what the code enforces today,
 including known gaps. Reports in these areas are especially welcome:
 
 - **Tenant isolation.** It relies on `team_id` predicates in application queries. There is no
diff --git a/docs/deployment.md b/docs/deployment.md
new file mode 100644
index 0000000..b7aa9c1
--- /dev/null
+++ b/docs/deployment.md
@@ -0,0 +1,126 @@
+# Deployment
+
+Vercel deploys the web app from `main`. Everything else in this guide is configured by hand in AWS, because the repo has no infrastructure as code. Replace `<...>` with your own values.
+
+## Web app (Vercel)
+
+Vercel runs `npm run build` on every push to `main`. The build needs no secrets, because configuration is validated per request. Set the variables from `src/lib/env.ts` in the Vercel project. The AWS SDK reads its credentials from the standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` variables, which belong to the `cleanstack-vercel` IAM user (see [IAM](#iam)).
+
+## Database migrations
+
+Apply migrations **before** you deploy code that depends on them. Every statement is idempotent, so you can re-run them safely.
+
+| Migration | Creates |
+|---|---|
+| `001_indexes.sql` | Hot-path indexes (`CREATE INDEX CONCURRENTLY`) |
+| `002_ai_usage_tables.sql` | `bedrock_usage`, `ai_spend_limits`. AI metering and spend caps need these tables |
+| `003_guest_sessions.sql` | `guest_sessions`, an index on `bedrock_usage(created_at)` and the demo template. `POST /api/guest` returns 500 without it |
+
+**Option A**, from a machine that can reach the cluster (IAM auth). This also applies `schema.sql` on a fresh database:
+
+```bash
+AURORA_HOST=<cluster endpoint> RDS_CA_BUNDLE=/path/to/global-bundle.pem \
+  node src/lib/migrations/run-migration.mjs        # MIGRATIONS_ONLY=1 skips schema.sql
+```
+
+**Option B**, from AWS CloudShell through the RDS Data API. Upload the `.sql` files first. The script runs one statement per call, split the same way as `run-migration.mjs`:
+
+```bash
+python3 - <<'PY'
+import boto3, pathlib
+rds = boto3.client("rds-data", region_name="us-east-1")
+CLUSTER, SECRET = "<cluster ARN>", "<DB secret ARN>"
+for f in ["001_indexes.sql", "002_ai_usage_tables.sql", "003_guest_sessions.sql"]:
+    lines = pathlib.Path(f).read_text().splitlines()
+    sql = "\n".join(l for l in lines if not l.strip().startswith("--"))
+    for stmt in filter(None, (s.strip() for s in sql.split(";"))):
+        rds.execute_statement(resourceArn=CLUSTER, secretArn=SECRET, database="cleanstack", sql=stmt)
+    print("applied", f)
+PY
+```
+
+## Lambdas
+
+There is no build or deploy script. Each function is deployed as its `handler.py` plus its pinned `requirements.txt` (ai-trigger has no dependencies beyond the standard library). The profiler and executor also attach the `AWSSDKPandas-Python312` layer, which provides pandas and numpy. The code is tested on pandas 2.2.3 and 3.0.6.
+
+All functions are in `us-east-1`:
+
+| Code | Function |
+|---|---|
+| `lambdas/profiler/handler.py` | `cleanstack-profiler` |
+| `lambdas/executor/handler.py` | `cleanstack-executor` |
+| `lambdas/ai-trigger/handler.py` | `cleanstack-ai-trigger` |
+| `lambdas/drift/handler.py` | `cleanstack-drift` |
+
+**Zip-swap (CloudShell).** When only `handler.py` changed, keep the deployed package and replace the handler. Upload each changed `handler.py` to CloudShell as `<name>/handler.py` (for example `executor/handler.py`), then run:
+
+```bash
+for name in profiler executor ai-trigger drift; do     # keep only the ones you changed
+  FN="cleanstack-$name"
+  curl -s -o current.zip "$(aws lambda get-function --function-name "$FN" --region us-east-1 --query Code.Location --output text)"
+  rm -rf pkg && mkdir pkg && unzip -q current.zip -d pkg
+  cp "$name/handler.py" pkg/handler.py
+  (cd pkg && rm -f ../new.zip && zip -qr ../new.zip .)
+  aws lambda update-function-code --function-name "$FN" --region us-east-1 --zip-file fileb://new.zip >/dev/null
+  aws lambda wait function-updated --function-name "$FN" --region us-east-1 && echo "deployed $FN"
+done
+```
+
+If `requirements.txt` changed, rebuild the package with `pip install -r requirements.txt -t pkg` instead, leaving out pandas and numpy, which come from the layer.
+
+### Lambda environment
+
+| Function | Variables |
+|---|---|
+| profiler | `DATABASE_URL` (host, user and database; the password is an IAM token), `APP_URL`, `WEBHOOK_SECRET`, `MAX_UPLOAD_MB` (default 100; keep it equal to the web app's), `SENTRY_DSN`, `AWS_REGION` |
+| ai-trigger | `APP_URL`, `WEBHOOK_SECRET` |
+| executor | `DB_SECRET_ARN`, `S3_RAW_BUCKET`, `S3_PROCESSED_BUCKET`, `SNS_DRIFT_TOPIC_ARN`, `EXECUTOR_MAX_ATTEMPTS` (default 3), `SEMANTIC_DEDUP_MAX_ROWS` (default 500000), `SEMANTIC_DEDUP_BUDGET_S` (default 600), `EXECUTOR_RESERVE_S` (default 120), `SENTRY_DSN`, `AWS_REGION` |
+| drift | `DB_SECRET_ARN`, `AWS_REGION` |
+
+The profiler, ai-trigger and web app must share the same `WEBHOOK_SECRET`.
+
+## AWS setup
+
+**S3**
+
+- **Raw bucket CORS:** allow `PUT`, `GET` and `POST` from the app origin. Uploads are presigned POSTs, and their `content-length-range` policy makes S3 reject oversized files.
+- **Profiler trigger:** a raw bucket event notification on **All object create events** (`s3:ObjectCreated:*`). A trigger on `Put` alone never fires for browser uploads, so runs stay in `pending`.
+- **Processed bucket CORS:** allow `GET` from the app origin. **Export As** fetches the presigned URL from the browser.
+- **Lifecycle rule (raw bucket):** expire current and noncurrent versions under prefix `guest_` after 1 day. This is a backstop in case the guest purge stops.
+
+**SQS**
+
+- **Executor queue (`cleanstack-jobs`):** triggers the executor. Set the visibility timeout to at least the executor's timeout (AWS recommends 6×), and add a dead-letter queue with `maxReceiveCount` ≥ `EXECUTOR_MAX_ATTEMPTS`. `BatchSize: 1` keeps retries simple.
+- **AI jobs queue:** triggers ai-trigger. It is used only when `AI_QUEUE_ENABLED=true`.
+
+**EventBridge schedules.** Both cron routes are `GET` and require `Authorization: Bearer $CRON_SECRET`. `vercel.json` defines no crons, because Vercel Hobby crons run at most once a day. Instead, EventBridge rules call the routes through API destinations. Both destinations share the connection `cleanstack-reconciler-auth`, which sends that header. Both rules run as the role `cleanstack-eventbridge-invoker`. That role has one inline policy per destination, granting `events:InvokeApiDestination`: the original reconciler policy, plus `invoke-purge-guests`.
+
+| Rule | Schedule | API destination | Route |
+|---|---|---|---|
+| `cleanstack-reconciler-5min` | `rate(5 minutes)` | `cleanstack-reconciler-destination` | `/api/cron/reconcile-runs`: marks runs stuck for 20 minutes as `failed` |
+| `cleanstack-purge-guests-hourly` | `rate(1 hour)` | `cleanstack-purge-guests-destination` | `/api/cron/purge-guests`: erases up to 20 expired guests per call |
+
+If you rotate `CRON_SECRET`, update the connection `cleanstack-reconciler-auth` at the same time as Vercel.
+
+## IAM
+
+**`cleanstack-vercel` (web app user)**, least privilege, based on the calls in `src/`:
+
+| Service | Actions | Resource |
+|---|---|---|
+| S3 | `s3:PutObject` | raw bucket (presigned POST, sample data, iteration copies) |
+| S3 | `s3:GetObject` | processed bucket (downloads, Export As, training export, iteration copy source) |
+| S3 | `s3:ListBucket`, `s3:ListBucketVersions`, `s3:DeleteObject`, `s3:DeleteObjectVersion` | both buckets (account deletion and the guest purge) |
+| SQS | `sqs:SendMessage` | `cleanstack-jobs` (executor) and the AI jobs queue |
+| RDS Data API | `rds-data:ExecuteStatement`, `BeginTransaction`, `CommitTransaction`, `RollbackTransaction` | the cluster |
+| Secrets Manager | `secretsmanager:GetSecretValue` | the DB secret |
+| Bedrock | `bedrock:InvokeModel`, `bedrock:InvokeModelWithResponseStream` (the chat builder streams) | the inference profile in `src/lib/ai-config.ts` and the foundation models it routes to |
+
+If the buckets use SSE-KMS, the user also needs `kms:GenerateDataKey` and `kms:Decrypt` on the key.
+
+**Lambda roles**, in addition to the basic execution role and their trigger permissions:
+
+- **profiler:** `s3:GetObject` and `s3:PutObject` (for `extracted_text.txt`) on the raw bucket, and `rds-db:connect` for its database user.
+- **executor:** `s3:GetObject`, `s3:ListBucketVersions`, `s3:DeleteObject` and `s3:DeleteObjectVersion` on the raw bucket, `s3:PutObject` on the processed bucket, `secretsmanager:GetSecretValue` on the DB secret, `rds-db:connect`, and `sns:Publish` on the drift topic. Without `s3:ListBucketVersions`, the executor deletes only the run's two known raw keys.
+- **drift:** `secretsmanager:GetSecretValue`, `rds-db:connect` and outbound HTTPS to `hooks.slack.com`.
+- **ai-trigger:** no AWS API calls beyond its SQS trigger.
diff --git a/docs/operations.md b/docs/operations.md
new file mode 100644
index 0000000..be2bf81
--- /dev/null
+++ b/docs/operations.md
@@ -0,0 +1,18 @@
+# Operations runbook
+
+**Logs.** Server routes write one JSON object per line (`src/lib/logger.ts`) with `ts`, `level`, `msg`, `service` and `route`. Filter by `run_id`, `route` or `level`. Keys that look like credentials are redacted. The Lambdas log plain text to CloudWatch.
+
+| Symptom | What to check or do |
+|---|---|
+| Run stuck in `profiling`, `awaiting_ai`, `queued` or `running` | `reconcile-runs` marks these runs `failed` after 20 minutes, so first confirm that the EventBridge rule `cleanstack-reconciler-5min` is firing. For a run stuck in `queued`, search the log for `SQS send failed; run left queued without a message`. If you find it, re-send the message: `aws sqs send-message --queue-url "$SQS_QUEUE_URL" --message-body '{"run_id":"<id>"}'`. This is safe because execution is idempotent. |
+| Runs stay in `pending` after upload | The profiler trigger is missing `Post`. It must fire on all object create events. |
+| Run `failed` with `Attempt n/3 failed, retrying` in its history | SQS retries transient errors up to `EXECUTOR_MAX_ATTEMPTS` times, and the last attempt marks the run failed. Look up the run id in the executor's CloudWatch logs. |
+| Approved rule shows **Not applied** | This is expected when a rule is unsafe or invalid. The page shows the reason, and the rest of the run is unaffected. |
+| User says original values are missing from the file | This is by design. Originals are in `processed/{pipeline}/{run}/audit.csv`. |
+| **Export As** fails but download works | Check CORS on the processed bucket. |
+| `DELETE /api/account` returns 500 "No account data was removed" | The S3 purge failed. Search the log for `S3 purge incomplete` or `S3 purge failed`, fix the IAM permissions and have the user retry. The database is untouched, so the retry erases everything. `all_versions_purged: false` in a 200 response means `s3:ListBucketVersions` is missing. |
+| Startup log `invalid or missing environment` | Set the listed variables in Vercel and redeploy. |
+| Rotating `WEBHOOK_SECRET` | Update Vercel and the profiler and ai-trigger Lambdas together. Requests fail with 401 while the values differ. |
+| AI spend | `GET /api/admin/ai-spend` with `x-admin-secret` lists this month's estimated spend per team. |
+| Guests see "Guest AI capacity is used up for today" | All guests together have reached $5 of estimated Bedrock spend since 00:00 UTC. The cap resets at midnight UTC (`GUEST_LIMITS.aiSpendAllGuestsPerDayUsd`). |
+| Guest data not being deleted | Check the rule `cleanstack-purge-guests-hourly` and its `invoke-purge-guests` policy. Each call erases up to 20 expired guests. The `guest_` lifecycle rule removes S3 objects after 1 day even if the purge stops, but the database rows remain until the purge runs. |
diff --git a/docs/security-model.md b/docs/security-model.md
new file mode 100644
index 0000000..d740b2e
--- /dev/null
+++ b/docs/security-model.md
@@ -0,0 +1,34 @@
+# Security model
+
+This page describes what the code enforces today. Settings that live only in AWS are covered in [deployment.md](deployment.md).
+
+- **Authentication.** Clerk handles sign-in. `src/middleware.ts` protects the app pages and the user API routes, and each route also calls `auth()` (`src/lib/auth.ts`) and returns 401 when there is no user.
+- **Tenant isolation is enforced in application code, not by the database.** Every user query filters on `pipelines.team_id = <user id>`. `queryWithTeam()` sets `app.team_id`, but `schema.sql` defines **no Row-Level Security policies**, so that setting has no effect. A new query that leaves out the `team_id` predicate would leak data across tenants.
+- **Service-to-service calls.** Lambda webhooks, crons and admin routes use shared secrets, compared with `safeCompare` (`src/lib/secrets.ts`: SHA-256, then `timingSafeEqual`). The check fails closed when the expected secret is not configured.
+- **File access.** The browser uploads straight to S3 with a 300 s presigned POST. Its policy pins the key and the content type and caps the size with `content-length-range` (`src/lib/upload-limits.ts`: 100 MB or `MAX_UPLOAD_MB`, 2 MB for guests). `/api/upload` rejects a larger declared size with 413. The profiler marks a run failed, without reading the object, if the stored object is over the limit. Downloads use a 120 s presigned GET, only for the caller's own completed run. Files never pass through the serverless functions.
+- **Data lifecycle.**
+  - After a successful execution, the executor deletes every object version under the run's raw prefix. This is the `auto_delete_raw` setting, which is on by default. Raw files of failed runs are kept.
+  - `DELETE /api/account?confirm=true` and the guest purge both use `eraseTeam()` (`src/lib/erase-team.ts`). It removes every version and delete marker under the team's S3 prefixes, then deletes the database rows. If the S3 step fails, it stops before touching the database. The guest purge keeps `bedrock_usage` rows so that the shared AI cap and cost reporting stay correct.
+- **AI.** Uploaded content is sent to Amazon Bedrock inside `<user_data>` tags, with an instruction to treat it as data. This reduces prompt-injection risk but does not prevent it, which is why rules need approval or a committee vote. `checkAiBudget` (`src/lib/bedrock-meter.ts`) runs in Postgres before every Bedrock call: suggest-transforms, the auto-clean committee, the chat builder and synthetic data. Every call is metered in `bedrock_usage`. Costs are estimates based on the prices in `src/lib/ai-config.ts`.
+- **Rate limits.** Upstash sliding windows per user: 20 uploads/h, 50 AI calls/h and 30 chat messages/h. They fail open when Redis is not configured. The guest limits below are enforced in Postgres and do not depend on Upstash.
+- **Headers.** `next.config.ts` sets a CSP (`src/lib/csp.ts`). For scripts it allows `'unsafe-inline'` and `'unsafe-eval'`, which Next.js and Clerk need without nonces. It adds the Cloudflare Turnstile origins when Turnstile is configured.
+- **Secrets.** Nothing secret is committed, and `.env.example` contains only placeholders. AWS account ids and ARNs from older commits remain in git history.
+
+## Guest access
+
+Guest access is off until `GUEST_COOKIE_SECRET` is set. `POST /api/guest` checks Turnstile when `TURNSTILE_SECRET_KEY` is set. It then issues an httpOnly, `SameSite=Lax`, `Secure` cookie `cs_guest` = `guest_<22 chars>.<expiry>.<HMAC-SHA256>`, which is valid for 24 h. `auth()` returns the `guest_…` id only when no Clerk user is signed in. That id is the guest's `team_id`, so tenant isolation applies unchanged. Sessions are recorded in `guest_sessions` with an HMAC of the client IP; the raw IP is never stored.
+
+| Limit | Value | Enforced in |
+|---|---|---|
+| File size | 2 MB | presigned POST policy, `/api/upload` (413), profiler |
+| Rows per run | 5,000 | `suggest-transforms`, before templates or AI (the run fails) |
+| Rows per guest | 10,000 across all passes | `checkQuota` (guest plan) |
+| Uploads | 3 per guest, 10 per IP hash per 24 h | capped `INSERT` in `src/lib/guest-quota.ts`, which also refuses expired sessions |
+| Pipelines | 5 per guest | `POST /api/pipelines` |
+| AI calls | 10 per rolling hour | `checkAiBudget` |
+| AI spend | $0.25 per guest, and $5 per UTC day for all guests together | `checkAiBudget` |
+| Sessions | 5 per IP hash, and 200 overall, per 24 h | `POST /api/guest` |
+
+The values live in `src/lib/guest-limits.ts`. Guests cannot use the chat builder or synthetic data, auto-clean, training export, Slack alerts, account deletion, templates or `/api/admin`. The middleware answers 403 for these, or redirects `/templates` to the dashboard, and each route also rejects guest ids (`forbidGuest`). **Try with sample data** (`POST /api/sample-data`) uploads the CSV from `src/lib/sample-data.ts` and attaches the demo template that migration 003 seeds, so it makes no Bedrock call. After 24 h the cookie stops working, and `/api/cron/purge-guests` erases the guest's data. Guest work is not carried over to a new account on sign-up.
+
+Report vulnerabilities as described in [SECURITY.md](../SECURITY.md).
diff --git a/docs/transform-rules.md b/docs/transform-rules.md
new file mode 100644
index 0000000..93f2fb2
--- /dev/null
+++ b/docs/transform-rules.md
@@ -0,0 +1,29 @@
+# Transform rules
+
+The model can suggest only rules that the executor implements, and a contract test (`lambdas/tests/test_contracts.py`) keeps the two lists in sync. In auto-clean, each rule's risk tier sets the number of committee votes it needs: LOW 1/3, MEDIUM 2/3, HIGH 3/3. When a rule fails or is unsafe (an unknown type, a missing column, more than 20% row loss, a bad regex, or a cast that would lose data), the executor restores the frame and records the rule as *not applied*, with the reason.
+
+## Tabular
+
+| Rule | What it does |
+|---|---|
+| `trim_whitespace` | Strips leading and trailing whitespace in text columns |
+| `deduplicate` | Drops exact duplicate rows |
+| `semantic_deduplicate` | Drops near-duplicate rows, keeping the first. Uses a MinHash Jaccard estimate ≥ `threshold` (default 0.8) over lowercased word sets, with LSH banding |
+| `fill_nulls` | Fills nulls with the mean, median, mode or a constant |
+| `drop_nulls` | With a column, drops rows where that column is null. Without one, drops rows with at least `ceil(threshold × columns)` nulls |
+| `type_cast` | Casts to float, int, datetime or str. Not applied if values would be lost (for example `2.5` → int) |
+| `normalize` | Text and date columns only. Converts dates to `YYYY-MM-DD` and trims and lowercases other text, or maps it through `value_map` |
+| `filter`, `filter_extended` | Removes rows that match a condition |
+| `rename`, `column_header_normalize` | Renames columns, or converts headers to snake_case |
+| `ner_redact` | Regex-based redaction of names, organizations, addresses or ZIP codes, and dates. This is a heuristic, not a trained NER model |
+| `ffill`, `bfill` | Fills nulls from the previous or next row |
+| `bool_cast` | Normalizes yes/no, true/false and 1/0 variants |
+| `outlier_cap` | Caps values at the IQR fences |
+| `multi_currency_strip` | Strips currency symbols and converts the values to numbers |
+| `split_column` | Splits a column on a delimiter into new columns |
+
+Rules that rewrite values keep the original in an `__orig_<column>` sidecar, which is written only to `audit.csv`.
+
+## Documents (PDF, DOCX, TXT)
+
+`strip_pii`, `ner_redact`, `fix_encoding`, `remove_headers_footers`, `remove_blank_lines`, `normalize_whitespace`, `strip_html` and `redact_pattern`. Patterns for `redact_pattern` are capped at 200 characters, and there is no protection against catastrophic backtracking.
diff --git a/src/app/api/cron/purge-guests/route.ts b/src/app/api/cron/purge-guests/route.ts
index 0cc2eec..83d886c 100644
--- a/src/app/api/cron/purge-guests/route.ts
+++ b/src/app/api/cron/purge-guests/route.ts
@@ -43,7 +43,7 @@ LIMIT $1`;
  * Erases expired guests: S3 objects (every version) and DB rows, through the
  * same helper as account deletion. bedrock_usage rows are kept for the AI
  * spend accounting. Call it hourly with Authorization: Bearer $CRON_SECRET
- * (README, Deployment). The S3 lifecycle rule on the guest_ prefix is the
+ * (docs/deployment.md). The S3 lifecycle rule on the guest_ prefix is the
  * backstop if this stops running.
  */
 export async function GET(req: Request) {
diff --git a/src/components/DownloadButton.tsx b/src/components/DownloadButton.tsx
index 0870e77..b588934 100644
--- a/src/components/DownloadButton.tsx
+++ b/src/components/DownloadButton.tsx
@@ -209,7 +209,7 @@ export default function DownloadButton({
     try {
       const link = await getDownloadLink(runId);
       // Cross-origin fetch of the presigned URL: requires a CORS GET rule on
-      // the processed bucket for the app origin (see README, ops runbook).
+      // the processed bucket for the app origin (see docs/deployment.md).
       const res = await fetch(link.url);
       if (!res.ok) throw new Error(`Download failed (HTTP ${res.status})`);
       const buf = await res.arrayBuffer();
diff --git a/src/lib/bedrock-meter.ts b/src/lib/bedrock-meter.ts
index 39f7bb1..9310f22 100644
--- a/src/lib/bedrock-meter.ts
+++ b/src/lib/bedrock-meter.ts
@@ -79,7 +79,7 @@ export type AiBudget =
  *   bedrock_usage, so it holds without Upstash.
  *
  * Usage is recorded after a call, so concurrent requests can overshoot by the
- * cost of the calls in flight; see README "AI spend".
+ * cost of the calls in flight; see README "Known limitations".
  */
 export async function checkAiBudget(teamId: string, calls = 1): Promise<AiBudget> {
   if (!isGuestId(teamId)) {
diff --git a/src/lib/erase-team.ts b/src/lib/erase-team.ts
index 2c4796a..2842671 100644
--- a/src/lib/erase-team.ts
+++ b/src/lib/erase-team.ts
@@ -75,7 +75,7 @@ export async function eraseTeam(s3: S3Client, teamId: string, opts: { keepUsage?
     const unversioned = purges.filter((p) => !p.versioned).map((p) => p.prefix);
     if (unversioned.length) {
       // Missing s3:ListBucketVersions: only current versions were removed and
-      // older versions may survive on a versioned bucket (README, security model).
+      // older versions may survive on a versioned bucket (docs/security-model.md).
       log.error("ListObjectVersions denied; only current versions purged", { prefixes: unversioned });
     }
 
diff --git a/src/lib/sample-data.test.ts b/src/lib/sample-data.test.ts
index d9f8baf..e3cf9c4 100644
--- a/src/lib/sample-data.test.ts
+++ b/src/lib/sample-data.test.ts
@@ -20,7 +20,7 @@ describe("sample data", () => {
   });
 
   it("every 003 statement is safe to send alone through the RDS Data API", () => {
-    // One statement per execute-statement call (README checklist): no semicolons inside statements.
+    // One statement per execute-statement call (docs/deployment.md, option B): no semicolons inside statements.
     const stmts = splitStatements(MIGRATION);
     expect(stmts.length).toBe(6);
     for (const s of stmts) expect(s).not.toContain(";");