Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions bin/crawlproof.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env node
// The `crawlproof` entry point.
//
// The CLI is TypeScript inside a Next.js app rather than a built package, so
// running it means running tsx against cli/index.ts. Everything here is about
// finding the repo and the right tsx no matter where the caller stood, because
// the alternative — telling people to cd into a checkout and type
// `npm run cli --` — is a setup step, and those do not survive contact.

import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const repo = resolve(here, "..");
const entry = join(repo, "cli", "index.ts");

if (!existsSync(entry)) {
console.error(`crawlproof: cannot find ${entry}`);
console.error("The launcher must sit in bin/ inside the crawlproof.com checkout.");
process.exit(1);
}

// Prefer the checkout's own tsx; fall back to npx so a fresh clone still runs.
const localTsx = join(repo, "node_modules", ".bin", "tsx");
const [cmd, prefix] = existsSync(localTsx) ? [localTsx, []] : ["npx", ["--yes", "tsx"]];

const child = spawn(cmd, [...prefix, entry, ...process.argv.slice(2)], {
cwd: repo,
stdio: "inherit",
env: process.env,
});

child.on("error", (err) => {
console.error(`crawlproof: could not start tsx (${err.message})`);
console.error(`Run \`npm install\` in ${repo}.`);
process.exit(1);
});

// Relay the child's fate rather than inventing one: a TUI killed by ctrl+c
// should not look like a clean exit to whatever called this.
child.on("exit", (code, signal) => {
if (signal) {
process.kill(process.pid, signal);
return;
}
process.exit(code ?? 0);
});
48 changes: 22 additions & 26 deletions cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,9 +185,25 @@
// Both talk to /api/ads/v1/* with a CrawlProof API token (Social → API
// tokens), the same token the MCP server and the myna plugin use.

/**
* The API token, from the flag, the environment, or ~/.crawlproof.json.
*
* The config file is last and exists so that using the CLI is not conditional
* on remembering to export a secret first. Same shape and same reasoning as
* ~/.coinpay.json, which `coinpayAuth` below reads.
*/
function apiToken(args: Args): string | null {
const token = (args.flags.token as string | undefined) ?? process.env.CRAWLPROOF_TOKEN ?? null;
return token && token.trim() ? token.trim() : null;
const direct = (args.flags.token as string | undefined) ?? process.env.CRAWLPROOF_TOKEN;
if (direct && direct.trim()) return direct.trim();

try {
const home = process.env.HOME ?? process.env.USERPROFILE ?? "";
const file = process.env.CRAWLPROOF_CONFIG ?? `${home}/.crawlproof.json`;
const token = (JSON.parse(readFileSync(file, "utf8")) as { token?: string }).token;
return token && token.trim() ? token.trim() : null;
} catch {
return null;
}
}

function apiBase(args: Args): string {
Expand All @@ -210,11 +226,11 @@
}
const res = await fetch(`${apiBase(args)}${path}`, {
method,
headers: {
authorization: `Bearer ${token}`,
accept: "application/json",
...(body ? { "content-type": "application/json" } : {}),
},

Check warning

Code scanning / CodeQL

File data in outbound network request Medium

Outbound network request depends on
file data
.
body: body ? JSON.stringify(body) : undefined,
});
const text = await res.text();
Expand Down Expand Up @@ -364,29 +380,8 @@
return 0;
}

const project = json.project as { name?: string; url?: string } | undefined;
const totals = json.totals as { visitors?: number; pageviews?: number } | undefined;
const list = (key: string) => (Array.isArray(json[key]) ? (json[key] as { label: string; value: number }[]) : []);

process.stdout.write(`${project?.name ?? "project"} ${range} ${who}\n`);
process.stdout.write(`${totals?.visitors ?? 0} visitors, ${totals?.pageviews ?? 0} pageviews\n`);

const section = (title: string, items: { label: string; value: number }[]) => {
if (!items.length) return;
process.stdout.write(`\n${title}\n`);
const width = Math.min(46, Math.max(...items.map((i) => i.label.length)));
for (const item of items.slice(0, 10)) {
process.stdout.write(` ${item.label.slice(0, width).padEnd(width)} ${item.value}\n`);
}
};
section("Sources", list("sources"));
section("Referrers", list("referrers"));
section("Pages", list("pages"));

// Nothing at all is a real answer, and the likeliest cause is worth naming.
if (!(totals?.pageviews ?? 0) && !list("sources").length) {
process.stdout.write(`\nNothing in this window. Check the tag is on the page, or widen --range.\n`);
}
const { renderStats } = await import("../lib/dashboard/stats-text");
process.stdout.write(renderStats(json as Parameters<typeof renderStats>[0], { range, who }));
return 0;
}

Expand Down Expand Up @@ -593,7 +588,8 @@
CRAWLPROOF_SITE_URL Override the API base URL for 'report', 'sweep', 'track', 'ads' and 'slots'.
CRAWLPROOF_PROJECT Default project UUID for 'track'.
CRAWLPROOF_TOKEN API token (crp_…) for 'ads', 'slots', 'stats' and
'dashboard'; --token overrides.
'dashboard'; --token overrides. Falls back to the
'token' field of ~/.crawlproof.json.
COINPAY_SESSION_TOKEN CoinPay merchant JWT for the money half of
'dashboard'. Defaults to jwtToken in ~/.coinpay.json,
which 'coinpay auth login' writes.
Expand Down
47 changes: 47 additions & 0 deletions lib/dashboard/stats-text.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Rendering a /api/tracker/v1/stats answer as plain text.
//
// Pure and string-returning rather than writing to stdout, because two CLIs
// print it now — the in-repo one and the published @profullstack/crawlproof —
// and a printer that owns the process is a printer that cannot be shared or
// tested.

export type StatsItem = { label: string; value: number };

export type StatsAnswerish = {
project?: { name?: string; url?: string } | null;
totals?: { visitors?: number; pageviews?: number } | null;
sources?: StatsItem[] | null;
referrers?: StatsItem[] | null;
pages?: StatsItem[] | null;
};

const list = (v: StatsItem[] | null | undefined): StatsItem[] => (Array.isArray(v) ? v : []);

export function renderStats(
answer: StatsAnswerish,
{ range, who, rows = 10 }: { range: string; who: string; rows?: number },
): string {
const totals = answer.totals ?? {};
const out: string[] = [
`${answer.project?.name ?? "project"} ${range} ${who}`,
`${totals.visitors ?? 0} visitors, ${totals.pageviews ?? 0} pageviews`,
];

const section = (title: string, items: StatsItem[]) => {
if (!items.length) return;
out.push("", title);
const width = Math.min(46, Math.max(...items.map((i) => i.label.length)));
for (const item of items.slice(0, rows)) {
out.push(` ${item.label.slice(0, width).padEnd(width)} ${item.value}`);
}
};
section("Sources", list(answer.sources));
section("Referrers", list(answer.referrers));
section("Pages", list(answer.pages));

// Nothing at all is a real answer, and the likeliest cause is worth naming.
if (!(totals.pageviews ?? 0) && !list(answer.sources).length) {
out.push("", "Nothing in this window. Check the tag is on the page, or widen --range.");
}
return `${out.join("\n")}\n`;
}
3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"name": "crawlproof",
"version": "0.3.0",
"bin": {
"crawlproof": "./bin/crawlproof.mjs"
},
"private": true,
"license": "AGPL-3.0-only",
"type": "module",
Expand Down
2 changes: 2 additions & 0 deletions packages/cli/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dist/
node_modules/
74 changes: 74 additions & 0 deletions packages/cli/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# @profullstack/crawlproof

What the fleet costs and what it returns, in a terminal.

```
npm install -g @profullstack/crawlproof
crawlproof dashboard
```

Five live screens over three feeds that are not otherwise in the same place:
CrawlProof's tracker for who arrived, its ad network for what was delivered,
and CoinPay for what the bank actually did.

| Screen | Answers |
| --- | --- |
| ROI | Monthly burn against revenue, cost per reader, break-even |
| Traffic | Every site on the account, ranked, with its share of the cost |
| Ads | Delivery as advertiser and as publisher, and ad-driven arrivals |
| Money | Earnings, bank position, invoices, income vs spending by month |
| Spend | Who we pay, largest first, and burn by category |

`1`–`5` or Tab switches screens, `w` cycles the window, `b` cycles humans /
all / bots, `r` refreshes, `?` explains the arithmetic, `q` quits.

## Two rules the numbers keep

**Self-deal is not revenue.** Where an account advertises on its own slots, ad
spend and ad earnings are one dollar moving between two pockets. They are shown
under *Internal* and counted as neither cost nor revenue.

**Personal money is not business cost.** A bank feed carries groceries next to
servers, so cost is the business scope only — joined from each transaction's
account to that account's scope.

Everything is normalised to a monthly rate and then prorated onto the traffic
window, because burn is a rate: the traffic side can be asked for an hour while
the bank side only answers in weeks.

The dashboard also reports what it cannot know. A site that did not answer is
missing rather than zero. A vendor list built from one page of a longer ledger
says so. A fleet whose visits run far above its pageviews — a "visitor" is any
non-crawler hit, which on a site with a machine-readable endpoint runs orders of
magnitude above pages anyone read — says that next to the number, and offers the
per-pageview figure instead.

## Commands

```
crawlproof dashboard [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all]
[--interval=60] [--sites=a.com,b.com] [--concurrency=8]
[--no-coinpay] [--json]
crawlproof stats [site] [--range=1d] [--who=humans] [--json]
```

`--json` prints the same snapshot the screens render, for a script or a box
with no terminal. Aliases for `dashboard`: `roi`, `tui`.

## Auth

| | |
| --- | --- |
| `CRAWLPROOF_TOKEN` | API token (`crp_…`) from Social → API tokens. Also read from the `token` field of `~/.crawlproof.json`. `--token` wins. |
| `CRAWLPROOF_SITE_URL` | API base, default `https://crawlproof.com`. |
| `COINPAY_SESSION_TOKEN` | CoinPay merchant JWT for the money screens. Defaults to `jwtToken` in `~/.coinpay.json`, which `coinpay auth login` writes. |

Without a CoinPay session the traffic and ads screens still work and the money
panels say what is missing, rather than showing zero.

Needs Node 22.6 or newer, and a terminal for the dashboard. `--json` needs
neither.

## License

MIT
36 changes: 36 additions & 0 deletions packages/cli/build.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Bundle the publishable CLI out of the app's own source.
//
// The point of bundling rather than moving files is that lib/dashboard/* and
// cli/dashboard.ts stay where the test suite already covers them and where the
// in-repo CLI already imports them. The package is a build artifact of the
// same code, so there is one implementation and it cannot drift.
//
// hqtui and the CoinPay SDK stay external: they are real dependencies with
// their own release cadence, declared in package.json and installed alongside.

import { build } from "esbuild";
import { chmod, readFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const out = join(here, "dist", "cli.mjs");

const pkg = JSON.parse(await readFile(join(here, "package.json"), "utf8"));

await build({
entryPoints: [join(here, "src", "bin.ts")],
outfile: out,
bundle: true,
platform: "node",
format: "esm",
target: "node22",
// Anything with its own version belongs in package.json, not inlined here.
external: [...Object.keys(pkg.dependencies ?? {}), "node:*"],
banner: { js: "#!/usr/bin/env node" },
legalComments: "none",
logLevel: "info",
});

await chmod(out, 0o755);
console.log(`built ${out}`);
44 changes: 44 additions & 0 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"name": "@profullstack/crawlproof",
"version": "0.1.0",
"description": "What the fleet costs and what it returns: a live terminal dashboard over CrawlProof traffic, ad delivery and CoinPay banking.",
"license": "MIT",
"type": "module",
"homepage": "https://crawlproof.com",
"bin": {
"crawlproof": "./dist/cli.mjs"
},
"files": [
"dist",
"README.md"
],
"engines": {
"node": ">=22.6"
},
"scripts": {
"build": "node build.mjs",
"prepublishOnly": "node build.mjs"
},
"dependencies": {
"@profullstack/coinpay": "^0.9.0",
"@profullstack/hqtui": "^0.1.11"
},
"devDependencies": {
"esbuild": "^0.25.0"
},
"publishConfig": {
"access": "public"
},
"keywords": [
"crawlproof",
"tui",
"dashboard",
"analytics",
"roi"
],
"repository": {
"type": "git",
"url": "git+https://github.com/profullstack/crawlproof.com.git",
"directory": "packages/cli"
}
}
7 changes: 7 additions & 0 deletions packages/cli/src/bin.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// The executable entry. No shebang here: build.mjs adds one as a banner, and
// two of them is a syntax error rather than a harmless duplicate.
import { main } from "./cli";

main(process.argv.slice(2)).then((code) => {
process.exitCode = code ?? 0;
});
Loading
Loading