feat!: generate compact parsers with corpus profiles

.github/workflows/robustness.yml

@@ -30,7 +30,6 @@ jobs:
       - run: bun install --frozen-lockfile
       - run: bun run build/ci
       - run: bun run test/ci-setup
-      # The packages ship the committed src/, so it must be what grammar.js generates.
       - run: git diff --exit-code -- src
       - run: bun run test/ci
   # Runs the parser under libFuzzer with AddressSanitizer and UndefinedBehaviorSanitizer, seeded with the

Cargo.toml

@@ -25,4 +25,4 @@ tree-sitter-language = "0.1"
 cc = "1.2"
 
 [dev-dependencies]
-tree-sitter = { package = "willbooster-tree-sitter", version = "1.2.2" }
+tree-sitter = { package = "willbooster-tree-sitter", version = "1.4.0" }

README.md

@@ -19,7 +19,7 @@ This fork fixes parsing bugs and raises conformance with [ECMAScript (ECMA-262)]
 
 The npm package ships `tree-sitter-javascript.wasm` for
 [@willbooster/web-tree-sitter](https://www.npmjs.com/package/@willbooster/web-tree-sitter), which runs in Node.js, Bun,
-browsers, and Cloudflare Workers. Use runtime 1.2.2 or newer. In Node.js and Bun, load it from the package:
+browsers, and Cloudflare Workers. Use runtime 1.3.0 or newer. In Node.js and Bun, load it from the package:
 
 ```js
 import { fileURLToPath } from 'node:url';
@@ -62,13 +62,12 @@ The package also ships the node types in `src/node-types.json`, the highlight, i
 `require('@willbooster/tree-sitter-javascript/grammar')`). The crate ships `tree-sitter.json` and `queries/` too.
 
 In Rust, depend on the [crate](https://crates.io/crates/willbooster-tree-sitter-javascript) and on
-[willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter) 1.2.2 or newer. The canonical constructor
-supertype query requires this runtime version. This is the runtime this package is tested and fuzzed with:
+[willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter) 1.3.0 or newer for the compact ABI 16 parser:
 
 ```toml
 [dependencies]
-tree-sitter = { package = "willbooster-tree-sitter", version = "1.2.2" }
-tree-sitter-javascript = { package = "willbooster-tree-sitter-javascript", version = "3" }
+tree-sitter = { package = "willbooster-tree-sitter", version = "1.3.0" }
+tree-sitter-javascript = { package = "willbooster-tree-sitter-javascript", version = "4" }
 ```
 
 ```rust
@@ -92,6 +91,13 @@ locked in `Cargo.lock` (`script/tree-sitter`), whose generator and runtime have
 `script/fork-cli` downloads that CLI into `.tmp/` from its GitHub Release on first use, or builds it with `cargo` when
 the download fails or the release has no binary that runs here.
 
+`bun run generate` records a fresh ABI 16 generation profile from the applicable `test/corpus` cases and Git-tracked
+files in `examples/`, then generates compact parser tables. The parser also embeds metadata from `tree-sitter.json`. After changing a grammar,
+`tree-sitter.json`, a corpus case, or a tracked example,
+regenerate and commit `src/`. Stage added or removed examples with `git add -A examples` before generation so the profile uses the intended file list.
+Profiles in `.tmp/generation-profiles/` are temporary and must not be committed. `bun run build-wasm`, `bun run build/ci`,
+and the release build regenerate the parsers before compiling them.
+
 `bun run test` runs:
 
 - the corpus in `test/corpus`, with the native build and with the Wasm build (the first run downloads the WASI SDK);

package.json

@@ -22,10 +22,11 @@
     "*.wasm"
   ],
   "scripts": {
-    "build-wasm": "script/tree-sitter build --wasm",
-    "build/ci": "script/tree-sitter generate && script/tree-sitter build --wasm",
+    "build-wasm": "script/generate --build-wasm",
+    "build/ci": "bun run build-wasm",
     "cleanup": "bun wb lint --fix --format",
     "format": "bun wb lint --format",
+    "generate": "script/generate",
     "lint": "bun wb lint",
     "lint-fix": "bun wb lint --fix",
     "prepare": "(bun lefthook install || true) && (test ! -f .lefthook/normalize-bun-lockfile.sh || bash .lefthook/normalize-bun-lockfile.sh)",
@@ -46,7 +47,7 @@
     "@willbooster/oxfmt-config": "1.2.2",
     "@willbooster/oxlint-config": "1.4.8",
     "@willbooster/wb": "22.13.1",
-    "@willbooster/web-tree-sitter": "1.2.2",
+    "@willbooster/web-tree-sitter": "1.4.0",
     "conventional-changelog-conventionalcommits": "10.4.0",
     "lefthook": "2.1.14",
     "oxfmt": "0.70.0",
@@ -62,7 +63,7 @@
     "wrangler": "4.136.2"
   },
   "peerDependencies": {
-    "@willbooster/web-tree-sitter": "^1.2.2"
+    "@willbooster/web-tree-sitter": "^1.3.0"
   },
   "peerDependenciesMeta": {
     "@willbooster/web-tree-sitter": {

script/generate

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+root=$(cd "$(dirname "$0")/.." && pwd)
+cd "$root"
+mkdir -p .tmp/generation-profiles
+
+lock="$root/.tmp/generation-profiles/lock"
+if ! mkdir "$lock" 2> /dev/null; then
+  echo "Generation lock exists: $lock" >&2
+  echo "Wait for the active build. If it was killed, stop its remaining processes, then run: rmdir '$lock'" >&2
+  exit 1
+fi
+trap 'rmdir "$lock"' EXIT
+
+workspace=$(mktemp -d "$root/.tmp/generation-profiles/run.XXXXXX")
+trap 'rm -rf "$workspace"; rmdir "$lock"' EXIT
+examples="$workspace/examples.z"
+git ls-files -z -- examples > "$examples"
+
+training=(--corpus "$root/test/corpus")
+while IFS= read -r -d '' file; do
+  training+=("$root/$file")
+done < "$examples"
+
+node -e 'const {grammars} = require("./tree-sitter.json"); console.log([...new Set(grammars.map(({path}) => path ?? "."))].sort().join("\n"));' | while IFS= read -r grammar; do
+  (
+    backup="$workspace/src"
+    cp -Rp "$grammar/src" "$backup"
+    trap 'status=$?; if (( status != 0 )); then rm -rf "$grammar/src"; mv "$backup" "$grammar/src"; fi' EXIT
+    script/tree-sitter generate "$grammar/grammar.js" --abi 16
+    name=$(node -e 'console.log(JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")).name)' "$grammar/src/grammar.json")
+    profile="$workspace/$name.json"
+    script/tree-sitter profile --grammar-path "$grammar" --output "$profile" "${training[@]}"
+    script/tree-sitter generate "$grammar/grammar.js" --abi 16 --profile "$profile"
+    if [[ ${1:-} == --build-wasm ]]; then
+      script/tree-sitter build --wasm "$grammar"
+    fi
+  )
+  rm -rf "$workspace/src"
+done
+
+mv "$examples" "$root/.tmp/generation-profiles/examples.z"

script/set-grammar-version

@@ -1,27 +1,17 @@
 #!/usr/bin/env node
-// Writes the release version into tree-sitter.json, the language metadata of the generated src/parser.c,
-// Cargo.toml, and package.json, none of which semantic-release updates.
-
 const fs = require('node:fs');
 const path = require('node:path');
 
 const version = process.argv[2];
 const match = /^(\d+)\.(\d+)\.(\d+)/.exec(version ?? '');
 if (!match) throw new Error('Usage: script/set-grammar-version <major.minor.patch>');
-const [, major, minor, patch] = match;
 
 const root = path.join(__dirname, '..');
 const configFile = path.join(root, 'tree-sitter.json');
 const config = JSON.parse(fs.readFileSync(configFile, 'utf8'));
 config.metadata.version = version;
 fs.writeFileSync(configFile, `${JSON.stringify(config, null, 2)}\n`);
 
-const parserFile = path.join(root, 'src', 'parser.c');
-const parser = fs.readFileSync(parserFile, 'utf8');
-const metadata = /(\.metadata = \{\s*\.major_version = )\d+(,\s*\.minor_version = )\d+(,\s*\.patch_version = )\d+/;
-if (!metadata.test(parser)) throw new Error('src/parser.c has no language metadata block');
-fs.writeFileSync(parserFile, parser.replace(metadata, `$1${major}$2${minor}$3${patch}`));
-
 const manifestFile = path.join(root, 'Cargo.toml');
 const manifest = fs.readFileSync(manifestFile, 'utf8');
 const crateVersion = /^(version = )"[^"]*"/m;