From bfbcb5fcbe355c94f6f286a33c7aa3d7b60168c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sat, 24 Oct 2026 20:03:35 -0400 Subject: [PATCH 1/7] perf: compress inline data with brotli instead of gzip Switch `gzipInline` to `brotliInline`, using maximum quality and the largest window so long-range repetition (e.g. in `Names/index.mjs`) is exploited. Generated modules now import `brotliDecompressSync` from `node:zlib`. For Unicode 18.0.0 this shrinks the package from 700 KB to 590 KB packed (1.96 MB to 1.79 MB unpacked); `Names/index.mjs` goes from 228 KB to 158 KB. Assisted-By: Claude Fable 5.1 --- scripts/utils.mjs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/utils.mjs b/scripts/utils.mjs index c294a44..4152812 100644 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -8,14 +8,21 @@ import { encodeRanges, encodeRegenerate } from './encode-ranges.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const gzipInline = (data) => { +const brotliInline = (data) => { if (data instanceof Map) { - return `new Map(${gzipInline([...data])})`; + return `new Map(${brotliInline([...data])})`; } const json = JSON.stringify(data); - const gzipBuffer = zlib.gzipSync(json); - const str = gzipBuffer.toString('base64'); - return `JSON.parse(gunzipSync(Buffer.from('${str}', 'base64')))`; + const brotliBuffer = zlib.brotliCompressSync(json, { + params: { + [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, + [zlib.constants.BROTLI_PARAM_LGWIN]: + zlib.constants.BROTLI_MAX_WINDOW_BITS, + [zlib.constants.BROTLI_PARAM_SIZE_HINT]: Buffer.byteLength(json), + }, + }); + const str = brotliBuffer.toString('base64'); + return `JSON.parse(brotliDecompressSync(Buffer.from('${str}', 'base64')))`; }; const range = (start, stop) => { @@ -164,7 +171,7 @@ const writeFiles = (options) => { // Sequence properties are special. if (type === 'Sequence_Property' || isNameAliases) { const sequences = codePoints; - const output = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(sequences)};\n`; + const output = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(sequences)};\n`; writeFileIfChanged(path.resolve(dir, 'index.mjs'), output); writeFileIfChanged( path.resolve(dir, 'index.d.mts'), @@ -222,8 +229,8 @@ const writeFiles = (options) => { codePointsFileContent = `export default ${jsesc(codePoints)};\n`; symbolsFileContent = `export default ${jsesc(symbols)};\n`; } else { - codePointsFileContent = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(codePoints)};\n`; - symbolsFileContent = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(symbols)};\n`; + codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(codePoints)};\n`; + symbolsFileContent = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(symbols)};\n`; } if ( (type === 'Case_Folding' && item === 'F') || @@ -264,7 +271,7 @@ const writeFiles = (options) => { } fs.mkdirSync(dir, { recursive: true }); // `Bidi_Mirroring_Glyph/index.mjs` - // Note: `Bidi_Mirroring_Glyph` doesn’t have repeated strings; don’t gzip. + // Note: `Bidi_Mirroring_Glyph` doesn’t have repeated strings; don’t compress. const output = [ `const chr = String.fromCodePoint;`, `const pair = (t, u, v) => [t ? u + v : v, chr(t ? u : u + v)];`, @@ -295,7 +302,7 @@ const writeFiles = (options) => { // or `bidi-brackets/index.mjs` // or `Names/index.mjs` const flatRuns = samePropertyRuns(auxMap[type]); - const output = `import { gunzipSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${gzipInline( + const output = `import { brotliDecompressSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${brotliInline( flatRuns, )});\n`; writeFileIfChanged(path.resolve(dir, 'index.mjs'), output); From 7fb6a67326cc3fb574d28243b9f6ddb7986514f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sat, 24 Oct 2026 20:11:11 -0400 Subject: [PATCH 2/7] perf: build regexes at runtime from ranges Each `regex.mjs` used to contain a regex literal produced by Regenerate, and for astral ranges that expands into long surrogate-pair alternations (470 KB per package, 190 KB of it surrogates). The data is already in `ranges.mjs`, so `regex.mjs` now imports it and calls the new `ranges-to-regex.mjs`, a port of Regenerate's serialization logic. The resulting `source` and `flags` are identical to the previous literals; a new test asserts this for every property against Regenerate itself, and the port was checked against all 19,610 regexes previously generated across the 35 Unicode versions. For Unicode 18.0.0 this shrinks the package from 590 KB to 444 KB packed (1.79 MB to 1.47 MB unpacked). Assisted-By: Claude Fable 5.1 --- index.mjs | 2 + scripts/utils.mjs | 2 +- static/ranges-to-regex.d.mts | 8 + static/ranges-to-regex.mjs | 635 +++++++++++++++++++++++++++++++++++ templates/README.md | 2 +- tests/tests.mjs | 41 +++ 6 files changed, 688 insertions(+), 2 deletions(-) create mode 100644 static/ranges-to-regex.d.mts create mode 100644 static/ranges-to-regex.mjs diff --git a/index.mjs b/index.mjs index 85660d8..94180d8 100644 --- a/index.mjs +++ b/index.mjs @@ -351,6 +351,8 @@ const generateData = async (version) => { 'decode-property-map.d.mts', 'decode-ranges.mjs', 'decode-ranges.d.mts', + 'ranges-to-regex.mjs', + 'ranges-to-regex.d.mts', ]; for (const file of staticFiles) { fs.copyFileSync( diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 4152812..bb28ec7 100644 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -202,7 +202,7 @@ const writeFiles = (options) => { ); writeFileIfChanged( path.resolve(dir, 'regex.mjs'), - 'export default /' + regenerate(codePoints).toString() + '/;\n', + `import ranges from './ranges.mjs';\nimport rangesToRegex from '../../ranges-to-regex.mjs';\n\nexport default rangesToRegex(ranges);\n`, ); writeFileIfChanged( path.resolve(dir, 'regex.d.mts'), diff --git a/static/ranges-to-regex.d.mts b/static/ranges-to-regex.d.mts new file mode 100644 index 0000000..89f03a6 --- /dev/null +++ b/static/ranges-to-regex.d.mts @@ -0,0 +1,8 @@ +import type { UnicodeRange } from './decode-ranges.mjs'; + +/** + * Build a regular expression that matches any symbol in `ranges`. + */ +declare function rangesToRegex(ranges: UnicodeRange[]): RegExp; + +export default rangesToRegex; diff --git a/static/ranges-to-regex.mjs b/static/ranges-to-regex.mjs new file mode 100644 index 0000000..22d15de --- /dev/null +++ b/static/ranges-to-regex.mjs @@ -0,0 +1,635 @@ +/** + * This file ships to end users. + * + * Builds a regular expression from the ranges produced by + * `decode-ranges.mjs`. This is a port of the serialization logic in + * Regenerate (MIT licensed), so the resulting + * `source` is byte-for-byte identical to `regenerate(set).toString()`. + * + * Throughout this file, “data” refers to Regenerate’s internal representation: + * a flat, sorted array of `[start, end)` pairs, e.g. `[0x41, 0x5B, 0x61, 0x7B]`. + */ + +const HIGH_SURROGATE_MIN = 0xd800; +const HIGH_SURROGATE_MAX = 0xdbff; +const LOW_SURROGATE_MIN = 0xdc00; +const LOW_SURROGATE_MAX = 0xdfff; + +const regexNull = /\\x00([^0123456789]|$)/g; + +const hex = (number) => number.toString(16).toUpperCase(); + +const pad = (string, totalCharacters) => { + return string.length < totalCharacters + ? ('0000' + string).slice(-totalCharacters) + : string; +}; + +const dataIsSingleton = (data) => { + // Check if the set only represents a single code point. + return data.length == 2 && data[0] + 1 == data[1]; +}; + +const dataAdd = (data, codePoint) => { + // Iterate over the data per `(start, end)` pair. + let index = 0; + let lastIndex = null; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1]; + + // Check if the code point is already in the set. + if (codePoint >= start && codePoint < end) { + return data; + } + + if (codePoint == start - 1) { + // Just replace `start` with a new value. + data[index] = codePoint; + return data; + } + + // At this point, if `start` is `greater` than `codePoint`, insert a new + // `[start, end]` pair before the current pair, or after the current pair + // if there is a known `lastIndex`. + if (start > codePoint) { + data.splice( + lastIndex != null ? lastIndex + 2 : 0, + 0, + codePoint, + codePoint + 1, + ); + return data; + } + + if (codePoint == end) { + // Check if adding this code point causes two separate ranges to become + // a single range, e.g. `dataAdd([0, 4, 5, 10], 4)` → `[0, 10]`. + if (codePoint + 1 == data[index + 2]) { + data.splice(index, 4, start, data[index + 3]); + return data; + } + // Else, just replace `end` with a new value. + data[index + 1] = codePoint + 1; + return data; + } + lastIndex = index; + index += 2; + } + // The loop has finished; add the new pair to the end of the data set. + data.push(codePoint, codePoint + 1); + return data; +}; + +const dataAddRange = (data, rangeStart, rangeEnd) => { + // Iterate over the data per `(start, end)` pair. + let index = 0; + let added = false; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1]; + + if (added) { + // The range has already been added to the set; at this point, we just + // need to get rid of the following ranges in case they overlap. + + // Check if this range can be combined with the previous range. + if (start == rangeEnd + 1) { + data.splice(index - 1, 2); + return data; + } + + // Exit as soon as no more possibly overlapping pairs can be found. + if (start > rangeEnd) { + return data; + } + + // E.g. `[0, 11, 12, 16]` and we’ve added 5-15, so we now have + // `[0, 16, 12, 16]`. Remove the `12,16` part, as it lies within the + // `0,16` range that was previously added. + if (start >= rangeStart && start <= rangeEnd) { + // `start` lies within the range that was previously added. + + if (end > rangeStart && end - 1 <= rangeEnd) { + // `end` lies within the range that was previously added as well, + // so remove this pair. + data.splice(index, 2); + index -= 2; + // Note: we cannot `return` just yet, as there may still be other + // overlapping pairs. + } else { + // `start` lies within the range that was previously added, but + // `end` doesn’t. E.g. `[0, 11, 12, 31]` and we’ve added 5-15, so + // now we have `[0, 16, 12, 31]`. This must be written as `[0, 31]`. + // Remove the previously added `end` and the current `start`. + data.splice(index - 1, 2); + index -= 2; + } + + // Note: we cannot return yet. + } + } else if (start == rangeEnd + 1 || start == rangeEnd) { + data[index] = rangeStart; + return data; + } + + // Check if a new pair must be inserted *before* the current one. + else if (start > rangeEnd) { + data.splice(index, 0, rangeStart, rangeEnd + 1); + return data; + } else if (rangeStart >= start && rangeStart < end && rangeEnd + 1 <= end) { + // The new range lies entirely within an existing range pair. No action + // needed. + return data; + } else if ( + // E.g. `[0, 11]` and you add 5-15 → `[0, 16]`. + (rangeStart >= start && rangeStart < end) || + // E.g. `[0, 3]` and you add 3-6 → `[0, 7]`. + end == rangeStart + ) { + // Replace `end` with the new value. + data[index + 1] = rangeEnd + 1; + // Make sure the next range pair doesn’t overlap, e.g. `[0, 11, 12, 14]` + // and you add 5-15 → `[0, 16]`, i.e. remove the `12,14` part. + added = true; + // Note: we cannot `return` just yet. + } else if (rangeStart <= start && rangeEnd + 1 >= end) { + // The new range is a superset of the old range. + data[index] = rangeStart; + data[index + 1] = rangeEnd + 1; + added = true; + } + + index += 2; + } + // The loop has finished without doing anything; add the new pair to the end + // of the data set. + if (!added) { + data.push(rangeStart, rangeEnd + 1); + } + return data; +}; + +// https://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae +const highSurrogate = (codePoint) => { + return Math.floor((codePoint - 0x10000) / 0x400) + HIGH_SURROGATE_MIN; +}; + +const lowSurrogate = (codePoint) => { + return ((codePoint - 0x10000) % 0x400) + LOW_SURROGATE_MIN; +}; + +const stringFromCharCode = String.fromCharCode; +const codePointToString = (codePoint) => { + // https://mathiasbynens.be/notes/javascript-escapes#single + // Note: the `\b` escape sequence for U+0008 BACKSPACE in strings has a + // different meaning in regular expressions (word boundary), so it cannot + // be used here. + if (codePoint == 0x09) { + return '\\t'; + } + // Note: IE < 9 treats `'\v'` as `'v'`, so avoid using it. + if (codePoint == 0x0a) { + return '\\n'; + } + if (codePoint == 0x0c) { + return '\\f'; + } + if (codePoint == 0x0d) { + return '\\r'; + } + if (codePoint == 0x2d) { + // https://mathiasbynens.be/notes/javascript-escapes#hexadecimal + // Note: `-` (U+002D HYPHEN-MINUS) is escaped in this way rather + // than by backslash-escaping, in case the output is used outside + // of a character class in a `u` RegExp. /\-/u throws, but + // /\x2D/u is fine. + return '\\x2D'; + } + if (codePoint == 0x5c) { + return '\\\\'; + } + if ( + codePoint == 0x24 || + (codePoint >= 0x28 && codePoint <= 0x2b) || + codePoint == 0x2e || + codePoint == 0x2f || + codePoint == 0x3f || + (codePoint >= 0x5b && codePoint <= 0x5e) || + (codePoint >= 0x7b && codePoint <= 0x7d) + ) { + // The code point maps to an unsafe printable ASCII character; + // backslash-escape it. Here’s the list of those symbols: + // + // $()*+./?[\]^{|} + // + // This matches SyntaxCharacters as well as `/` (U+002F SOLIDUS). + // https://tc39.github.io/ecma262/#prod-SyntaxCharacter + return '\\' + stringFromCharCode(codePoint); + } + if (codePoint >= 0x20 && codePoint <= 0x7e) { + // The code point maps to one of these printable ASCII symbols + // (including the space character): + // + // !"#%&',/0123456789:;<=>@ABCDEFGHIJKLMNO + // PQRSTUVWXYZ_`abcdefghijklmnopqrstuvwxyz~ + // + // These can safely be used directly. + return stringFromCharCode(codePoint); + } + if (codePoint <= 0xff) { + return '\\x' + pad(hex(codePoint), 2); + } + // `codePoint <= 0xFFFF` holds true. + // https://mathiasbynens.be/notes/javascript-escapes#unicode + return '\\u' + pad(hex(codePoint), 4); +}; + +const createBMPCharacterClasses = (data) => { + // Iterate over the data per `(start, end)` pair. + let result = ''; + let index = 0; + const length = data.length; + if (dataIsSingleton(data)) { + return codePointToString(data[0]); + } + while (index < length) { + const start = data[index]; + const end = data[index + 1] - 1; // Note: the `- 1` makes `end` inclusive. + if (start == end) { + result += codePointToString(start); + } else if (start + 1 == end) { + result += codePointToString(start) + codePointToString(end); + } else { + result += codePointToString(start) + '-' + codePointToString(end); + } + index += 2; + } + return '[' + result + ']'; +}; + +const splitAtBMP = (data) => { + // Iterate over the data per `(start, end)` pair. + const loneHighSurrogates = []; + const loneLowSurrogates = []; + const bmp = []; + const astral = []; + let index = 0; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1] - 1; // Note: the `- 1` makes `end` inclusive. + + if (start < HIGH_SURROGATE_MIN) { + // The range starts and ends before the high surrogate range. + // E.g. (0, 0x10). + if (end < HIGH_SURROGATE_MIN) { + bmp.push(start, end + 1); + } + + // The range starts before the high surrogate range and ends within it. + // E.g. (0, 0xD855). + if (end >= HIGH_SURROGATE_MIN && end <= HIGH_SURROGATE_MAX) { + bmp.push(start, HIGH_SURROGATE_MIN); + loneHighSurrogates.push(HIGH_SURROGATE_MIN, end + 1); + } + + // The range starts before the high surrogate range and ends in the low + // surrogate range. E.g. (0, 0xDCFF). + if (end >= LOW_SURROGATE_MIN && end <= LOW_SURROGATE_MAX) { + bmp.push(start, HIGH_SURROGATE_MIN); + loneHighSurrogates.push(HIGH_SURROGATE_MIN, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, end + 1); + } + + // The range starts before the high surrogate range and ends after the + // low surrogate range. E.g. (0, 0x10FFFF). + if (end > LOW_SURROGATE_MAX) { + bmp.push(start, HIGH_SURROGATE_MIN); + loneHighSurrogates.push(HIGH_SURROGATE_MIN, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, LOW_SURROGATE_MAX + 1); + if (end <= 0xffff) { + bmp.push(LOW_SURROGATE_MAX + 1, end + 1); + } else { + bmp.push(LOW_SURROGATE_MAX + 1, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } + } else if (start >= HIGH_SURROGATE_MIN && start <= HIGH_SURROGATE_MAX) { + // The range starts and ends in the high surrogate range. + // E.g. (0xD855, 0xD866). + if (end >= HIGH_SURROGATE_MIN && end <= HIGH_SURROGATE_MAX) { + loneHighSurrogates.push(start, end + 1); + } + + // The range starts in the high surrogate range and ends in the low + // surrogate range. E.g. (0xD855, 0xDCFF). + if (end >= LOW_SURROGATE_MIN && end <= LOW_SURROGATE_MAX) { + loneHighSurrogates.push(start, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, end + 1); + } + + // The range starts in the high surrogate range and ends after the low + // surrogate range. E.g. (0xD855, 0x10FFFF). + if (end > LOW_SURROGATE_MAX) { + loneHighSurrogates.push(start, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, LOW_SURROGATE_MAX + 1); + if (end <= 0xffff) { + bmp.push(LOW_SURROGATE_MAX + 1, end + 1); + } else { + bmp.push(LOW_SURROGATE_MAX + 1, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } + } else if (start >= LOW_SURROGATE_MIN && start <= LOW_SURROGATE_MAX) { + // The range starts and ends in the low surrogate range. + // E.g. (0xDCFF, 0xDDFF). + if (end >= LOW_SURROGATE_MIN && end <= LOW_SURROGATE_MAX) { + loneLowSurrogates.push(start, end + 1); + } + + // The range starts in the low surrogate range and ends after the low + // surrogate range. E.g. (0xDCFF, 0x10FFFF). + if (end > LOW_SURROGATE_MAX) { + loneLowSurrogates.push(start, LOW_SURROGATE_MAX + 1); + if (end <= 0xffff) { + bmp.push(LOW_SURROGATE_MAX + 1, end + 1); + } else { + bmp.push(LOW_SURROGATE_MAX + 1, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } + } else if (start > LOW_SURROGATE_MAX && start <= 0xffff) { + // The range starts and ends after the low surrogate range. + // E.g. (0xFFAA, 0x10FFFF). + if (end <= 0xffff) { + bmp.push(start, end + 1); + } else { + bmp.push(start, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } else { + // The range starts and ends in the astral range. + astral.push(start, end + 1); + } + + index += 2; + } + return { loneHighSurrogates, loneLowSurrogates, bmp, astral }; +}; + +const optimizeByLowSurrogates = (surrogateMappings) => { + if (surrogateMappings.length == 1) { + return surrogateMappings; + } + let index = -1; + let innerIndex = -1; + while (++index < surrogateMappings.length) { + const mapping = surrogateMappings[index]; + const lowSurrogates = mapping[1]; + const lowSurrogateStart = lowSurrogates[0]; + const lowSurrogateEnd = lowSurrogates[1]; + innerIndex = index; // Note: the loop starts at the next index. + while (++innerIndex < surrogateMappings.length) { + const otherMapping = surrogateMappings[innerIndex]; + const otherLowSurrogates = otherMapping[1]; + const otherLowSurrogateStart = otherLowSurrogates[0]; + const otherLowSurrogateEnd = otherLowSurrogates[1]; + if ( + lowSurrogateStart == otherLowSurrogateStart && + lowSurrogateEnd == otherLowSurrogateEnd && + otherLowSurrogates.length === 2 + ) { + // Add the code points in the other item to this one. + if (dataIsSingleton(otherMapping[0])) { + mapping[0] = dataAdd(mapping[0], otherMapping[0][0]); + } else { + mapping[0] = dataAddRange( + mapping[0], + otherMapping[0][0], + otherMapping[0][1] - 1, + ); + } + // Remove the other, now redundant, item. + surrogateMappings.splice(innerIndex, 1); + --innerIndex; + } + } + } + return surrogateMappings; +}; + +const optimizeSurrogateMappings = (surrogateMappings) => { + const result = []; + let tmpLow = []; + let addLow = false; + let index = -1; + const length = surrogateMappings.length; + while (++index < length) { + let mapping = surrogateMappings[index]; + let nextMapping = surrogateMappings[index + 1]; + if (!nextMapping) { + result.push(mapping); + continue; + } + let highSurrogates = mapping[0]; + let lowSurrogates = mapping[1]; + let nextHighSurrogates = nextMapping[0]; + let nextLowSurrogates = nextMapping[1]; + + // Check for identical high surrogate ranges. + tmpLow = lowSurrogates; + while ( + nextHighSurrogates && + highSurrogates[0] == nextHighSurrogates[0] && + highSurrogates[1] == nextHighSurrogates[1] + ) { + // Merge with the next item. + if (dataIsSingleton(nextLowSurrogates)) { + tmpLow = dataAdd(tmpLow, nextLowSurrogates[0]); + } else { + tmpLow = dataAddRange( + tmpLow, + nextLowSurrogates[0], + nextLowSurrogates[1] - 1, + ); + } + ++index; + mapping = surrogateMappings[index]; + highSurrogates = mapping[0]; + lowSurrogates = mapping[1]; + nextMapping = surrogateMappings[index + 1]; + nextHighSurrogates = nextMapping && nextMapping[0]; + nextLowSurrogates = nextMapping && nextMapping[1]; + addLow = true; + } + result.push([highSurrogates, addLow ? tmpLow : lowSurrogates]); + addLow = false; + } + return optimizeByLowSurrogates(result); +}; + +const surrogateSet = (data) => { + // Exit early if `data` is an empty set. + if (!data.length) { + return []; + } + + // Iterate over the data per `(start, end)` pair. + let index = 0; + const surrogateMappings = []; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1] - 1; + + const startHigh = highSurrogate(start); + const startLow = lowSurrogate(start); + const endHigh = highSurrogate(end); + const endLow = lowSurrogate(end); + + const startsWithLowestLowSurrogate = startLow == LOW_SURROGATE_MIN; + const endsWithHighestLowSurrogate = endLow == LOW_SURROGATE_MAX; + let complete = false; + + // Append the previous high-surrogate-to-low-surrogate mappings. + // Step 1: `(startHigh, startLow)` to `(startHigh, LOW_SURROGATE_MAX)`. + if ( + startHigh == endHigh || + (startsWithLowestLowSurrogate && endsWithHighestLowSurrogate) + ) { + surrogateMappings.push([ + [startHigh, endHigh + 1], + [startLow, endLow + 1], + ]); + complete = true; + } else { + surrogateMappings.push([ + [startHigh, startHigh + 1], + [startLow, LOW_SURROGATE_MAX + 1], + ]); + } + + // Step 2: `(startHigh + 1, LOW_SURROGATE_MIN)` to + // `(endHigh - 1, LOW_SURROGATE_MAX)`. + if (!complete && startHigh + 1 < endHigh) { + if (endsWithHighestLowSurrogate) { + // Combine step 2 and step 3. + surrogateMappings.push([ + [startHigh + 1, endHigh + 1], + [LOW_SURROGATE_MIN, endLow + 1], + ]); + complete = true; + } else { + surrogateMappings.push([ + [startHigh + 1, endHigh], + [LOW_SURROGATE_MIN, LOW_SURROGATE_MAX + 1], + ]); + } + } + + // Step 3. `(endHigh, LOW_SURROGATE_MIN)` to `(endHigh, endLow)`. + if (!complete) { + surrogateMappings.push([ + [endHigh, endHigh + 1], + [LOW_SURROGATE_MIN, endLow + 1], + ]); + } + + index += 2; + } + + // The format of `surrogateMappings` is as follows: + // + // [ surrogateMapping1, surrogateMapping2 ] + // + // i.e.: + // + // [ + // [ highSurrogates1, lowSurrogates1 ], + // [ highSurrogates2, lowSurrogates2 ] + // ] + return optimizeSurrogateMappings(surrogateMappings); +}; + +const createSurrogateCharacterClasses = (surrogateMappings) => { + const result = []; + for (const surrogateMapping of surrogateMappings) { + const highSurrogates = surrogateMapping[0]; + const lowSurrogates = surrogateMapping[1]; + result.push( + createBMPCharacterClasses(highSurrogates) + + createBMPCharacterClasses(lowSurrogates), + ); + } + return result.join('|'); +}; + +const createCharacterClassesFromData = (data) => { + const result = []; + + const { loneHighSurrogates, loneLowSurrogates, bmp, astral } = + splitAtBMP(data); + + const surrogateMappings = surrogateSet(astral); + + if (bmp.length) { + // The data set contains BMP code points that are not high surrogates + // needed for astral code points in the set. + result.push(createBMPCharacterClasses(bmp)); + } + if (surrogateMappings.length) { + // The data set contains astral code points; append character classes + // based on their surrogate pairs. + result.push(createSurrogateCharacterClasses(surrogateMappings)); + } + // https://gist.github.com/mathiasbynens/bbe7f870208abcfec860 + if (loneHighSurrogates.length) { + result.push( + createBMPCharacterClasses(loneHighSurrogates) + + // Make sure the high surrogates aren’t part of a surrogate pair. + '(?![\\uDC00-\\uDFFF])', + ); + } + if (loneLowSurrogates.length) { + result.push( + // It is not possible to accurately assert the low surrogates aren’t + // part of a surrogate pair, since JavaScript regular expressions do + // not support lookbehind. + '(?:[^\\uD800-\\uDBFF]|^)' + createBMPCharacterClasses(loneLowSurrogates), + ); + } + return result.join('|'); +}; + +/** + * Serialize `data` the same way `regenerate(set).toString()` would. + * @param {number[]} data + */ +const dataToRegexSource = (data) => { + const result = createCharacterClassesFromData(data); + if (!result) { + // For an empty set, return something that can be inserted `/here/` to + // form a valid regular expression. Avoid `(?:)` since that matches the + // empty string. + return '[]'; + } + // Use `\0` instead of `\x00` where possible. + return result.replace(regexNull, '\\0$1'); +}; + +/** + * Build a regular expression that matches any symbol in `ranges`. + * @param {Array<{ begin: number, end: number }>} ranges + */ +const rangesToRegex = (ranges) => { + const data = []; + for (const range of ranges) { + data.push(range.begin, range.end); + } + return new RegExp(dataToRegexSource(data)); +}; + +export default rangesToRegex; diff --git a/templates/README.md b/templates/README.md index 12baa3f..1c49647 100644 --- a/templates/README.md +++ b/templates/README.md @@ -14,7 +14,7 @@ npm install @unicode/unicode-<%= version %> --save-dev ## Regular expressions -The Unicode data modules ship with pre-compiled regular expressions for categories, scripts, script extensions, blocks, and properties. But maybe you want to create a single regular expression that combines several categories, scripts, etc. In that case, [***you should use Regenerate***](https://mths.be/regenerate). For example, to construct a regex that matches all symbols in the Arabic and Greek scripts as per Unicode v6.3.0: +The Unicode data modules ship with regular expressions for categories, scripts, script extensions, blocks, and properties. But maybe you want to create a single regular expression that combines several categories, scripts, etc. In that case, [***you should use Regenerate***](https://mths.be/regenerate). For example, to construct a regex that matches all symbols in the Arabic and Greek scripts as per Unicode v6.3.0: ```js import regenerate from 'regenerate'; diff --git a/tests/tests.mjs b/tests/tests.mjs index 248492d..64b65cf 100644 --- a/tests/tests.mjs +++ b/tests/tests.mjs @@ -1,6 +1,12 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { suite, test } from 'node:test'; +import regenerate from 'regenerate'; import resources from '../data/resources.mjs'; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const oldest = resources[0].version; const newest = resources.at(-1).version; @@ -11,6 +17,35 @@ if (!process.env.CI) { await generateData(newest); } +// Every `regex.mjs` is built at runtime from `ranges.mjs`; its source must be +// identical to what Regenerate produces for the same set of code points. +const assertRegexesMatchRegenerate = async (t, version) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${version}`); + let count = 0; + for (const type of fs.readdirSync(root)) { + const typeDir = path.join(root, type); + if (type.startsWith('.') || !fs.statSync(typeDir).isDirectory()) { + continue; + } + for (const name of fs.readdirSync(typeDir)) { + const dir = path.join(typeDir, name); + if (!fs.existsSync(path.join(dir, 'regex.mjs'))) { + continue; + } + const { default: ranges } = await import(path.join(dir, 'ranges.mjs')); + const { default: regex } = await import(path.join(dir, 'regex.mjs')); + const set = regenerate(); + for (const range of ranges) { + set.addRange(range.begin, range.end - 1); + } + t.assert.strictEqual(regex.source, set.toString(), `${type}/${name}`); + t.assert.strictEqual(regex.flags, '', `${type}/${name}`); + count++; + } + } + t.assert.ok(count > 0, 'found regex.mjs files'); +}; + suite(`The generated Unicode ${oldest} js`, () => { test('General_Category/Modifier_Letter should match the snapshot', async (t) => { const { default: data } = await import( @@ -30,6 +65,9 @@ suite(`The generated Unicode ${oldest} js`, () => { ); t.assert.snapshot(data); }); + test('every regex.mjs should match Regenerate output', async (t) => { + await assertRegexesMatchRegenerate(t, oldest); + }); }); suite(`The generated latest Unicode js`, () => { @@ -106,4 +144,7 @@ suite(`The generated latest Unicode js`, () => { ); t.assert.snapshot([...map.entries()]); }); + test('every regex.mjs should match Regenerate output', async (t) => { + await assertRegexesMatchRegenerate(t, newest); + }); }); From 4b181116085e14108af224f99adb504d7ecc821d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sat, 24 Oct 2026 20:12:37 -0400 Subject: [PATCH 3/7] perf: derive mapping symbols.mjs from code-points.mjs For Case_Folding, Simple_Case_Mapping, Special_Casing and Decomposition_Mapping, `symbols.mjs` shipped a second compressed copy of the same data as `code-points.mjs`, differing only in that every code point was replaced by its string. It now imports `code-points.mjs` and derives the map through the new `mapping-to-symbols.mjs`. Small mappings (fewer than 10 entries) keep their inline literals. The resulting maps are identical, including entry order; a new test checks this for every mapping directory. For Unicode 18.0.0 this shrinks the package from 444 KB to 410 KB packed (1.47 MB to 1.43 MB unpacked). Assisted-By: Claude Fable 5.1 --- index.mjs | 2 ++ scripts/utils.mjs | 24 +++++++++++----------- static/mapping-to-symbols.d.mts | 9 +++++++++ static/mapping-to-symbols.mjs | 25 +++++++++++++++++++++++ tests/tests.mjs | 35 +++++++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 static/mapping-to-symbols.d.mts create mode 100644 static/mapping-to-symbols.mjs diff --git a/index.mjs b/index.mjs index 94180d8..1488c91 100644 --- a/index.mjs +++ b/index.mjs @@ -353,6 +353,8 @@ const generateData = async (version) => { 'decode-ranges.d.mts', 'ranges-to-regex.mjs', 'ranges-to-regex.d.mts', + 'mapping-to-symbols.mjs', + 'mapping-to-symbols.d.mts', ]; for (const file of staticFiles) { fs.copyFileSync( diff --git a/scripts/utils.mjs b/scripts/utils.mjs index bb28ec7..ad716f0 100644 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -215,22 +215,24 @@ const writeFiles = (options) => { symbolsFileContent = `export default ${jsesc(codePointsAsArray.map((cp) => String.fromCodePoint(cp)))};\n`; } } else { - const symbols = new Map(); - for (let [from, to] of codePoints) { - from = String.fromCodePoint(from); - if (Array.isArray(to)) { - to = String.fromCodePoint.apply(null, to); - } else { - to = String.fromCodePoint(to); - } - symbols.set(from, to); - } if (codePoints.size < 10) { + const symbols = new Map(); + for (let [from, to] of codePoints) { + from = String.fromCodePoint(from); + if (Array.isArray(to)) { + to = String.fromCodePoint.apply(null, to); + } else { + to = String.fromCodePoint(to); + } + symbols.set(from, to); + } codePointsFileContent = `export default ${jsesc(codePoints)};\n`; symbolsFileContent = `export default ${jsesc(symbols)};\n`; } else { codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(codePoints)};\n`; - symbolsFileContent = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(symbols)};\n`; + // The symbols map is derived from the code points map at runtime + // rather than shipped as a second copy of the same data. + symbolsFileContent = `import codePoints from './code-points.mjs';\nimport mappingToSymbols from '../../mapping-to-symbols.mjs';\n\nexport default mappingToSymbols(codePoints);\n`; } if ( (type === 'Case_Folding' && item === 'F') || diff --git a/static/mapping-to-symbols.d.mts b/static/mapping-to-symbols.d.mts new file mode 100644 index 0000000..9dc555c --- /dev/null +++ b/static/mapping-to-symbols.d.mts @@ -0,0 +1,9 @@ +/** + * Derive the symbol-to-symbol(s) map of a case mapping, case folding, or + * decomposition mapping from its code-point-to-code-point(s) map. + */ +declare function mappingToSymbols( + codePoints: Map, +): Map; + +export default mappingToSymbols; diff --git a/static/mapping-to-symbols.mjs b/static/mapping-to-symbols.mjs new file mode 100644 index 0000000..616cfad --- /dev/null +++ b/static/mapping-to-symbols.mjs @@ -0,0 +1,25 @@ +/** + * This file ships to end users. + * + * Derive the symbol-to-symbol(s) map of a case mapping, case folding, or + * decomposition mapping from its code-point-to-code-point(s) map. + */ + +const fromCodePoint = String.fromCodePoint; + +/** + * @param {Map} codePoints + * @returns {Map} + */ +const mappingToSymbols = (codePoints) => { + const symbols = new Map(); + for (const [from, to] of codePoints) { + symbols.set( + fromCodePoint(from), + Array.isArray(to) ? fromCodePoint(...to) : fromCodePoint(to), + ); + } + return symbols; +}; + +export default mappingToSymbols; diff --git a/tests/tests.mjs b/tests/tests.mjs index 64b65cf..49dda46 100644 --- a/tests/tests.mjs +++ b/tests/tests.mjs @@ -46,6 +46,38 @@ const assertRegexesMatchRegenerate = async (t, version) => { t.assert.ok(count > 0, 'found regex.mjs files'); }; +// Every mapping `symbols.mjs` must equal its `code-points.mjs` with each code +// point replaced by the corresponding symbol. +const assertSymbolsMatchCodePoints = async (t, version) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${version}`); + let count = 0; + for (const type of [ + 'Case_Folding', + 'Simple_Case_Mapping', + 'Special_Casing', + 'Decomposition_Mapping', + ]) { + const typeDir = path.join(root, type); + if (!fs.existsSync(typeDir)) { + continue; + } + for (const name of fs.readdirSync(typeDir)) { + const dir = path.join(typeDir, name); + const { default: codePoints } = await import( + path.join(dir, 'code-points.mjs') + ); + const { default: symbols } = await import(path.join(dir, 'symbols.mjs')); + const expected = [...codePoints].map(([from, to]) => [ + String.fromCodePoint(from), + String.fromCodePoint(...(Array.isArray(to) ? to : [to])), + ]); + t.assert.deepStrictEqual([...symbols], expected, `${type}/${name}`); + count++; + } + } + t.assert.ok(count > 0, 'found mapping directories'); +}; + suite(`The generated Unicode ${oldest} js`, () => { test('General_Category/Modifier_Letter should match the snapshot', async (t) => { const { default: data } = await import( @@ -147,4 +179,7 @@ suite(`The generated latest Unicode js`, () => { test('every regex.mjs should match Regenerate output', async (t) => { await assertRegexesMatchRegenerate(t, newest); }); + test('every mapping symbols.mjs should match code-points.mjs', async (t) => { + await assertSymbolsMatchCodePoints(t, newest); + }); }); From 86b542efb39d6559595cefffbbcb2680de8cbcc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sat, 24 Oct 2026 20:17:54 -0400 Subject: [PATCH 4/7] perf: front-code the Names table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Names/index.mjs` stored every character name as a JSON string inside the run-length array shared with the other property maps. Names are sorted by code point and neighbouring names share long prefixes (e.g. `LATIN SMALL LETTER A WITH …`), so the table is now written as text: a first line with the `[gap, length]` run pairs as JSON, followed by one front-coded name per line (a base64 digit for the number of characters shared with the previous name, then the remainder). The new `decode-names.mjs` rebuilds the same `Map`, with the same entries in the same order; this was verified against all 35 versions, and a new test checks representative lookups. For Unicode 18.0.0 `Names/index.mjs` shrinks from 158 KB to 123 KB, and the package from 410 KB to 390 KB packed (1.43 MB to 1.39 MB unpacked). Assisted-By: Claude Fable 5.1 --- index.mjs | 2 ++ scripts/encode-ranges.mjs | 2 +- scripts/utils.mjs | 58 +++++++++++++++++++++++++++++++-------- static/decode-names.d.mts | 7 +++++ static/decode-names.mjs | 40 +++++++++++++++++++++++++++ tests/tests.mjs | 22 +++++++++++++++ 6 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 static/decode-names.d.mts create mode 100644 static/decode-names.mjs diff --git a/index.mjs b/index.mjs index 1488c91..7040888 100644 --- a/index.mjs +++ b/index.mjs @@ -355,6 +355,8 @@ const generateData = async (version) => { 'ranges-to-regex.d.mts', 'mapping-to-symbols.mjs', 'mapping-to-symbols.d.mts', + 'decode-names.mjs', + 'decode-names.d.mts', ]; for (const file of staticFiles) { fs.copyFileSync( diff --git a/scripts/encode-ranges.mjs b/scripts/encode-ranges.mjs index 8700d04..fc63021 100644 --- a/scripts/encode-ranges.mjs +++ b/scripts/encode-ranges.mjs @@ -57,4 +57,4 @@ const encodeRegenerate = (regenerateSet) => { return encodeDeltas(deltas); }; -export { encodeRanges, encodeRegenerate }; +export { base64enc, encodeRanges, encodeRegenerate }; diff --git a/scripts/utils.mjs b/scripts/utils.mjs index ad716f0..790d183 100644 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -4,25 +4,30 @@ import { fileURLToPath } from 'node:url'; import zlib from 'node:zlib'; import jsesc from 'jsesc'; import regenerate from 'regenerate'; -import { encodeRanges, encodeRegenerate } from './encode-ranges.mjs'; +import { base64enc, encodeRanges, encodeRegenerate } from './encode-ranges.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const brotliInline = (data) => { - if (data instanceof Map) { - return `new Map(${brotliInline([...data])})`; - } - const json = JSON.stringify(data); - const brotliBuffer = zlib.brotliCompressSync(json, { +// Brotli-compress `text` and return a JS expression that decompresses it into +// a Buffer at runtime. +const brotliInlineBuffer = (text) => { + const brotliBuffer = zlib.brotliCompressSync(text, { params: { [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, [zlib.constants.BROTLI_PARAM_LGWIN]: zlib.constants.BROTLI_MAX_WINDOW_BITS, - [zlib.constants.BROTLI_PARAM_SIZE_HINT]: Buffer.byteLength(json), + [zlib.constants.BROTLI_PARAM_SIZE_HINT]: Buffer.byteLength(text), }, }); const str = brotliBuffer.toString('base64'); - return `JSON.parse(brotliDecompressSync(Buffer.from('${str}', 'base64')))`; + return `brotliDecompressSync(Buffer.from('${str}', 'base64'))`; +}; + +const brotliInline = (data) => { + if (data instanceof Map) { + return `new Map(${brotliInline([...data])})`; + } + return `JSON.parse(${brotliInlineBuffer(JSON.stringify(data))})`; }; const range = (start, stop) => { @@ -83,6 +88,30 @@ const samePropertyRuns = (codePointProperties) => { return result; }; +// Front-code the names in the flat runs produced by `samePropertyRuns`. +// See static/decode-names.mjs for the format and the decoder. +const encodeNames = (flatRuns) => { + const runs = []; + const lines = []; + let previous = ''; + for (let i = 0; i < flatRuns.length; i += 3) { + runs.push(flatRuns[i], flatRuns[i + 1]); + const name = flatRuns[i + 2]; + let shared = 0; + while ( + shared < 63 && + shared < name.length && + shared < previous.length && + name.charCodeAt(shared) === previous.charCodeAt(shared) + ) { + shared++; + } + lines.push(base64enc[shared] + name.slice(shared)); + previous = name; + } + return JSON.stringify(runs) + '\n' + lines.join('\n'); +}; + // Only write the file if its content has changed. // Reading files is cheaper than writing files, and most files don’t change between builds. const writeFileIfChanged = (filePath, content) => { @@ -304,9 +333,14 @@ const writeFiles = (options) => { // or `bidi-brackets/index.mjs` // or `Names/index.mjs` const flatRuns = samePropertyRuns(auxMap[type]); - const output = `import { brotliDecompressSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${brotliInline( - flatRuns, - )});\n`; + const output = + type === 'Names' + ? `import { brotliDecompressSync } from 'node:zlib';\nimport decodeNames from '../decode-names.mjs';\n\nexport default decodeNames(${brotliInlineBuffer( + encodeNames(flatRuns), + )}.toString());\n` + : `import { brotliDecompressSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${brotliInline( + flatRuns, + )});\n`; writeFileIfChanged(path.resolve(dir, 'index.mjs'), output); writeFileIfChanged( path.resolve(dir, 'index.d.mts'), diff --git a/static/decode-names.d.mts b/static/decode-names.d.mts new file mode 100644 index 0000000..54d1d83 --- /dev/null +++ b/static/decode-names.d.mts @@ -0,0 +1,7 @@ +/** + * Decode a front-coded table of character names into a map from code point + * to name. + */ +declare function decodeNames(input: string): Map; + +export default decodeNames; diff --git a/static/decode-names.mjs b/static/decode-names.mjs new file mode 100644 index 0000000..b5f3946 --- /dev/null +++ b/static/decode-names.mjs @@ -0,0 +1,40 @@ +/** + * This file ships to end users. + * See `encodeNames` in scripts/utils.mjs for the encoder. + */ + +const base64enc = + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; + +/** + * Decode a front-coded table of character names into a map from code point + * to name. + * + * The first line of `input` is a JSON array of `[gap, length]` pairs, one per + * name, describing the run of code points each name applies to: `gap` is the + * number of unnamed code points since the end of the previous run, and + * `length` is the number of consecutive code points sharing the name. + * + * Every following line holds one name, front-coded against the previous + * name: the first character is the base64 digit for the number of leading + * characters shared with the previous name, and the rest is the remainder. + * @param {string} input + */ +const decodeNames = (input) => { + const lines = input.split('\n'); + const runs = JSON.parse(lines[0]); + const map = new Map(); + let previous = ''; + for (let codePoint = 0, i = 1, j = 0; i < lines.length; i++, j += 2) { + const line = lines[i]; + const name = previous.slice(0, base64enc.indexOf(line[0])) + line.slice(1); + previous = name; + codePoint += runs[j]; + for (const end = codePoint + runs[j + 1]; codePoint < end; codePoint++) { + map.set(codePoint, name); + } + } + return map; +}; + +export default decodeNames; diff --git a/tests/tests.mjs b/tests/tests.mjs index 49dda46..8441270 100644 --- a/tests/tests.mjs +++ b/tests/tests.mjs @@ -179,6 +179,28 @@ suite(`The generated latest Unicode js`, () => { test('every regex.mjs should match Regenerate output', async (t) => { await assertRegexesMatchRegenerate(t, newest); }); + test('Names should decode to the expected names', async (t) => { + const { default: names } = await import( + `../output/unicode-${newest}/Names/index.mjs` + ); + const entries = [...names]; + t.assert.deepStrictEqual(entries[0], [0, '']); + t.assert.strictEqual(names.get(0x41), 'LATIN CAPITAL LETTER A'); + t.assert.strictEqual(names.get(0x4e00), 'CJK Ideograph'); + t.assert.strictEqual(names.get(0xac00), 'Hangul Syllable'); + t.assert.strictEqual(names.get(0xf900), 'CJK COMPATIBILITY IDEOGRAPH-F900'); + t.assert.strictEqual(names.get(0x1f600), 'GRINNING FACE'); + t.assert.deepStrictEqual(entries.at(-1), [ + 0x10fffd, + 'Plane 16 Private Use', + ]); + // Entries are in code point order. + for (let i = 1; i < entries.length; i++) { + if (entries[i][0] <= entries[i - 1][0]) { + t.assert.fail(`entry ${i} out of order`); + } + } + }); test('every mapping symbols.mjs should match code-points.mjs', async (t) => { await assertSymbolsMatchCodePoints(t, newest); }); From 0086ab837b6fdb9c9414f39d595ad1ca17eafd60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sat, 24 Oct 2026 20:23:12 -0400 Subject: [PATCH 5/7] perf: delta-encode case and decomposition mappings The `code-points.mjs` of Case_Folding, Simple_Case_Mapping, Special_Casing and Decomposition_Mapping stored the map as compressed JSON `[from, to]` pairs. Mapped code points are almost always close to their keys, so the map is now written as a stream of variable-length integers using the same base64 scheme as `ranges.mjs`: per entry, the zigzag-encoded difference between its key and the previous key, a count, and the zigzag-encoded differences between consecutive mapped code points. A count of 0 marks an entry mapping to a single code point, so files mixing scalar and array values (e.g. Unicode 3.0.1 Case_Folding/E) round-trip exactly. The stream is brotli-compressed as before and decoded by the new `decode-mapping.mjs`; `decode-ranges.mjs` now also exports `decodeDeltas` for it. All 1,699 maps across the 35 versions are identical to the previous output, entry order included. For Unicode 18.0.0, `Simple_Case_Mapping/Uppercase/code-points.mjs` shrinks from 4.1 KB to 1.1 KB, and the package from 390 KB to 378 KB packed (1.39 MB to 1.38 MB unpacked). Assisted-By: Claude Fable 5.1 --- index.mjs | 2 ++ scripts/encode-ranges.mjs | 2 +- scripts/utils.mjs | 40 ++++++++++++++++++++++++++++++-- static/decode-mapping.d.mts | 7 ++++++ static/decode-mapping.mjs | 46 +++++++++++++++++++++++++++++++++++++ static/decode-ranges.d.mts | 6 +++++ static/decode-ranges.mjs | 1 + 7 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 static/decode-mapping.d.mts create mode 100644 static/decode-mapping.mjs diff --git a/index.mjs b/index.mjs index 7040888..78d9fc6 100644 --- a/index.mjs +++ b/index.mjs @@ -357,6 +357,8 @@ const generateData = async (version) => { 'mapping-to-symbols.d.mts', 'decode-names.mjs', 'decode-names.d.mts', + 'decode-mapping.mjs', + 'decode-mapping.d.mts', ]; for (const file of staticFiles) { fs.copyFileSync( diff --git a/scripts/encode-ranges.mjs b/scripts/encode-ranges.mjs index fc63021..0524214 100644 --- a/scripts/encode-ranges.mjs +++ b/scripts/encode-ranges.mjs @@ -57,4 +57,4 @@ const encodeRegenerate = (regenerateSet) => { return encodeDeltas(deltas); }; -export { base64enc, encodeRanges, encodeRegenerate }; +export { base64enc, encodeDeltas, encodeRanges, encodeRegenerate }; diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 790d183..2bb3f94 100644 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -4,7 +4,12 @@ import { fileURLToPath } from 'node:url'; import zlib from 'node:zlib'; import jsesc from 'jsesc'; import regenerate from 'regenerate'; -import { base64enc, encodeRanges, encodeRegenerate } from './encode-ranges.mjs'; +import { + base64enc, + encodeDeltas, + encodeRanges, + encodeRegenerate, +} from './encode-ranges.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -112,6 +117,35 @@ const encodeNames = (flatRuns) => { return JSON.stringify(runs) + '\n' + lines.join('\n'); }; +// Zigzag encoding maps signed integers to unsigned ones: 0 → 0, -1 → 1, 1 → 2, +// -2 → 3, … so that small differences in either direction stay small. +const zigzag = (value) => ((value << 1) ^ (value >> 31)) >>> 0; + +// Delta-encode a mapping from code points to code point(s), e.g. a case +// mapping or a decomposition mapping, preserving entry order and whether +// each entry maps to a single code point or to an array. +// See static/decode-mapping.mjs for the format and the decoder. +const encodeMapping = (codePoints) => { + const output = []; + let previousFrom = 0; + for (const [from, to] of codePoints) { + output.push(zigzag(from - previousFrom)); + previousFrom = from; + let previous = from; + if (Array.isArray(to)) { + console.assert(to.length > 0, `empty mapping for U+${from.toString(16)}`); + output.push(to.length); + for (const codePoint of to) { + output.push(zigzag(codePoint - previous)); + previous = codePoint; + } + } else { + output.push(0, zigzag(to - previous)); + } + } + return encodeDeltas(output); +}; + // Only write the file if its content has changed. // Reading files is cheaper than writing files, and most files don’t change between builds. const writeFileIfChanged = (filePath, content) => { @@ -258,7 +292,9 @@ const writeFiles = (options) => { codePointsFileContent = `export default ${jsesc(codePoints)};\n`; symbolsFileContent = `export default ${jsesc(symbols)};\n`; } else { - codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(codePoints)};\n`; + codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\nimport decodeMapping from '../../decode-mapping.mjs';\n\nexport default decodeMapping(${brotliInlineBuffer( + encodeMapping(codePoints), + )}.toString());\n`; // The symbols map is derived from the code points map at runtime // rather than shipped as a second copy of the same data. symbolsFileContent = `import codePoints from './code-points.mjs';\nimport mappingToSymbols from '../../mapping-to-symbols.mjs';\n\nexport default mappingToSymbols(codePoints);\n`; diff --git a/static/decode-mapping.d.mts b/static/decode-mapping.d.mts new file mode 100644 index 0000000..57d9033 --- /dev/null +++ b/static/decode-mapping.d.mts @@ -0,0 +1,7 @@ +/** + * Decode a delta-encoded mapping from code points to code point(s), e.g. a + * case mapping or a decomposition mapping. + */ +declare function decodeMapping(input: string): Map; + +export default decodeMapping; diff --git a/static/decode-mapping.mjs b/static/decode-mapping.mjs new file mode 100644 index 0000000..c54cbac --- /dev/null +++ b/static/decode-mapping.mjs @@ -0,0 +1,46 @@ +/** + * This file ships to end users. + * See `encodeMapping` in scripts/utils.mjs for the encoder. + */ + +import { decodeDeltas } from './decode-ranges.mjs'; + +// Undo zigzag encoding: 0 → 0, 1 → -1, 2 → 1, 3 → -2, … +const unzigzag = (value) => (value >>> 1) ^ -(value & 1); + +/** + * Decode a delta-encoded mapping from code points to code point(s), e.g. a + * case mapping or a decomposition mapping. + * + * `input` is a base64 stream of variable-length integers (see + * `decodeDeltas`). Each entry consists of the zigzag-encoded difference + * between its key and the previous key, a count, and then `count` values + * (or one value, if `count` is 0). Each value is the zigzag-encoded difference + * from the previous code point in the entry, starting from the key. A count + * of 0 means the entry maps to a single code point rather than to an array. + * @param {string} input + * @returns {Map} + */ +const decodeMapping = (input) => { + const deltas = decodeDeltas(input); + const map = new Map(); + for (let from = 0, i = 0; i < deltas.length;) { + from += unzigzag(deltas[i++]); + const count = deltas[i++]; + let codePoint = from; + if (count === 0) { + codePoint += unzigzag(deltas[i++]); + map.set(from, codePoint); + } else { + const codePoints = []; + for (let j = 0; j < count; j++) { + codePoint += unzigzag(deltas[i++]); + codePoints.push(codePoint); + } + map.set(from, codePoints); + } + } + return map; +}; + +export default decodeMapping; diff --git a/static/decode-ranges.d.mts b/static/decode-ranges.d.mts index 4cfa706..58c30b1 100644 --- a/static/decode-ranges.d.mts +++ b/static/decode-ranges.d.mts @@ -17,4 +17,10 @@ declare namespace decodeRanges { */ declare function decodeRanges(input: string): decodeRanges.UnicodeRange[]; +/** + * Base64 decode variable-length deltas (5/10/15/21-bit). + */ +declare function decodeDeltas(input: string): number[]; + +export { decodeDeltas }; export default decodeRanges; diff --git a/static/decode-ranges.mjs b/static/decode-ranges.mjs index 7e908c0..7463e8d 100644 --- a/static/decode-ranges.mjs +++ b/static/decode-ranges.mjs @@ -82,4 +82,5 @@ const decodeRanges = (input) => { return ranges; }; +export { decodeDeltas }; export default decodeRanges; From 0506410b270c2c2751dc254ed63a80a973f43f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sat, 24 Oct 2026 20:37:29 -0400 Subject: [PATCH 6/7] perf: share type declarations via typesVersions Every property directory shipped four identical declaration files (`ranges.d.mts`, `regex.d.mts`, `code-points.d.mts`, `symbols.d.mts`), about 4,000 files per package. They are now written once under `types/` and selected through a generated `typesVersions` map in package.json. Mapping directories (Case_Folding, Simple_Case_Mapping, Special_Casing, Decomposition_Mapping) keep their own `code-points`/`symbols` declarations, since their value types differ per directory; more specific patterns route them there. The build removes per-directory declarations left over from earlier runs. `decode-ranges.d.mts` now exports the `UnicodeRange` type at module level; previously `ranges.d.mts` imported a name that was only declared inside a namespace, so TypeScript consumers saw an error. The README now documents TypeScript 4.7 as the minimum, which is what `.d.mts` requires. Verified with TypeScript 5.9 (node16, bundler and node10 resolution) and TypeScript 4.7: imports resolve to the same types as before. A new test checks the declaration layout and the typesVersions targets. For Unicode 18.0.0 this shrinks the package from 378 KB to 312 KB packed (1.38 MB to 1.08 MB unpacked) and from 8,352 to 4,320 files. Assisted-By: Claude Fable 5.1 --- index.mjs | 11 +++++- scripts/utils.mjs | 72 ++++++++++++++++++++++++---------- static/decode-ranges.d.mts | 3 ++ static/types/code-points.d.mts | 2 + static/types/ranges.d.mts | 4 ++ static/types/regex.d.mts | 2 + static/types/symbols.d.mts | 2 + templates/README.md | 2 + templates/package.json | 3 +- tests/tests.mjs | 41 +++++++++++++++++++ 10 files changed, 119 insertions(+), 23 deletions(-) create mode 100644 static/types/code-points.d.mts create mode 100644 static/types/ranges.d.mts create mode 100644 static/types/regex.d.mts create mode 100644 static/types/symbols.d.mts diff --git a/index.mjs b/index.mjs index 78d9fc6..453f50a 100644 --- a/index.mjs +++ b/index.mjs @@ -334,7 +334,12 @@ const generateData = async (version) => { ); fs.writeFileSync( path.resolve(__dirname, `output/unicode-${version}/package.json`), - compilePackage({ version: version }), + compilePackage({ + version: version, + typesVersions: JSON.stringify(utils.typesVersions(dirMap), null, '\t') + // Indent to match the surrounding JSON. + .replace(/\n/g, '\n\t'), + }), ); fs.mkdirSync( path.resolve(__dirname, `output/unicode-${version}/.github/workflows`), @@ -342,6 +347,9 @@ const generateData = async (version) => { recursive: true, }, ); + fs.mkdirSync(path.resolve(__dirname, `output/unicode-${version}/types`), { + recursive: true, + }); const staticFiles = [ '.github/workflows/publish-on-tag.yml', '.gitattributes', @@ -359,6 +367,7 @@ const generateData = async (version) => { 'decode-names.d.mts', 'decode-mapping.mjs', 'decode-mapping.d.mts', + ...utils.SHARED_DECLARATIONS.map((file) => `types/${file}.d.mts`), ]; for (const file of staticFiles) { fs.copyFileSync( diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 2bb3f94..5727001 100644 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -35,6 +35,18 @@ const brotliInline = (data) => { return `JSON.parse(${brotliInlineBuffer(JSON.stringify(data))})`; }; +// Modules whose declarations are shared via `types/*.d.mts` and +// `typesVersions` instead of being repeated in every directory. +const SHARED_DECLARATIONS = ['ranges', 'regex', 'code-points', 'symbols']; + +// Directories under these types declare their own `code-points`/`symbols` types. +const MAPPING_TYPES = [ + 'Case_Folding', + 'Simple_Case_Mapping', + 'Special_Casing', + 'Decomposition_Mapping', +]; + const range = (start, stop) => { // Inclusive, e.g. `range(1, 3)` → `[1, 2, 3]`. const result = []; @@ -183,11 +195,7 @@ const writeFiles = (options) => { const codePoints = map[item]; const type = typeof options.type === 'function' ? options.type(item) : options.type; - const isCodePointMapping = - type === 'Case_Folding' || - type === 'Simple_Case_Mapping' || - type === 'Special_Casing' || - type === 'Decomposition_Mapping'; + const isCodePointMapping = MAPPING_TYPES.includes(type); const isNamesCanon = type === 'Names' && !subType; const isNameAliases = type === 'Names' && subType === 'name-aliases'; const subdir = isNameAliases @@ -259,18 +267,10 @@ const writeFiles = (options) => { path.resolve(dir, 'ranges.mjs'), `import decodeRanges from '../../decode-ranges.mjs';\n\nexport default decodeRanges('${encodedRanges}');\n`, ); - writeFileIfChanged( - path.resolve(dir, 'ranges.d.mts'), - 'import type { UnicodeRange } from "../../decode-ranges.mjs";\n\ndeclare const ranges: UnicodeRange[];\nexport default ranges;\n', - ); writeFileIfChanged( path.resolve(dir, 'regex.mjs'), `import ranges from './ranges.mjs';\nimport rangesToRegex from '../../ranges-to-regex.mjs';\n\nexport default rangesToRegex(ranges);\n`, ); - writeFileIfChanged( - path.resolve(dir, 'regex.d.mts'), - 'declare const regex: RegExp;\nexport default regex;\n', - ); if (codePointsSizeLt(codePoints, 10)) { const codePointsAsArray = codePoints instanceof regenerate ? codePoints.toArray() : codePoints; @@ -314,15 +314,26 @@ const writeFiles = (options) => { path.resolve(dir, 'code-points.mjs'), codePointsFileContent, ); - writeFileIfChanged( - path.resolve(dir, 'code-points.d.mts'), - `declare const codePoints: ${codePointsType};\nexport default codePoints;\n`, - ); writeFileIfChanged(path.resolve(dir, 'symbols.mjs'), symbolsFileContent); - writeFileIfChanged( - path.resolve(dir, 'symbols.d.mts'), - `declare const symbols: ${symbolsType};\nexport default symbols;\n`, - ); + if (isCodePointMapping) { + // Mapping directories declare their own types, since the value type + // differs per directory (`Map` vs. `Map`). + writeFileIfChanged( + path.resolve(dir, 'code-points.d.mts'), + `declare const codePoints: ${codePointsType};\nexport default codePoints;\n`, + ); + writeFileIfChanged( + path.resolve(dir, 'symbols.d.mts'), + `declare const symbols: ${symbolsType};\nexport default symbols;\n`, + ); + } else { + // All other directories share the declarations in `types/`, selected + // through `typesVersions` in package.json. Remove per-directory + // declarations left over from earlier builds. + for (const file of SHARED_DECLARATIONS) { + fs.rmSync(path.resolve(dir, file + '.d.mts'), { force: true }); + } + } } if (options.type === 'Bidi_Mirroring_Glyph') { const type = options.type; @@ -415,7 +426,26 @@ const readDataFile = (version, type) => { } }; +// The `typesVersions` map for package.json. TypeScript picks the most specific +// matching pattern, so mapping directories keep their own declarations while +// everything else resolves to the shared ones in `types/`. +const typesVersions = (dirMap) => { + const map = {}; + for (const type of MAPPING_TYPES) { + if (hasKey(dirMap, type)) { + map[`${type}/*/code-points.mjs`] = [`${type}/*/code-points.d.mts`]; + map[`${type}/*/symbols.mjs`] = [`${type}/*/symbols.d.mts`]; + } + } + for (const file of SHARED_DECLARATIONS) { + map[`*/${file}.mjs`] = [`types/${file}.d.mts`]; + } + return { '*': map }; +}; + export default { + SHARED_DECLARATIONS, + typesVersions, range, append, extend, diff --git a/static/decode-ranges.d.mts b/static/decode-ranges.d.mts index 58c30b1..c90bb08 100644 --- a/static/decode-ranges.d.mts +++ b/static/decode-ranges.d.mts @@ -22,5 +22,8 @@ declare function decodeRanges(input: string): decodeRanges.UnicodeRange[]; */ declare function decodeDeltas(input: string): number[]; +type UnicodeRange = decodeRanges.UnicodeRange; + export { decodeDeltas }; +export type { UnicodeRange }; export default decodeRanges; diff --git a/static/types/code-points.d.mts b/static/types/code-points.d.mts new file mode 100644 index 0000000..2a81857 --- /dev/null +++ b/static/types/code-points.d.mts @@ -0,0 +1,2 @@ +declare const codePoints: number[]; +export default codePoints; diff --git a/static/types/ranges.d.mts b/static/types/ranges.d.mts new file mode 100644 index 0000000..051e65d --- /dev/null +++ b/static/types/ranges.d.mts @@ -0,0 +1,4 @@ +import type { UnicodeRange } from '../decode-ranges.mjs'; + +declare const ranges: UnicodeRange[]; +export default ranges; diff --git a/static/types/regex.d.mts b/static/types/regex.d.mts new file mode 100644 index 0000000..0d2af1f --- /dev/null +++ b/static/types/regex.d.mts @@ -0,0 +1,2 @@ +declare const regex: RegExp; +export default regex; diff --git a/static/types/symbols.d.mts b/static/types/symbols.d.mts new file mode 100644 index 0000000..0915d8e --- /dev/null +++ b/static/types/symbols.d.mts @@ -0,0 +1,2 @@ +declare const symbols: string[]; +export default symbols; diff --git a/templates/README.md b/templates/README.md index 1c49647..f42c322 100644 --- a/templates/README.md +++ b/templates/README.md @@ -12,6 +12,8 @@ npm install @unicode/unicode-<%= version %> --save-dev **Note:** _@unicode/unicode-<%= version %>_ is supposed to be used in build scripts (i.e. as a `devDependency`), and not at runtime (i.e. as a regular `dependency`). +TypeScript declarations are included. They require TypeScript 4.7 or newer, which is the first version that understands `.d.mts` declaration files. + ## Regular expressions The Unicode data modules ship with regular expressions for categories, scripts, script extensions, blocks, and properties. But maybe you want to create a single regular expression that combines several categories, scripts, etc. In that case, [***you should use Regenerate***](https://mths.be/regenerate). For example, to construct a regex that matches all symbols in the Arabic and Greek scripts as per Unicode v6.3.0: diff --git a/templates/package.json b/templates/package.json index 768215c..39f2805 100644 --- a/templates/package.json +++ b/templates/package.json @@ -25,5 +25,6 @@ "type": "git", "url": "git+https://github.com/node-unicode/unicode-<%= version %>.git" }, - "bugs": "https://github.com/node-unicode/node-unicode-data/issues" + "bugs": "https://github.com/node-unicode/node-unicode-data/issues", + "typesVersions": <%= typesVersions %> } diff --git a/tests/tests.mjs b/tests/tests.mjs index 8441270..0b16d2b 100644 --- a/tests/tests.mjs +++ b/tests/tests.mjs @@ -201,6 +201,47 @@ suite(`The generated latest Unicode js`, () => { } } }); + test('type declarations should be shared via typesVersions', async (t) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${newest}`); + const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'))); + // Every typesVersions target that is not a wildcard must exist. + for (const targets of Object.values(pkg.typesVersions['*'])) { + for (const target of targets) { + if (!target.includes('*')) { + t.assert.ok(fs.existsSync(path.join(root, target)), target); + } + } + } + for (const type of fs.readdirSync(root)) { + const typeDir = path.join(root, type); + if (type.startsWith('.') || !fs.statSync(typeDir).isDirectory()) { + continue; + } + const isMapping = Object.hasOwn( + pkg.typesVersions['*'], + `${type}/*/symbols.mjs`, + ); + for (const name of fs.readdirSync(typeDir)) { + const dir = path.join(typeDir, name); + if (!fs.existsSync(path.join(dir, 'symbols.mjs'))) { + continue; + } + // Mapping directories keep their own declarations; others share. + t.assert.strictEqual( + fs.existsSync(path.join(dir, 'symbols.d.mts')), + isMapping, + dir, + ); + t.assert.strictEqual( + fs.existsSync(path.join(dir, 'code-points.d.mts')), + isMapping, + dir, + ); + t.assert.ok(!fs.existsSync(path.join(dir, 'ranges.d.mts')), dir); + t.assert.ok(!fs.existsSync(path.join(dir, 'regex.d.mts')), dir); + } + } + }); test('every mapping symbols.mjs should match code-points.mjs', async (t) => { await assertSymbolsMatchCodePoints(t, newest); }); From 22387c78308ee2064c54b8c431bd4c28d1199303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Sat, 24 Oct 2026 20:38:59 -0400 Subject: [PATCH 7/7] fix: declare the default export of the package index `index.mjs` has a single default export, an object mapping each property type to its list of directories, but `index.d.mts` declared named exports instead, so `import dirs from '@unicode/unicode-X'` failed to type-check. The declaration now describes the default export with one `readonly : string[]` property per type. A test checks that the declared keys match the object exported at runtime. Assisted-By: Claude Fable 5.1 --- index.mjs | 12 +++++++++--- tests/tests.mjs | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/index.mjs b/index.mjs index 453f50a..5a9b5b8 100644 --- a/index.mjs +++ b/index.mjs @@ -328,9 +328,15 @@ const generateData = async (version) => { ); fs.writeFileSync( path.resolve(__dirname, `output/unicode-${version}/index.d.mts`), - Object.keys(dirMap) - .map((key) => `export const ${key}: string[];`) - .join('\n'), + // `index.mjs` has a single default export: an object mapping each + // property type to the list of its directories. + [ + 'declare const data: {', + ...Object.keys(dirMap).map((key) => `\treadonly ${key}: string[];`), + '};', + 'export default data;', + '', + ].join('\n'), ); fs.writeFileSync( path.resolve(__dirname, `output/unicode-${version}/package.json`), diff --git a/tests/tests.mjs b/tests/tests.mjs index 0b16d2b..eb1112a 100644 --- a/tests/tests.mjs +++ b/tests/tests.mjs @@ -201,6 +201,16 @@ suite(`The generated latest Unicode js`, () => { } } }); + test('index.d.mts should declare the default export of index.mjs', async (t) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${newest}`); + const { default: data } = await import(path.join(root, 'index.mjs')); + const declaration = fs.readFileSync(path.join(root, 'index.d.mts'), 'utf8'); + t.assert.match(declaration, /^export default data;$/m); + const declaredKeys = [ + ...declaration.matchAll(/^\treadonly (\w+): string\[\];$/gm), + ].map((m) => m[1]); + t.assert.deepStrictEqual(declaredKeys, Object.keys(data)); + }); test('type declarations should be shared via typesVersions', async (t) => { const root = path.resolve(__dirname, '..', 'output', `unicode-${newest}`); const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json')));