diff --git a/index.mjs b/index.mjs index 85660d8..5a9b5b8 100644 --- a/index.mjs +++ b/index.mjs @@ -328,13 +328,24 @@ const generateData = async (version) => { ); fs.writeFileSync( path.resolve(__dirname, `output/unicode-${version}/index.d.mts`), - Object.keys(dirMap) - .map((key) => `export const ${key}: string[];`) - .join('\n'), + // `index.mjs` has a single default export: an object mapping each + // property type to the list of its directories. + [ + 'declare const data: {', + ...Object.keys(dirMap).map((key) => `\treadonly ${key}: string[];`), + '};', + 'export default data;', + '', + ].join('\n'), ); fs.writeFileSync( path.resolve(__dirname, `output/unicode-${version}/package.json`), - compilePackage({ version: version }), + compilePackage({ + version: version, + typesVersions: JSON.stringify(utils.typesVersions(dirMap), null, '\t') + // Indent to match the surrounding JSON. + .replace(/\n/g, '\n\t'), + }), ); fs.mkdirSync( path.resolve(__dirname, `output/unicode-${version}/.github/workflows`), @@ -342,6 +353,9 @@ const generateData = async (version) => { recursive: true, }, ); + fs.mkdirSync(path.resolve(__dirname, `output/unicode-${version}/types`), { + recursive: true, + }); const staticFiles = [ '.github/workflows/publish-on-tag.yml', '.gitattributes', @@ -351,6 +365,15 @@ const generateData = async (version) => { 'decode-property-map.d.mts', 'decode-ranges.mjs', 'decode-ranges.d.mts', + 'ranges-to-regex.mjs', + 'ranges-to-regex.d.mts', + 'mapping-to-symbols.mjs', + 'mapping-to-symbols.d.mts', + 'decode-names.mjs', + 'decode-names.d.mts', + 'decode-mapping.mjs', + 'decode-mapping.d.mts', + ...utils.SHARED_DECLARATIONS.map((file) => `types/${file}.d.mts`), ]; for (const file of staticFiles) { fs.copyFileSync( diff --git a/scripts/encode-ranges.mjs b/scripts/encode-ranges.mjs index 8700d04..0524214 100644 --- a/scripts/encode-ranges.mjs +++ b/scripts/encode-ranges.mjs @@ -57,4 +57,4 @@ const encodeRegenerate = (regenerateSet) => { return encodeDeltas(deltas); }; -export { encodeRanges, encodeRegenerate }; +export { base64enc, encodeDeltas, encodeRanges, encodeRegenerate }; diff --git a/scripts/utils.mjs b/scripts/utils.mjs index c294a44..5727001 100644 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -4,20 +4,49 @@ import { fileURLToPath } from 'node:url'; import zlib from 'node:zlib'; import jsesc from 'jsesc'; import regenerate from 'regenerate'; -import { encodeRanges, encodeRegenerate } from './encode-ranges.mjs'; +import { + base64enc, + encodeDeltas, + encodeRanges, + encodeRegenerate, +} from './encode-ranges.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const gzipInline = (data) => { +// Brotli-compress `text` and return a JS expression that decompresses it into +// a Buffer at runtime. +const brotliInlineBuffer = (text) => { + const brotliBuffer = zlib.brotliCompressSync(text, { + params: { + [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, + [zlib.constants.BROTLI_PARAM_LGWIN]: + zlib.constants.BROTLI_MAX_WINDOW_BITS, + [zlib.constants.BROTLI_PARAM_SIZE_HINT]: Buffer.byteLength(text), + }, + }); + const str = brotliBuffer.toString('base64'); + return `brotliDecompressSync(Buffer.from('${str}', 'base64'))`; +}; + +const brotliInline = (data) => { if (data instanceof Map) { - return `new Map(${gzipInline([...data])})`; + return `new Map(${brotliInline([...data])})`; } - const json = JSON.stringify(data); - const gzipBuffer = zlib.gzipSync(json); - const str = gzipBuffer.toString('base64'); - return `JSON.parse(gunzipSync(Buffer.from('${str}', 'base64')))`; + return `JSON.parse(${brotliInlineBuffer(JSON.stringify(data))})`; }; +// Modules whose declarations are shared via `types/*.d.mts` and +// `typesVersions` instead of being repeated in every directory. +const SHARED_DECLARATIONS = ['ranges', 'regex', 'code-points', 'symbols']; + +// Directories under these types declare their own `code-points`/`symbols` types. +const MAPPING_TYPES = [ + 'Case_Folding', + 'Simple_Case_Mapping', + 'Special_Casing', + 'Decomposition_Mapping', +]; + const range = (start, stop) => { // Inclusive, e.g. `range(1, 3)` → `[1, 2, 3]`. const result = []; @@ -76,6 +105,59 @@ const samePropertyRuns = (codePointProperties) => { return result; }; +// Front-code the names in the flat runs produced by `samePropertyRuns`. +// See static/decode-names.mjs for the format and the decoder. +const encodeNames = (flatRuns) => { + const runs = []; + const lines = []; + let previous = ''; + for (let i = 0; i < flatRuns.length; i += 3) { + runs.push(flatRuns[i], flatRuns[i + 1]); + const name = flatRuns[i + 2]; + let shared = 0; + while ( + shared < 63 && + shared < name.length && + shared < previous.length && + name.charCodeAt(shared) === previous.charCodeAt(shared) + ) { + shared++; + } + lines.push(base64enc[shared] + name.slice(shared)); + previous = name; + } + return JSON.stringify(runs) + '\n' + lines.join('\n'); +}; + +// Zigzag encoding maps signed integers to unsigned ones: 0 → 0, -1 → 1, 1 → 2, +// -2 → 3, … so that small differences in either direction stay small. +const zigzag = (value) => ((value << 1) ^ (value >> 31)) >>> 0; + +// Delta-encode a mapping from code points to code point(s), e.g. a case +// mapping or a decomposition mapping, preserving entry order and whether +// each entry maps to a single code point or to an array. +// See static/decode-mapping.mjs for the format and the decoder. +const encodeMapping = (codePoints) => { + const output = []; + let previousFrom = 0; + for (const [from, to] of codePoints) { + output.push(zigzag(from - previousFrom)); + previousFrom = from; + let previous = from; + if (Array.isArray(to)) { + console.assert(to.length > 0, `empty mapping for U+${from.toString(16)}`); + output.push(to.length); + for (const codePoint of to) { + output.push(zigzag(codePoint - previous)); + previous = codePoint; + } + } else { + output.push(0, zigzag(to - previous)); + } + } + return encodeDeltas(output); +}; + // Only write the file if its content has changed. // Reading files is cheaper than writing files, and most files don’t change between builds. const writeFileIfChanged = (filePath, content) => { @@ -113,11 +195,7 @@ const writeFiles = (options) => { const codePoints = map[item]; const type = typeof options.type === 'function' ? options.type(item) : options.type; - const isCodePointMapping = - type === 'Case_Folding' || - type === 'Simple_Case_Mapping' || - type === 'Special_Casing' || - type === 'Decomposition_Mapping'; + const isCodePointMapping = MAPPING_TYPES.includes(type); const isNamesCanon = type === 'Names' && !subType; const isNameAliases = type === 'Names' && subType === 'name-aliases'; const subdir = isNameAliases @@ -164,7 +242,7 @@ const writeFiles = (options) => { // Sequence properties are special. if (type === 'Sequence_Property' || isNameAliases) { const sequences = codePoints; - const output = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(sequences)};\n`; + const output = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(sequences)};\n`; writeFileIfChanged(path.resolve(dir, 'index.mjs'), output); writeFileIfChanged( path.resolve(dir, 'index.d.mts'), @@ -189,17 +267,9 @@ const writeFiles = (options) => { path.resolve(dir, 'ranges.mjs'), `import decodeRanges from '../../decode-ranges.mjs';\n\nexport default decodeRanges('${encodedRanges}');\n`, ); - writeFileIfChanged( - path.resolve(dir, 'ranges.d.mts'), - 'import type { UnicodeRange } from "../../decode-ranges.mjs";\n\ndeclare const ranges: UnicodeRange[];\nexport default ranges;\n', - ); writeFileIfChanged( path.resolve(dir, 'regex.mjs'), - 'export default /' + regenerate(codePoints).toString() + '/;\n', - ); - writeFileIfChanged( - path.resolve(dir, 'regex.d.mts'), - 'declare const regex: RegExp;\nexport default regex;\n', + `import ranges from './ranges.mjs';\nimport rangesToRegex from '../../ranges-to-regex.mjs';\n\nexport default rangesToRegex(ranges);\n`, ); if (codePointsSizeLt(codePoints, 10)) { const codePointsAsArray = @@ -208,22 +278,26 @@ const writeFiles = (options) => { symbolsFileContent = `export default ${jsesc(codePointsAsArray.map((cp) => String.fromCodePoint(cp)))};\n`; } } else { - const symbols = new Map(); - for (let [from, to] of codePoints) { - from = String.fromCodePoint(from); - if (Array.isArray(to)) { - to = String.fromCodePoint.apply(null, to); - } else { - to = String.fromCodePoint(to); - } - symbols.set(from, to); - } if (codePoints.size < 10) { + const symbols = new Map(); + for (let [from, to] of codePoints) { + from = String.fromCodePoint(from); + if (Array.isArray(to)) { + to = String.fromCodePoint.apply(null, to); + } else { + to = String.fromCodePoint(to); + } + symbols.set(from, to); + } codePointsFileContent = `export default ${jsesc(codePoints)};\n`; symbolsFileContent = `export default ${jsesc(symbols)};\n`; } else { - codePointsFileContent = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(codePoints)};\n`; - symbolsFileContent = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(symbols)};\n`; + codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\nimport decodeMapping from '../../decode-mapping.mjs';\n\nexport default decodeMapping(${brotliInlineBuffer( + encodeMapping(codePoints), + )}.toString());\n`; + // The symbols map is derived from the code points map at runtime + // rather than shipped as a second copy of the same data. + symbolsFileContent = `import codePoints from './code-points.mjs';\nimport mappingToSymbols from '../../mapping-to-symbols.mjs';\n\nexport default mappingToSymbols(codePoints);\n`; } if ( (type === 'Case_Folding' && item === 'F') || @@ -240,15 +314,26 @@ const writeFiles = (options) => { path.resolve(dir, 'code-points.mjs'), codePointsFileContent, ); - writeFileIfChanged( - path.resolve(dir, 'code-points.d.mts'), - `declare const codePoints: ${codePointsType};\nexport default codePoints;\n`, - ); writeFileIfChanged(path.resolve(dir, 'symbols.mjs'), symbolsFileContent); - writeFileIfChanged( - path.resolve(dir, 'symbols.d.mts'), - `declare const symbols: ${symbolsType};\nexport default symbols;\n`, - ); + if (isCodePointMapping) { + // Mapping directories declare their own types, since the value type + // differs per directory (`Map` vs. `Map`). + writeFileIfChanged( + path.resolve(dir, 'code-points.d.mts'), + `declare const codePoints: ${codePointsType};\nexport default codePoints;\n`, + ); + writeFileIfChanged( + path.resolve(dir, 'symbols.d.mts'), + `declare const symbols: ${symbolsType};\nexport default symbols;\n`, + ); + } else { + // All other directories share the declarations in `types/`, selected + // through `typesVersions` in package.json. Remove per-directory + // declarations left over from earlier builds. + for (const file of SHARED_DECLARATIONS) { + fs.rmSync(path.resolve(dir, file + '.d.mts'), { force: true }); + } + } } if (options.type === 'Bidi_Mirroring_Glyph') { const type = options.type; @@ -264,7 +349,7 @@ const writeFiles = (options) => { } fs.mkdirSync(dir, { recursive: true }); // `Bidi_Mirroring_Glyph/index.mjs` - // Note: `Bidi_Mirroring_Glyph` doesn’t have repeated strings; don’t gzip. + // Note: `Bidi_Mirroring_Glyph` doesn’t have repeated strings; don’t compress. const output = [ `const chr = String.fromCodePoint;`, `const pair = (t, u, v) => [t ? u + v : v, chr(t ? u : u + v)];`, @@ -295,9 +380,14 @@ const writeFiles = (options) => { // or `bidi-brackets/index.mjs` // or `Names/index.mjs` const flatRuns = samePropertyRuns(auxMap[type]); - const output = `import { gunzipSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${gzipInline( - flatRuns, - )});\n`; + const output = + type === 'Names' + ? `import { brotliDecompressSync } from 'node:zlib';\nimport decodeNames from '../decode-names.mjs';\n\nexport default decodeNames(${brotliInlineBuffer( + encodeNames(flatRuns), + )}.toString());\n` + : `import { brotliDecompressSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${brotliInline( + flatRuns, + )});\n`; writeFileIfChanged(path.resolve(dir, 'index.mjs'), output); writeFileIfChanged( path.resolve(dir, 'index.d.mts'), @@ -336,7 +426,26 @@ const readDataFile = (version, type) => { } }; +// The `typesVersions` map for package.json. TypeScript picks the most specific +// matching pattern, so mapping directories keep their own declarations while +// everything else resolves to the shared ones in `types/`. +const typesVersions = (dirMap) => { + const map = {}; + for (const type of MAPPING_TYPES) { + if (hasKey(dirMap, type)) { + map[`${type}/*/code-points.mjs`] = [`${type}/*/code-points.d.mts`]; + map[`${type}/*/symbols.mjs`] = [`${type}/*/symbols.d.mts`]; + } + } + for (const file of SHARED_DECLARATIONS) { + map[`*/${file}.mjs`] = [`types/${file}.d.mts`]; + } + return { '*': map }; +}; + export default { + SHARED_DECLARATIONS, + typesVersions, range, append, extend, diff --git a/static/decode-mapping.d.mts b/static/decode-mapping.d.mts new file mode 100644 index 0000000..57d9033 --- /dev/null +++ b/static/decode-mapping.d.mts @@ -0,0 +1,7 @@ +/** + * Decode a delta-encoded mapping from code points to code point(s), e.g. a + * case mapping or a decomposition mapping. + */ +declare function decodeMapping(input: string): Map; + +export default decodeMapping; diff --git a/static/decode-mapping.mjs b/static/decode-mapping.mjs new file mode 100644 index 0000000..c54cbac --- /dev/null +++ b/static/decode-mapping.mjs @@ -0,0 +1,46 @@ +/** + * This file ships to end users. + * See `encodeMapping` in scripts/utils.mjs for the encoder. + */ + +import { decodeDeltas } from './decode-ranges.mjs'; + +// Undo zigzag encoding: 0 → 0, 1 → -1, 2 → 1, 3 → -2, … +const unzigzag = (value) => (value >>> 1) ^ -(value & 1); + +/** + * Decode a delta-encoded mapping from code points to code point(s), e.g. a + * case mapping or a decomposition mapping. + * + * `input` is a base64 stream of variable-length integers (see + * `decodeDeltas`). Each entry consists of the zigzag-encoded difference + * between its key and the previous key, a count, and then `count` values + * (or one value, if `count` is 0). Each value is the zigzag-encoded difference + * from the previous code point in the entry, starting from the key. A count + * of 0 means the entry maps to a single code point rather than to an array. + * @param {string} input + * @returns {Map} + */ +const decodeMapping = (input) => { + const deltas = decodeDeltas(input); + const map = new Map(); + for (let from = 0, i = 0; i < deltas.length;) { + from += unzigzag(deltas[i++]); + const count = deltas[i++]; + let codePoint = from; + if (count === 0) { + codePoint += unzigzag(deltas[i++]); + map.set(from, codePoint); + } else { + const codePoints = []; + for (let j = 0; j < count; j++) { + codePoint += unzigzag(deltas[i++]); + codePoints.push(codePoint); + } + map.set(from, codePoints); + } + } + return map; +}; + +export default decodeMapping; diff --git a/static/decode-names.d.mts b/static/decode-names.d.mts new file mode 100644 index 0000000..54d1d83 --- /dev/null +++ b/static/decode-names.d.mts @@ -0,0 +1,7 @@ +/** + * Decode a front-coded table of character names into a map from code point + * to name. + */ +declare function decodeNames(input: string): Map; + +export default decodeNames; diff --git a/static/decode-names.mjs b/static/decode-names.mjs new file mode 100644 index 0000000..b5f3946 --- /dev/null +++ b/static/decode-names.mjs @@ -0,0 +1,40 @@ +/** + * This file ships to end users. + * See `encodeNames` in scripts/utils.mjs for the encoder. + */ + +const base64enc = + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; + +/** + * Decode a front-coded table of character names into a map from code point + * to name. + * + * The first line of `input` is a JSON array of `[gap, length]` pairs, one per + * name, describing the run of code points each name applies to: `gap` is the + * number of unnamed code points since the end of the previous run, and + * `length` is the number of consecutive code points sharing the name. + * + * Every following line holds one name, front-coded against the previous + * name: the first character is the base64 digit for the number of leading + * characters shared with the previous name, and the rest is the remainder. + * @param {string} input + */ +const decodeNames = (input) => { + const lines = input.split('\n'); + const runs = JSON.parse(lines[0]); + const map = new Map(); + let previous = ''; + for (let codePoint = 0, i = 1, j = 0; i < lines.length; i++, j += 2) { + const line = lines[i]; + const name = previous.slice(0, base64enc.indexOf(line[0])) + line.slice(1); + previous = name; + codePoint += runs[j]; + for (const end = codePoint + runs[j + 1]; codePoint < end; codePoint++) { + map.set(codePoint, name); + } + } + return map; +}; + +export default decodeNames; diff --git a/static/decode-ranges.d.mts b/static/decode-ranges.d.mts index 4cfa706..c90bb08 100644 --- a/static/decode-ranges.d.mts +++ b/static/decode-ranges.d.mts @@ -17,4 +17,13 @@ declare namespace decodeRanges { */ declare function decodeRanges(input: string): decodeRanges.UnicodeRange[]; +/** + * Base64 decode variable-length deltas (5/10/15/21-bit). + */ +declare function decodeDeltas(input: string): number[]; + +type UnicodeRange = decodeRanges.UnicodeRange; + +export { decodeDeltas }; +export type { UnicodeRange }; export default decodeRanges; diff --git a/static/decode-ranges.mjs b/static/decode-ranges.mjs index 7e908c0..7463e8d 100644 --- a/static/decode-ranges.mjs +++ b/static/decode-ranges.mjs @@ -82,4 +82,5 @@ const decodeRanges = (input) => { return ranges; }; +export { decodeDeltas }; export default decodeRanges; diff --git a/static/mapping-to-symbols.d.mts b/static/mapping-to-symbols.d.mts new file mode 100644 index 0000000..9dc555c --- /dev/null +++ b/static/mapping-to-symbols.d.mts @@ -0,0 +1,9 @@ +/** + * Derive the symbol-to-symbol(s) map of a case mapping, case folding, or + * decomposition mapping from its code-point-to-code-point(s) map. + */ +declare function mappingToSymbols( + codePoints: Map, +): Map; + +export default mappingToSymbols; diff --git a/static/mapping-to-symbols.mjs b/static/mapping-to-symbols.mjs new file mode 100644 index 0000000..616cfad --- /dev/null +++ b/static/mapping-to-symbols.mjs @@ -0,0 +1,25 @@ +/** + * This file ships to end users. + * + * Derive the symbol-to-symbol(s) map of a case mapping, case folding, or + * decomposition mapping from its code-point-to-code-point(s) map. + */ + +const fromCodePoint = String.fromCodePoint; + +/** + * @param {Map} codePoints + * @returns {Map} + */ +const mappingToSymbols = (codePoints) => { + const symbols = new Map(); + for (const [from, to] of codePoints) { + symbols.set( + fromCodePoint(from), + Array.isArray(to) ? fromCodePoint(...to) : fromCodePoint(to), + ); + } + return symbols; +}; + +export default mappingToSymbols; diff --git a/static/ranges-to-regex.d.mts b/static/ranges-to-regex.d.mts new file mode 100644 index 0000000..89f03a6 --- /dev/null +++ b/static/ranges-to-regex.d.mts @@ -0,0 +1,8 @@ +import type { UnicodeRange } from './decode-ranges.mjs'; + +/** + * Build a regular expression that matches any symbol in `ranges`. + */ +declare function rangesToRegex(ranges: UnicodeRange[]): RegExp; + +export default rangesToRegex; diff --git a/static/ranges-to-regex.mjs b/static/ranges-to-regex.mjs new file mode 100644 index 0000000..22d15de --- /dev/null +++ b/static/ranges-to-regex.mjs @@ -0,0 +1,635 @@ +/** + * This file ships to end users. + * + * Builds a regular expression from the ranges produced by + * `decode-ranges.mjs`. This is a port of the serialization logic in + * Regenerate (MIT licensed), so the resulting + * `source` is byte-for-byte identical to `regenerate(set).toString()`. + * + * Throughout this file, “data” refers to Regenerate’s internal representation: + * a flat, sorted array of `[start, end)` pairs, e.g. `[0x41, 0x5B, 0x61, 0x7B]`. + */ + +const HIGH_SURROGATE_MIN = 0xd800; +const HIGH_SURROGATE_MAX = 0xdbff; +const LOW_SURROGATE_MIN = 0xdc00; +const LOW_SURROGATE_MAX = 0xdfff; + +const regexNull = /\\x00([^0123456789]|$)/g; + +const hex = (number) => number.toString(16).toUpperCase(); + +const pad = (string, totalCharacters) => { + return string.length < totalCharacters + ? ('0000' + string).slice(-totalCharacters) + : string; +}; + +const dataIsSingleton = (data) => { + // Check if the set only represents a single code point. + return data.length == 2 && data[0] + 1 == data[1]; +}; + +const dataAdd = (data, codePoint) => { + // Iterate over the data per `(start, end)` pair. + let index = 0; + let lastIndex = null; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1]; + + // Check if the code point is already in the set. + if (codePoint >= start && codePoint < end) { + return data; + } + + if (codePoint == start - 1) { + // Just replace `start` with a new value. + data[index] = codePoint; + return data; + } + + // At this point, if `start` is `greater` than `codePoint`, insert a new + // `[start, end]` pair before the current pair, or after the current pair + // if there is a known `lastIndex`. + if (start > codePoint) { + data.splice( + lastIndex != null ? lastIndex + 2 : 0, + 0, + codePoint, + codePoint + 1, + ); + return data; + } + + if (codePoint == end) { + // Check if adding this code point causes two separate ranges to become + // a single range, e.g. `dataAdd([0, 4, 5, 10], 4)` → `[0, 10]`. + if (codePoint + 1 == data[index + 2]) { + data.splice(index, 4, start, data[index + 3]); + return data; + } + // Else, just replace `end` with a new value. + data[index + 1] = codePoint + 1; + return data; + } + lastIndex = index; + index += 2; + } + // The loop has finished; add the new pair to the end of the data set. + data.push(codePoint, codePoint + 1); + return data; +}; + +const dataAddRange = (data, rangeStart, rangeEnd) => { + // Iterate over the data per `(start, end)` pair. + let index = 0; + let added = false; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1]; + + if (added) { + // The range has already been added to the set; at this point, we just + // need to get rid of the following ranges in case they overlap. + + // Check if this range can be combined with the previous range. + if (start == rangeEnd + 1) { + data.splice(index - 1, 2); + return data; + } + + // Exit as soon as no more possibly overlapping pairs can be found. + if (start > rangeEnd) { + return data; + } + + // E.g. `[0, 11, 12, 16]` and we’ve added 5-15, so we now have + // `[0, 16, 12, 16]`. Remove the `12,16` part, as it lies within the + // `0,16` range that was previously added. + if (start >= rangeStart && start <= rangeEnd) { + // `start` lies within the range that was previously added. + + if (end > rangeStart && end - 1 <= rangeEnd) { + // `end` lies within the range that was previously added as well, + // so remove this pair. + data.splice(index, 2); + index -= 2; + // Note: we cannot `return` just yet, as there may still be other + // overlapping pairs. + } else { + // `start` lies within the range that was previously added, but + // `end` doesn’t. E.g. `[0, 11, 12, 31]` and we’ve added 5-15, so + // now we have `[0, 16, 12, 31]`. This must be written as `[0, 31]`. + // Remove the previously added `end` and the current `start`. + data.splice(index - 1, 2); + index -= 2; + } + + // Note: we cannot return yet. + } + } else if (start == rangeEnd + 1 || start == rangeEnd) { + data[index] = rangeStart; + return data; + } + + // Check if a new pair must be inserted *before* the current one. + else if (start > rangeEnd) { + data.splice(index, 0, rangeStart, rangeEnd + 1); + return data; + } else if (rangeStart >= start && rangeStart < end && rangeEnd + 1 <= end) { + // The new range lies entirely within an existing range pair. No action + // needed. + return data; + } else if ( + // E.g. `[0, 11]` and you add 5-15 → `[0, 16]`. + (rangeStart >= start && rangeStart < end) || + // E.g. `[0, 3]` and you add 3-6 → `[0, 7]`. + end == rangeStart + ) { + // Replace `end` with the new value. + data[index + 1] = rangeEnd + 1; + // Make sure the next range pair doesn’t overlap, e.g. `[0, 11, 12, 14]` + // and you add 5-15 → `[0, 16]`, i.e. remove the `12,14` part. + added = true; + // Note: we cannot `return` just yet. + } else if (rangeStart <= start && rangeEnd + 1 >= end) { + // The new range is a superset of the old range. + data[index] = rangeStart; + data[index + 1] = rangeEnd + 1; + added = true; + } + + index += 2; + } + // The loop has finished without doing anything; add the new pair to the end + // of the data set. + if (!added) { + data.push(rangeStart, rangeEnd + 1); + } + return data; +}; + +// https://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae +const highSurrogate = (codePoint) => { + return Math.floor((codePoint - 0x10000) / 0x400) + HIGH_SURROGATE_MIN; +}; + +const lowSurrogate = (codePoint) => { + return ((codePoint - 0x10000) % 0x400) + LOW_SURROGATE_MIN; +}; + +const stringFromCharCode = String.fromCharCode; +const codePointToString = (codePoint) => { + // https://mathiasbynens.be/notes/javascript-escapes#single + // Note: the `\b` escape sequence for U+0008 BACKSPACE in strings has a + // different meaning in regular expressions (word boundary), so it cannot + // be used here. + if (codePoint == 0x09) { + return '\\t'; + } + // Note: IE < 9 treats `'\v'` as `'v'`, so avoid using it. + if (codePoint == 0x0a) { + return '\\n'; + } + if (codePoint == 0x0c) { + return '\\f'; + } + if (codePoint == 0x0d) { + return '\\r'; + } + if (codePoint == 0x2d) { + // https://mathiasbynens.be/notes/javascript-escapes#hexadecimal + // Note: `-` (U+002D HYPHEN-MINUS) is escaped in this way rather + // than by backslash-escaping, in case the output is used outside + // of a character class in a `u` RegExp. /\-/u throws, but + // /\x2D/u is fine. + return '\\x2D'; + } + if (codePoint == 0x5c) { + return '\\\\'; + } + if ( + codePoint == 0x24 || + (codePoint >= 0x28 && codePoint <= 0x2b) || + codePoint == 0x2e || + codePoint == 0x2f || + codePoint == 0x3f || + (codePoint >= 0x5b && codePoint <= 0x5e) || + (codePoint >= 0x7b && codePoint <= 0x7d) + ) { + // The code point maps to an unsafe printable ASCII character; + // backslash-escape it. Here’s the list of those symbols: + // + // $()*+./?[\]^{|} + // + // This matches SyntaxCharacters as well as `/` (U+002F SOLIDUS). + // https://tc39.github.io/ecma262/#prod-SyntaxCharacter + return '\\' + stringFromCharCode(codePoint); + } + if (codePoint >= 0x20 && codePoint <= 0x7e) { + // The code point maps to one of these printable ASCII symbols + // (including the space character): + // + // !"#%&',/0123456789:;<=>@ABCDEFGHIJKLMNO + // PQRSTUVWXYZ_`abcdefghijklmnopqrstuvwxyz~ + // + // These can safely be used directly. + return stringFromCharCode(codePoint); + } + if (codePoint <= 0xff) { + return '\\x' + pad(hex(codePoint), 2); + } + // `codePoint <= 0xFFFF` holds true. + // https://mathiasbynens.be/notes/javascript-escapes#unicode + return '\\u' + pad(hex(codePoint), 4); +}; + +const createBMPCharacterClasses = (data) => { + // Iterate over the data per `(start, end)` pair. + let result = ''; + let index = 0; + const length = data.length; + if (dataIsSingleton(data)) { + return codePointToString(data[0]); + } + while (index < length) { + const start = data[index]; + const end = data[index + 1] - 1; // Note: the `- 1` makes `end` inclusive. + if (start == end) { + result += codePointToString(start); + } else if (start + 1 == end) { + result += codePointToString(start) + codePointToString(end); + } else { + result += codePointToString(start) + '-' + codePointToString(end); + } + index += 2; + } + return '[' + result + ']'; +}; + +const splitAtBMP = (data) => { + // Iterate over the data per `(start, end)` pair. + const loneHighSurrogates = []; + const loneLowSurrogates = []; + const bmp = []; + const astral = []; + let index = 0; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1] - 1; // Note: the `- 1` makes `end` inclusive. + + if (start < HIGH_SURROGATE_MIN) { + // The range starts and ends before the high surrogate range. + // E.g. (0, 0x10). + if (end < HIGH_SURROGATE_MIN) { + bmp.push(start, end + 1); + } + + // The range starts before the high surrogate range and ends within it. + // E.g. (0, 0xD855). + if (end >= HIGH_SURROGATE_MIN && end <= HIGH_SURROGATE_MAX) { + bmp.push(start, HIGH_SURROGATE_MIN); + loneHighSurrogates.push(HIGH_SURROGATE_MIN, end + 1); + } + + // The range starts before the high surrogate range and ends in the low + // surrogate range. E.g. (0, 0xDCFF). + if (end >= LOW_SURROGATE_MIN && end <= LOW_SURROGATE_MAX) { + bmp.push(start, HIGH_SURROGATE_MIN); + loneHighSurrogates.push(HIGH_SURROGATE_MIN, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, end + 1); + } + + // The range starts before the high surrogate range and ends after the + // low surrogate range. E.g. (0, 0x10FFFF). + if (end > LOW_SURROGATE_MAX) { + bmp.push(start, HIGH_SURROGATE_MIN); + loneHighSurrogates.push(HIGH_SURROGATE_MIN, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, LOW_SURROGATE_MAX + 1); + if (end <= 0xffff) { + bmp.push(LOW_SURROGATE_MAX + 1, end + 1); + } else { + bmp.push(LOW_SURROGATE_MAX + 1, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } + } else if (start >= HIGH_SURROGATE_MIN && start <= HIGH_SURROGATE_MAX) { + // The range starts and ends in the high surrogate range. + // E.g. (0xD855, 0xD866). + if (end >= HIGH_SURROGATE_MIN && end <= HIGH_SURROGATE_MAX) { + loneHighSurrogates.push(start, end + 1); + } + + // The range starts in the high surrogate range and ends in the low + // surrogate range. E.g. (0xD855, 0xDCFF). + if (end >= LOW_SURROGATE_MIN && end <= LOW_SURROGATE_MAX) { + loneHighSurrogates.push(start, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, end + 1); + } + + // The range starts in the high surrogate range and ends after the low + // surrogate range. E.g. (0xD855, 0x10FFFF). + if (end > LOW_SURROGATE_MAX) { + loneHighSurrogates.push(start, HIGH_SURROGATE_MAX + 1); + loneLowSurrogates.push(LOW_SURROGATE_MIN, LOW_SURROGATE_MAX + 1); + if (end <= 0xffff) { + bmp.push(LOW_SURROGATE_MAX + 1, end + 1); + } else { + bmp.push(LOW_SURROGATE_MAX + 1, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } + } else if (start >= LOW_SURROGATE_MIN && start <= LOW_SURROGATE_MAX) { + // The range starts and ends in the low surrogate range. + // E.g. (0xDCFF, 0xDDFF). + if (end >= LOW_SURROGATE_MIN && end <= LOW_SURROGATE_MAX) { + loneLowSurrogates.push(start, end + 1); + } + + // The range starts in the low surrogate range and ends after the low + // surrogate range. E.g. (0xDCFF, 0x10FFFF). + if (end > LOW_SURROGATE_MAX) { + loneLowSurrogates.push(start, LOW_SURROGATE_MAX + 1); + if (end <= 0xffff) { + bmp.push(LOW_SURROGATE_MAX + 1, end + 1); + } else { + bmp.push(LOW_SURROGATE_MAX + 1, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } + } else if (start > LOW_SURROGATE_MAX && start <= 0xffff) { + // The range starts and ends after the low surrogate range. + // E.g. (0xFFAA, 0x10FFFF). + if (end <= 0xffff) { + bmp.push(start, end + 1); + } else { + bmp.push(start, 0xffff + 1); + astral.push(0xffff + 1, end + 1); + } + } else { + // The range starts and ends in the astral range. + astral.push(start, end + 1); + } + + index += 2; + } + return { loneHighSurrogates, loneLowSurrogates, bmp, astral }; +}; + +const optimizeByLowSurrogates = (surrogateMappings) => { + if (surrogateMappings.length == 1) { + return surrogateMappings; + } + let index = -1; + let innerIndex = -1; + while (++index < surrogateMappings.length) { + const mapping = surrogateMappings[index]; + const lowSurrogates = mapping[1]; + const lowSurrogateStart = lowSurrogates[0]; + const lowSurrogateEnd = lowSurrogates[1]; + innerIndex = index; // Note: the loop starts at the next index. + while (++innerIndex < surrogateMappings.length) { + const otherMapping = surrogateMappings[innerIndex]; + const otherLowSurrogates = otherMapping[1]; + const otherLowSurrogateStart = otherLowSurrogates[0]; + const otherLowSurrogateEnd = otherLowSurrogates[1]; + if ( + lowSurrogateStart == otherLowSurrogateStart && + lowSurrogateEnd == otherLowSurrogateEnd && + otherLowSurrogates.length === 2 + ) { + // Add the code points in the other item to this one. + if (dataIsSingleton(otherMapping[0])) { + mapping[0] = dataAdd(mapping[0], otherMapping[0][0]); + } else { + mapping[0] = dataAddRange( + mapping[0], + otherMapping[0][0], + otherMapping[0][1] - 1, + ); + } + // Remove the other, now redundant, item. + surrogateMappings.splice(innerIndex, 1); + --innerIndex; + } + } + } + return surrogateMappings; +}; + +const optimizeSurrogateMappings = (surrogateMappings) => { + const result = []; + let tmpLow = []; + let addLow = false; + let index = -1; + const length = surrogateMappings.length; + while (++index < length) { + let mapping = surrogateMappings[index]; + let nextMapping = surrogateMappings[index + 1]; + if (!nextMapping) { + result.push(mapping); + continue; + } + let highSurrogates = mapping[0]; + let lowSurrogates = mapping[1]; + let nextHighSurrogates = nextMapping[0]; + let nextLowSurrogates = nextMapping[1]; + + // Check for identical high surrogate ranges. + tmpLow = lowSurrogates; + while ( + nextHighSurrogates && + highSurrogates[0] == nextHighSurrogates[0] && + highSurrogates[1] == nextHighSurrogates[1] + ) { + // Merge with the next item. + if (dataIsSingleton(nextLowSurrogates)) { + tmpLow = dataAdd(tmpLow, nextLowSurrogates[0]); + } else { + tmpLow = dataAddRange( + tmpLow, + nextLowSurrogates[0], + nextLowSurrogates[1] - 1, + ); + } + ++index; + mapping = surrogateMappings[index]; + highSurrogates = mapping[0]; + lowSurrogates = mapping[1]; + nextMapping = surrogateMappings[index + 1]; + nextHighSurrogates = nextMapping && nextMapping[0]; + nextLowSurrogates = nextMapping && nextMapping[1]; + addLow = true; + } + result.push([highSurrogates, addLow ? tmpLow : lowSurrogates]); + addLow = false; + } + return optimizeByLowSurrogates(result); +}; + +const surrogateSet = (data) => { + // Exit early if `data` is an empty set. + if (!data.length) { + return []; + } + + // Iterate over the data per `(start, end)` pair. + let index = 0; + const surrogateMappings = []; + const length = data.length; + while (index < length) { + const start = data[index]; + const end = data[index + 1] - 1; + + const startHigh = highSurrogate(start); + const startLow = lowSurrogate(start); + const endHigh = highSurrogate(end); + const endLow = lowSurrogate(end); + + const startsWithLowestLowSurrogate = startLow == LOW_SURROGATE_MIN; + const endsWithHighestLowSurrogate = endLow == LOW_SURROGATE_MAX; + let complete = false; + + // Append the previous high-surrogate-to-low-surrogate mappings. + // Step 1: `(startHigh, startLow)` to `(startHigh, LOW_SURROGATE_MAX)`. + if ( + startHigh == endHigh || + (startsWithLowestLowSurrogate && endsWithHighestLowSurrogate) + ) { + surrogateMappings.push([ + [startHigh, endHigh + 1], + [startLow, endLow + 1], + ]); + complete = true; + } else { + surrogateMappings.push([ + [startHigh, startHigh + 1], + [startLow, LOW_SURROGATE_MAX + 1], + ]); + } + + // Step 2: `(startHigh + 1, LOW_SURROGATE_MIN)` to + // `(endHigh - 1, LOW_SURROGATE_MAX)`. + if (!complete && startHigh + 1 < endHigh) { + if (endsWithHighestLowSurrogate) { + // Combine step 2 and step 3. + surrogateMappings.push([ + [startHigh + 1, endHigh + 1], + [LOW_SURROGATE_MIN, endLow + 1], + ]); + complete = true; + } else { + surrogateMappings.push([ + [startHigh + 1, endHigh], + [LOW_SURROGATE_MIN, LOW_SURROGATE_MAX + 1], + ]); + } + } + + // Step 3. `(endHigh, LOW_SURROGATE_MIN)` to `(endHigh, endLow)`. + if (!complete) { + surrogateMappings.push([ + [endHigh, endHigh + 1], + [LOW_SURROGATE_MIN, endLow + 1], + ]); + } + + index += 2; + } + + // The format of `surrogateMappings` is as follows: + // + // [ surrogateMapping1, surrogateMapping2 ] + // + // i.e.: + // + // [ + // [ highSurrogates1, lowSurrogates1 ], + // [ highSurrogates2, lowSurrogates2 ] + // ] + return optimizeSurrogateMappings(surrogateMappings); +}; + +const createSurrogateCharacterClasses = (surrogateMappings) => { + const result = []; + for (const surrogateMapping of surrogateMappings) { + const highSurrogates = surrogateMapping[0]; + const lowSurrogates = surrogateMapping[1]; + result.push( + createBMPCharacterClasses(highSurrogates) + + createBMPCharacterClasses(lowSurrogates), + ); + } + return result.join('|'); +}; + +const createCharacterClassesFromData = (data) => { + const result = []; + + const { loneHighSurrogates, loneLowSurrogates, bmp, astral } = + splitAtBMP(data); + + const surrogateMappings = surrogateSet(astral); + + if (bmp.length) { + // The data set contains BMP code points that are not high surrogates + // needed for astral code points in the set. + result.push(createBMPCharacterClasses(bmp)); + } + if (surrogateMappings.length) { + // The data set contains astral code points; append character classes + // based on their surrogate pairs. + result.push(createSurrogateCharacterClasses(surrogateMappings)); + } + // https://gist.github.com/mathiasbynens/bbe7f870208abcfec860 + if (loneHighSurrogates.length) { + result.push( + createBMPCharacterClasses(loneHighSurrogates) + + // Make sure the high surrogates aren’t part of a surrogate pair. + '(?![\\uDC00-\\uDFFF])', + ); + } + if (loneLowSurrogates.length) { + result.push( + // It is not possible to accurately assert the low surrogates aren’t + // part of a surrogate pair, since JavaScript regular expressions do + // not support lookbehind. + '(?:[^\\uD800-\\uDBFF]|^)' + createBMPCharacterClasses(loneLowSurrogates), + ); + } + return result.join('|'); +}; + +/** + * Serialize `data` the same way `regenerate(set).toString()` would. + * @param {number[]} data + */ +const dataToRegexSource = (data) => { + const result = createCharacterClassesFromData(data); + if (!result) { + // For an empty set, return something that can be inserted `/here/` to + // form a valid regular expression. Avoid `(?:)` since that matches the + // empty string. + return '[]'; + } + // Use `\0` instead of `\x00` where possible. + return result.replace(regexNull, '\\0$1'); +}; + +/** + * Build a regular expression that matches any symbol in `ranges`. + * @param {Array<{ begin: number, end: number }>} ranges + */ +const rangesToRegex = (ranges) => { + const data = []; + for (const range of ranges) { + data.push(range.begin, range.end); + } + return new RegExp(dataToRegexSource(data)); +}; + +export default rangesToRegex; diff --git a/static/types/code-points.d.mts b/static/types/code-points.d.mts new file mode 100644 index 0000000..2a81857 --- /dev/null +++ b/static/types/code-points.d.mts @@ -0,0 +1,2 @@ +declare const codePoints: number[]; +export default codePoints; diff --git a/static/types/ranges.d.mts b/static/types/ranges.d.mts new file mode 100644 index 0000000..051e65d --- /dev/null +++ b/static/types/ranges.d.mts @@ -0,0 +1,4 @@ +import type { UnicodeRange } from '../decode-ranges.mjs'; + +declare const ranges: UnicodeRange[]; +export default ranges; diff --git a/static/types/regex.d.mts b/static/types/regex.d.mts new file mode 100644 index 0000000..0d2af1f --- /dev/null +++ b/static/types/regex.d.mts @@ -0,0 +1,2 @@ +declare const regex: RegExp; +export default regex; diff --git a/static/types/symbols.d.mts b/static/types/symbols.d.mts new file mode 100644 index 0000000..0915d8e --- /dev/null +++ b/static/types/symbols.d.mts @@ -0,0 +1,2 @@ +declare const symbols: string[]; +export default symbols; diff --git a/templates/README.md b/templates/README.md index 12baa3f..f42c322 100644 --- a/templates/README.md +++ b/templates/README.md @@ -12,9 +12,11 @@ npm install @unicode/unicode-<%= version %> --save-dev **Note:** _@unicode/unicode-<%= version %>_ is supposed to be used in build scripts (i.e. as a `devDependency`), and not at runtime (i.e. as a regular `dependency`). +TypeScript declarations are included. They require TypeScript 4.7 or newer, which is the first version that understands `.d.mts` declaration files. + ## Regular expressions -The Unicode data modules ship with pre-compiled regular expressions for categories, scripts, script extensions, blocks, and properties. But maybe you want to create a single regular expression that combines several categories, scripts, etc. In that case, [***you should use Regenerate***](https://mths.be/regenerate). For example, to construct a regex that matches all symbols in the Arabic and Greek scripts as per Unicode v6.3.0: +The Unicode data modules ship with regular expressions for categories, scripts, script extensions, blocks, and properties. But maybe you want to create a single regular expression that combines several categories, scripts, etc. In that case, [***you should use Regenerate***](https://mths.be/regenerate). For example, to construct a regex that matches all symbols in the Arabic and Greek scripts as per Unicode v6.3.0: ```js import regenerate from 'regenerate'; diff --git a/templates/package.json b/templates/package.json index 768215c..39f2805 100644 --- a/templates/package.json +++ b/templates/package.json @@ -25,5 +25,6 @@ "type": "git", "url": "git+https://github.com/node-unicode/unicode-<%= version %>.git" }, - "bugs": "https://github.com/node-unicode/node-unicode-data/issues" + "bugs": "https://github.com/node-unicode/node-unicode-data/issues", + "typesVersions": <%= typesVersions %> } diff --git a/tests/tests.mjs b/tests/tests.mjs index 248492d..eb1112a 100644 --- a/tests/tests.mjs +++ b/tests/tests.mjs @@ -1,6 +1,12 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { suite, test } from 'node:test'; +import regenerate from 'regenerate'; import resources from '../data/resources.mjs'; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const oldest = resources[0].version; const newest = resources.at(-1).version; @@ -11,6 +17,67 @@ if (!process.env.CI) { await generateData(newest); } +// Every `regex.mjs` is built at runtime from `ranges.mjs`; its source must be +// identical to what Regenerate produces for the same set of code points. +const assertRegexesMatchRegenerate = async (t, version) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${version}`); + let count = 0; + for (const type of fs.readdirSync(root)) { + const typeDir = path.join(root, type); + if (type.startsWith('.') || !fs.statSync(typeDir).isDirectory()) { + continue; + } + for (const name of fs.readdirSync(typeDir)) { + const dir = path.join(typeDir, name); + if (!fs.existsSync(path.join(dir, 'regex.mjs'))) { + continue; + } + const { default: ranges } = await import(path.join(dir, 'ranges.mjs')); + const { default: regex } = await import(path.join(dir, 'regex.mjs')); + const set = regenerate(); + for (const range of ranges) { + set.addRange(range.begin, range.end - 1); + } + t.assert.strictEqual(regex.source, set.toString(), `${type}/${name}`); + t.assert.strictEqual(regex.flags, '', `${type}/${name}`); + count++; + } + } + t.assert.ok(count > 0, 'found regex.mjs files'); +}; + +// Every mapping `symbols.mjs` must equal its `code-points.mjs` with each code +// point replaced by the corresponding symbol. +const assertSymbolsMatchCodePoints = async (t, version) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${version}`); + let count = 0; + for (const type of [ + 'Case_Folding', + 'Simple_Case_Mapping', + 'Special_Casing', + 'Decomposition_Mapping', + ]) { + const typeDir = path.join(root, type); + if (!fs.existsSync(typeDir)) { + continue; + } + for (const name of fs.readdirSync(typeDir)) { + const dir = path.join(typeDir, name); + const { default: codePoints } = await import( + path.join(dir, 'code-points.mjs') + ); + const { default: symbols } = await import(path.join(dir, 'symbols.mjs')); + const expected = [...codePoints].map(([from, to]) => [ + String.fromCodePoint(from), + String.fromCodePoint(...(Array.isArray(to) ? to : [to])), + ]); + t.assert.deepStrictEqual([...symbols], expected, `${type}/${name}`); + count++; + } + } + t.assert.ok(count > 0, 'found mapping directories'); +}; + suite(`The generated Unicode ${oldest} js`, () => { test('General_Category/Modifier_Letter should match the snapshot', async (t) => { const { default: data } = await import( @@ -30,6 +97,9 @@ suite(`The generated Unicode ${oldest} js`, () => { ); t.assert.snapshot(data); }); + test('every regex.mjs should match Regenerate output', async (t) => { + await assertRegexesMatchRegenerate(t, oldest); + }); }); suite(`The generated latest Unicode js`, () => { @@ -106,4 +176,83 @@ suite(`The generated latest Unicode js`, () => { ); t.assert.snapshot([...map.entries()]); }); + test('every regex.mjs should match Regenerate output', async (t) => { + await assertRegexesMatchRegenerate(t, newest); + }); + test('Names should decode to the expected names', async (t) => { + const { default: names } = await import( + `../output/unicode-${newest}/Names/index.mjs` + ); + const entries = [...names]; + t.assert.deepStrictEqual(entries[0], [0, '']); + t.assert.strictEqual(names.get(0x41), 'LATIN CAPITAL LETTER A'); + t.assert.strictEqual(names.get(0x4e00), 'CJK Ideograph'); + t.assert.strictEqual(names.get(0xac00), 'Hangul Syllable'); + t.assert.strictEqual(names.get(0xf900), 'CJK COMPATIBILITY IDEOGRAPH-F900'); + t.assert.strictEqual(names.get(0x1f600), 'GRINNING FACE'); + t.assert.deepStrictEqual(entries.at(-1), [ + 0x10fffd, + 'Plane 16 Private Use', + ]); + // Entries are in code point order. + for (let i = 1; i < entries.length; i++) { + if (entries[i][0] <= entries[i - 1][0]) { + t.assert.fail(`entry ${i} out of order`); + } + } + }); + test('index.d.mts should declare the default export of index.mjs', async (t) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${newest}`); + const { default: data } = await import(path.join(root, 'index.mjs')); + const declaration = fs.readFileSync(path.join(root, 'index.d.mts'), 'utf8'); + t.assert.match(declaration, /^export default data;$/m); + const declaredKeys = [ + ...declaration.matchAll(/^\treadonly (\w+): string\[\];$/gm), + ].map((m) => m[1]); + t.assert.deepStrictEqual(declaredKeys, Object.keys(data)); + }); + test('type declarations should be shared via typesVersions', async (t) => { + const root = path.resolve(__dirname, '..', 'output', `unicode-${newest}`); + const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'))); + // Every typesVersions target that is not a wildcard must exist. + for (const targets of Object.values(pkg.typesVersions['*'])) { + for (const target of targets) { + if (!target.includes('*')) { + t.assert.ok(fs.existsSync(path.join(root, target)), target); + } + } + } + for (const type of fs.readdirSync(root)) { + const typeDir = path.join(root, type); + if (type.startsWith('.') || !fs.statSync(typeDir).isDirectory()) { + continue; + } + const isMapping = Object.hasOwn( + pkg.typesVersions['*'], + `${type}/*/symbols.mjs`, + ); + for (const name of fs.readdirSync(typeDir)) { + const dir = path.join(typeDir, name); + if (!fs.existsSync(path.join(dir, 'symbols.mjs'))) { + continue; + } + // Mapping directories keep their own declarations; others share. + t.assert.strictEqual( + fs.existsSync(path.join(dir, 'symbols.d.mts')), + isMapping, + dir, + ); + t.assert.strictEqual( + fs.existsSync(path.join(dir, 'code-points.d.mts')), + isMapping, + dir, + ); + t.assert.ok(!fs.existsSync(path.join(dir, 'ranges.d.mts')), dir); + t.assert.ok(!fs.existsSync(path.join(dir, 'regex.d.mts')), dir); + } + } + }); + test('every mapping symbols.mjs should match code-points.mjs', async (t) => { + await assertSymbolsMatchCodePoints(t, newest); + }); });