From d47ea46126d7728a4437c10002f4f93489e03815 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Sat, 15 Aug 2026 23:39:19 +0000 Subject: [PATCH] fs: use sized reads for large files in readFileUtf8 fs.readFileSync(path, 'utf8') read the whole file in 8 KiB read() calls appended to a std::string, i.e. one syscall and a potential reallocation per 8 KiB (an 8 MiB file took ~1400 read() calls). Keep the exact old sequence for small files (one read into the 8 KiB stack buffer, one read reporting EOF). Once a read fills the stack buffer, read the rest directly into one heap buffer sized from fstat() (plus one byte so that the EOF read does not force growth), growing geometrically only when the size is unavailable or wrong. The size is only an allocation hint: reading continues until read() reports EOF, so procfs/sysfs files, FIFOs, files that change while being read and file descriptors positioned mid-file behave as before, and the bytes handed to StringBytes::Encode() are exactly the ones read. Signed-off-by: Shelley Vohr --- src/node_file.cc | 64 +++++++++++++- .../test-fs-readfilesync-utf8-sizes.js | 88 +++++++++++++++++++ 2 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 test/parallel/test-fs-readfilesync-utf8-sizes.js diff --git a/src/node_file.cc b/src/node_file.cc index ae0d9f34f8e1..d83284c6f147 100644 --- a/src/node_file.cc +++ b/src/node_file.cc @@ -2924,10 +2924,20 @@ static void ReadFileUtf8(const FunctionCallbackInfo& args) { uv_fs_req_cleanup(&req); }); + // Past the first 8 KiB, read into one heap buffer sized from fstat(); the + // size is only a hint, reading continues until read() reports EOF. std::string result{}; char buffer[8192]; uv_buf_t buf = uv_buf_init(buffer, sizeof(buffer)); + char* big = nullptr; + size_t big_len = 0; + size_t big_cap = 0; + bool sized = false; + auto free_big = OnScopeLeave([&big]() { free(big); }); + constexpr size_t kMinChunk = 64 * 1024; + constexpr size_t kMaxChunk = 8 * 1024 * 1024; + FS_SYNC_TRACE_BEGIN(read); while (true) { auto r = uv_fs_read(nullptr, &req, file, &buf, 1, -1, nullptr); @@ -2940,12 +2950,62 @@ static void ReadFileUtf8(const FunctionCallbackInfo& args) { if (r <= 0) { break; } - result.append(buf.base, r); + if (big == nullptr) { + result.append(buf.base, r); + if (static_cast(r) < sizeof(buffer)) { + continue; + } + // Switch to the heap buffer. + uv_fs_req_cleanup(&req); + big_cap = kMinChunk; + big = UncheckedMalloc(big_cap); + if (big == nullptr) { + FS_SYNC_TRACE_END(read); + return THROW_ERR_MEMORY_ALLOCATION_FAILED(env); + } + memcpy(big, result.data(), result.size()); + big_len = result.size(); + result = std::string(); + } else { + big_len += static_cast(r); + } + if (big_len == big_cap) { + // +1 leaves room for the read() that reports EOF. + size_t new_cap = + big_cap + std::min(kMaxChunk, std::max(kMinChunk, big_cap)); + if (!sized) { + sized = true; + uv_fs_req_cleanup(&req); + uv_fs_t stat_req; + if (uv_fs_fstat(nullptr, &stat_req, file, nullptr) == 0) { + const uv_stat_t* const st = + static_cast(stat_req.ptr); + if ((st->st_mode & S_IFMT) == S_IFREG && + static_cast(st->st_size) > big_len && + static_cast(st->st_size) < + static_cast(v8::String::kMaxLength)) { + new_cap = static_cast(st->st_size) + 1; + } + } + uv_fs_req_cleanup(&stat_req); + } + char* const grown = UncheckedRealloc(big, new_cap); + if (grown == nullptr) { + FS_SYNC_TRACE_END(read); + return THROW_ERR_MEMORY_ALLOCATION_FAILED(env); + } + big = grown; + big_cap = new_cap; + } + buf = uv_buf_init(big + big_len, std::min(kMaxChunk, big_cap - big_len)); } FS_SYNC_TRACE_END(read); Local val; - if (!ToV8Value(env->context(), result, isolate).ToLocal(&val)) { + const std::string_view content = big != nullptr + ? std::string_view(big, big_len) + : std::string_view(result); + if (!ToV8Value(env->context(), content, isolate).ToLocal(&val)) { return; } diff --git a/test/parallel/test-fs-readfilesync-utf8-sizes.js b/test/parallel/test-fs-readfilesync-utf8-sizes.js new file mode 100644 index 000000000000..37d434ad4195 --- /dev/null +++ b/test/parallel/test-fs-readfilesync-utf8-sizes.js @@ -0,0 +1,88 @@ +'use strict'; +// fs.readFileSync(path, 'utf8') takes a dedicated native path. Its result must +// equal fs.readFileSync(path).toString('utf8') for every file size (in +// particular around its internal 8 KiB stack buffer and for multi-megabyte +// files), for file descriptors positioned mid-file, and for files whose +// reported size is wrong (procfs reports 0, sysfs reports a page). +const common = require('../common'); +const tmpdir = require('../common/tmpdir'); +const assert = require('assert'); +const fs = require('fs'); + +tmpdir.refresh(); + +function content(size) { + // Multi-byte characters straddling every possible chunk boundary. + const unit = 'abcdé€\u{1F600}\n'; + let s = unit.repeat(Math.ceil(size / unit.length)); + s = s.slice(0, size); + // Avoid ending on a lone surrogate produced by slice(). + if (/[\ud800-\udbff]$/.test(s)) s = s.slice(0, -1) + 'x'; + return s; +} + +const sizes = [0, 1, 8190, 8191, 8192, 8193, 8194, 16383, 16384, 16385, + 65535, 65536, 65537, 100000, (1 << 20) - 1, 1 << 20, (1 << 20) + 1, + (8 << 20) + 5]; +for (const size of sizes) { + const file = tmpdir.resolve(`f-${size}.txt`); + const str = content(size); + fs.writeFileSync(file, str); + const expected = fs.readFileSync(file).toString('utf8'); + assert.strictEqual(fs.readFileSync(file, 'utf8'), expected, `size ${size} by path`); + assert.strictEqual(fs.readFileSync(file, { encoding: 'utf-8' }), expected, `size ${size} utf-8 alias`); + // By fd: from the start (leaves the fd at EOF), then at EOF, then from a + // mid-file position on a fresh fd. + let fd = fs.openSync(file, 'r'); + try { + assert.strictEqual(fs.readFileSync(fd, 'utf8'), expected, `size ${size} by fd`); + assert.strictEqual(fs.readFileSync(fd, 'utf8'), '', `size ${size} by fd at EOF`); + } finally { + fs.closeSync(fd); + } + if (size > 10) { + fd = fs.openSync(file, 'r'); + try { + // Advance the fd 3 bytes (inside the ASCII prefix, so still valid UTF-8). + assert.strictEqual(fs.readSync(fd, Buffer.alloc(3), 0, 3, null), 3); + assert.strictEqual(fs.readFileSync(fd, 'utf8'), Buffer.from(expected).subarray(3).toString('utf8'), + `size ${size} by fd at offset 3`); + } finally { + fs.closeSync(fd); + } + } +} + +// Binary garbage is decoded with replacement characters identically. +{ + const file = tmpdir.resolve('binary.bin'); + const buf = Buffer.alloc(20000); + for (let i = 0; i < buf.length; i++) buf[i] = (i * 7919) & 0xff; + fs.writeFileSync(file, buf); + assert.strictEqual(fs.readFileSync(file, 'utf8'), buf.toString('utf8')); +} + +// Files whose st_size does not describe their content. +if (common.isLinux) { + for (const file of ['/proc/self/status', '/proc/self/maps', '/proc/cpuinfo', + '/proc/version', '/sys/kernel/mm/transparent_hugepage/enabled']) { + let viaBuffer; + try { + viaBuffer = fs.readFileSync(file); + } catch { + continue; // Not available in this environment. + } + const viaUtf8 = fs.readFileSync(file, 'utf8'); + if (file !== '/proc/version' && file.startsWith('/proc/')) { + // Content legitimately differs between two reads; compare shape instead. + assert.ok(viaUtf8.length > 0); + assert.strictEqual(viaUtf8.split('\n').length > 5, true, file); + if (file === '/proc/self/maps') assert.ok(viaUtf8.length > 8192, 'maps should exceed one stack buffer'); + } else { + assert.strictEqual(viaUtf8, viaBuffer.toString('utf8'), file); + } + } +} + +// Directory: same error either way. +assert.throws(() => fs.readFileSync(tmpdir.path, 'utf8'), { code: 'EISDIR' });