From e925f720c8b3392fe876d01f2f43723557f40c05 Mon Sep 17 00:00:00 2001 From: 284km Date: Sat, 7 Nov 2026 14:20:20 +0900 Subject: [PATCH 1/2] Skip compaction of unchanged live string pages --- bench/README.md | 20 ++++++++++++ bench/unchanged_string_pages.rb | 12 +++++++ corpus/505_string_pages_across_collections.rb | 32 +++++++++++++++++++ m_state.mere | 15 +++++++-- 4 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 bench/unchanged_string_pages.rb create mode 100644 corpus/505_string_pages_across_collections.rb diff --git a/bench/README.md b/bench/README.md index 665dc22..682da53 100644 --- a/bench/README.md +++ b/bench/README.md @@ -391,6 +391,26 @@ changed from 0.352 seconds to 0.021 seconds. Peak mark-table entries in these runs changed from 301,767 to 1,772. Both builds print the same result. These numbers measure this collector workload, not the duration of the CI workflow. +## `unchanged_string_pages.rb` + +```sh +ruby bench/unchanged_string_pages.rb [strings] [collections] +./mere-ruby bench/unchanged_string_pages.rb [strings] [collections] +``` + +Keep 20,000 strings across 20 collections, check every value, mutate a subset, +and collect and check them again. Compaction now copies a live string page +only after allocation or mutation changed it; completely dead pages still +return to the pool. The mutation flag is separate from byte-accounting +dirtiness, which can be cleared between collections. Deferred compaction +retains the flag until compaction is safe. Retired arenas remain managed by +the runtime's global retirement list. + +On Apple silicon at O2 with Mere `991f299b`, compared with `a4eb486` +(2026-11-07), two runs changed the timed collections from 0.069–0.077 seconds +to 0.038–0.040 seconds. Both validation checks match Ruby 4.0.9. This measures +the collection workload, not whole-process or CI duration. + ## `exit_gc_workload.rb` ```sh diff --git a/bench/unchanged_string_pages.rb b/bench/unchanged_string_pages.rb new file mode 100644 index 0000000..bf9a29d --- /dev/null +++ b/bench/unchanged_string_pages.rb @@ -0,0 +1,12 @@ +# Keep strings across repeated collections, then change some old strings. +count = Integer(ARGV.fetch(0, '20000')) +rounds = Integer(ARGV.fetch(1, '20')) +strings = Array.new(count) { |i| ('x' * 256) + i.to_s } +GC.start +start = Process.clock_gettime(Process::CLOCK_MONOTONIC) +rounds.times { GC.start } +warn "collections=#{rounds} seconds=#{Process.clock_gettime(Process::CLOCK_MONOTONIC) - start}" +p strings.each_with_index.all? { |s, i| s == ('x' * 256) + i.to_s } +strings.each_with_index { |s, i| s.replace("changed#{i}") if i % 499 == 0 } +GC.start +p strings.each_with_index.all? { |s, i| s == (i % 499 == 0 ? "changed#{i}" : ('x' * 256) + i.to_s) } diff --git a/corpus/505_string_pages_across_collections.rb b/corpus/505_string_pages_across_collections.rb new file mode 100644 index 0000000..48c1c90 --- /dev/null +++ b/corpus/505_string_pages_across_collections.rb @@ -0,0 +1,32 @@ +strings = Array.new(9000) { |i| "value#{i}" } +empty = Array.new(50) { '' } +binary = "a\0\xff".b +held = Fiber.new do + strings[4096].each_char { |char| Fiber.yield(char) } + :done +end +first = held.resume +3.times { GC.start } +p strings.each_with_index.all? { |s, i| s == "value#{i}" } +p [empty.all?(&:empty?), binary.bytes] + +# Old pages must become eligible for compaction again after a mutation. +strings[0].replace('changed') +strings[4095] << ' appended' +strings[8192].clear +binary << "\x80".b +3.times { GC.start } +p [strings[0], strings[4095], strings[8192], binary.bytes] +rest = '' +while held.alive? + char = held.resume + rest << char unless char == :done +end +p first + rest + +# Drop the old values and exercise pages reused by fresh allocations. +strings = nil +GC.start +fresh = Array.new(9000) { |i| i.to_s } +GC.start +p fresh.each_with_index.all? { |s, i| s == i.to_s } diff --git a/m_state.mere b/m_state.mere index 7e43de4..8bed3c0 100644 --- a/m_state.mere +++ b/m_state.mere @@ -671,6 +671,9 @@ let str_count = fn (u: unit) -> vec_get str_n 0; // (str_pg_dq) and str_bytes measures just those, keeping the sum. let str_pg_bytes = vec_new (); let str_pg_dirty = vec_new (); +// Measurement dirtiness is cleared by str_bytes. Compaction needs its own +// mutation bit: an unchanged live page has no overwritten bytes to reclaim. +let str_pg_changed = vec_new (); let str_pg_dq = vec_new (); let str_pg_dq_n = vec_new (); let _ = vec_push str_pg_dq_n 0; @@ -697,7 +700,9 @@ let sput = fn (id: int) -> fn (s: str) -> let i = id - handle_base in let pg = vec_get str_store (i / 4096) in let o = i - (i / 4096) * 4096 in - if o < vec_len pg then (let _ = str_pg_touch (i / 4096) in vec_set pg o s) else (); + if o < vec_len pg then + (let _ = vec_set str_pg_changed (i / 4096) 1 in + let _ = str_pg_touch (i / 4096) in vec_set pg o s) else (); // (the sweep: surv holds the slots that lived, k of them -- a page with none // is wound back, the others compacted unless nc) let str_pg_live = vec_new (); @@ -719,8 +724,10 @@ let str_pages_sweep = fn surv -> fn (k: int) -> fn (nc: bool) -> let _ = vec_set str_pg_pool_n 0 (n9 + 1) in let _ = vec_set str_store p str_pg_empty in vec_set str_pg_dead p 1) - else if nc then () - else (let _ = str_pg_touch p in vec_compact (vec_get str_store p))) in + else if nc || vec_get str_pg_changed p == 0 then () + else (let _ = str_pg_touch p in + let _ = vec_compact (vec_get str_store p) in + vec_set str_pg_changed p 0)) in pw (p + 1))) in pw 0; // the bytes of the pages not wound back: the kept sum, after measuring the @@ -767,12 +774,14 @@ let mkstr = fn (s: str) -> vec_push str_store pg)) in let _ = vec_push str_pg_bytes 0 in let _ = vec_push str_pg_dirty 0 in + let _ = vec_push str_pg_changed 1 in let _ = vec_push str_pg_dead 0 in // (the page it fills next, and the one it filled, are measured) let _ = (if n > 0 then str_pg_touch (n / 4096 - 1) else ()) in str_pg_touch (n / 4096)) else ()) in let _ = vec_set str_pg_add 0 (vec_get str_pg_add 0 + str_len s) in + let _ = vec_set str_pg_changed (n / 4096) 1 in let _ = vec_push (vec_get str_store (n / 4096)) s in let _ = vec_set str_n 0 (n + 1) in VStr (handle_base + n); From 8c85dd5182994ad9c373c222642f3a31422416dd Mon Sep 17 00:00:00 2001 From: 284km Date: Sat, 7 Nov 2026 15:48:35 +0900 Subject: [PATCH 2/2] Retry string page compaction after native pin deferrals --- .github/workflows/ci.yml | 4 ++ bench/README.md | 12 ++--- m_state.mere | 11 ++++- unittest/string_page_compact_check.py | 63 +++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 unittest/string_page_compact_check.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26c6bd2..ac5625a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,8 @@ jobs: run: sh tools/build.sh - name: Record binary provenance run: python3 unittest/ci.py build-info --compiler .mere-compiler --output build-info.json + - name: Check deferred string-page compaction + run: python3 unittest/string_page_compact_check.py - uses: actions/upload-artifact@v4 with: name: mere-ruby-macos @@ -567,6 +569,8 @@ jobs: run: sh tools/build.sh - name: Check C nesting headroom run: MR_C=mr.c sh tools/bracket_depth_check.sh + - name: Check deferred string-page compaction + run: python3 unittest/string_page_compact_check.py - name: Smoke test run: | ulimit -s 524288 diff --git a/bench/README.md b/bench/README.md index 682da53..faf1b81 100644 --- a/bench/README.md +++ b/bench/README.md @@ -402,13 +402,15 @@ Keep 20,000 strings across 20 collections, check every value, mutate a subset, and collect and check them again. Compaction now copies a live string page only after allocation or mutation changed it; completely dead pages still return to the pool. The mutation flag is separate from byte-accounting -dirtiness, which can be cleared between collections. Deferred compaction -retains the flag until compaction is safe. Retired arenas remain managed by -the runtime's global retirement list. +dirtiness, which can be cleared between collections. The flag is cleared +only when compaction reduces the page's capacity: the compiler's unit result +does not distinguish a copy from a pin-induced deferral. Deferred and +equal-capacity compactions therefore remain pending for another collection. +Retired arenas remain managed by the runtime's global retirement list. On Apple silicon at O2 with Mere `991f299b`, compared with `a4eb486` -(2026-11-07), two runs changed the timed collections from 0.069–0.077 seconds -to 0.038–0.040 seconds. Both validation checks match Ruby 4.0.9. This measures +(2026-11-07), two runs changed the timed collections from 0.057–0.058 seconds +to 0.035–0.039 seconds. Both validation checks match Ruby 4.0.9. This measures the collection workload, not whole-process or CI duration. ## `exit_gc_workload.rb` diff --git a/m_state.mere b/m_state.mere index 8bed3c0..6b9105c 100644 --- a/m_state.mere +++ b/m_state.mere @@ -706,6 +706,13 @@ let sput = fn (id: int) -> fn (s: str) -> // (the sweep: surv holds the slots that lived, k of them -- a page with none // is wound back, the others compacted unless nc) let str_pg_live = vec_new (); +// vec_compact returns unit even when a native pin defers the copy. Only a +// strict capacity decrease proves reclamation with the current compiler API. +// Equal-capacity copies conservatively remain pending for another collection. +let str_pg_try_compact = fn pg -> + let before = vec_bytes pg in + let _ = vec_compact pg in + vec_bytes pg < before; let str_pages_sweep = fn surv -> fn (k: int) -> fn (nc: bool) -> let rec pz = fn (p: int) -> (if p >= vec_len str_store then () else (let _ = (if p < vec_len str_pg_live then vec_set str_pg_live p 0 else vec_push str_pg_live 0) in pz (p + 1))) in let _ = pz 0 in @@ -726,8 +733,8 @@ let str_pages_sweep = fn surv -> fn (k: int) -> fn (nc: bool) -> vec_set str_pg_dead p 1) else if nc || vec_get str_pg_changed p == 0 then () else (let _ = str_pg_touch p in - let _ = vec_compact (vec_get str_store p) in - vec_set str_pg_changed p 0)) in + if str_pg_try_compact (vec_get str_store p) + then vec_set str_pg_changed p 0 else ())) in pw (p + 1))) in pw 0; // the bytes of the pages not wound back: the kept sum, after measuring the diff --git a/unittest/string_page_compact_check.py b/unittest/string_page_compact_check.py new file mode 100644 index 0000000..347c747 --- /dev/null +++ b/unittest/string_page_compact_check.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Exercise the production compaction decision against the pinned Vec runtime.""" +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parent.parent +compiler = os.environ.get('MERE', str(ROOT / '.mere-compiler/_build/default/bin/mere.exe')) +state = (ROOT / 'm_state.mere').read_text() +# Compile the actual production helper, rather than duplicating its decision +# in a C model. The controlled pin exercises both native early-return paths. +helper = re.search(r'let str_pg_try_compact = fn pg ->.*?;', state, re.S) +assert helper, 'production compaction helper not found' +program = ''' +extern fn test_set_pin: int -> int; +let pg = vec_new (); +let _ = vec_push pg "kept"; +let _ = vec_compact pg; +let _ = vec_set pg 0 (str_repeat "x" 8388608); +let _ = vec_set pg 0 "kept"; +let before = vec_bytes pg; +let _ = print_bool (before > 8388608); +let _ = test_set_pin 1; +// First establish the native baseline; then defer for below-threshold growth. +let _ = print_bool (not (str_pg_try_compact pg)); +let _ = print_bool (not (str_pg_try_compact pg)); +let _ = print_bool (vec_bytes pg == before); +let _ = print_bool (vec_get pg 0 == "kept"); +let _ = test_set_pin 0; +// No intervening page mutation: the pending work must reclaim old storage. +let _ = print_bool (str_pg_try_compact pg); +let _ = print_bool (vec_bytes pg == 4096); +let _ = print_bool (vec_get pg 0 == "kept"); +// Equal-capacity copies cannot be distinguished from deferrals; retry safely. +let _ = print_bool (not (str_pg_try_compact pg)); +let _ = print_bool (vec_get pg 0 == "kept"); +''' +native = ''' +static int test_forced_pin(__lang_region *region, int growth) { + (void)region; (void)growth; + return 1; +} +int test_set_pin(int enabled) { + __lang_region_pinned = enabled ? test_forced_pin : 0; + return 0; +} +''' +with tempfile.TemporaryDirectory(prefix='mere-string-page-') as directory: + path = Path(directory) + source = path / 'check.mere' + source.write_text(helper[0] + '\n' + program) + emitted = subprocess.run([compiler, '-c', str(source)], capture_output=True, + check=True, timeout=60).stdout + cfile = path / 'check.c' + cfile.write_bytes(emitted + native.encode()) + binary = path / 'check' + subprocess.run(['clang', '-O2', '-w', '-fbracket-depth=1024', str(cfile), + '-lm', '-o', str(binary)], check=True, timeout=60) + result = subprocess.run([str(binary)], capture_output=True, check=True, timeout=20) + assert result.stdout == b'true\n' * 10, result.stdout +print('String page compaction: deferred pins retry and reclaim without another write')