From 99e14551ec920ff6df0fae22a46c179d431ad124 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 23:23:36 -0700 Subject: [PATCH 1/9] Stop is_initial matching CJK characters (#320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An initial is a single letter standing in for a name. v1's regex used \w, which is Unicode-aware, so a Han ideograph or hangul syllable plus a period matched -- and is_suffix_strict applies is_initial as a veto, which pushed period-written CJK honorifics ('씨.', '様.') out of the suffix vocabulary entirely. Split the shape test from the repertoire test rather than narrowing the character class. _INITIAL stays v1-verbatim, so its three pinned copies (config, _vocab, _render) stay byte-identical; _NO_INITIALS names the scripts whose characters cannot be an initial and is_initial composes the two. Narrowing to [A-Za-z] would have been wrong in the other direction -- it flips 'Й.' from initial to conjunction, the regression the 2.1.0 Ukrainian entry claims to prevent. Co-Authored-By: Claude Opus 5 --- nameparser/_pipeline/_vocab.py | 22 ++++++++++++++++++---- nameparser/_policy.py | 17 +++++++++++++++++ tests/v2/pipeline/test_vocab.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index d0e9b31..2e1e771 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -13,8 +13,8 @@ from collections.abc import Callable, Iterable from nameparser._lexicon import Lexicon, _normalize -from nameparser._policy import (Script, _JA_SCRIPTS, _SCRIPT_RANGES, - _script_matcher) +from nameparser._policy import (Script, _JA_SCRIPTS, _NO_INITIALS, + _SCRIPT_RANGES, _script_matcher) # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus # its empty-string alternative -- WorkToken text is never empty. Kept in @@ -51,10 +51,24 @@ # effective_script's kana license. _wholly_ja = _script_matcher(*_JA_SCRIPTS, whole=True) +# The repertoire half of is_initial (_policy._NO_INITIALS), kept apart +# from _INITIAL's SHAPE half so the pattern itself stays v1-verbatim +# and its three copies stay pinned by tests/v2/test_regex_sync.py. +# contains-any, not whole=True: the shape half has already admitted the +# trailing period, so the text reaching here is '씨.' rather than '씨' +# and a wholly-of match would be False for every case this exists for. +_in_initialless_script = _script_matcher(*_NO_INITIALS, whole=False) + def is_initial(text: str) -> bool: - """'A.' / 'j.' / bare capital -- v1's is_an_initial.""" - return bool(_INITIAL.fullmatch(text)) + """'A.' / 'j.' / bare capital -- v1's is_an_initial, narrowed to + scripts that HAVE initials (#320). v1's \\w is Unicode-aware and + matched CJK too, which vetoed period-written CJK honorifics ('씨.') + out of the suffix vocabulary -- and, downstream of that, left the + glued honorific in a name carrying such a token unpeeled + ('田中さん 様.').""" + return bool(_INITIAL.fullmatch(text)) \ + and not _in_initialless_script(text) def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool: diff --git a/nameparser/_policy.py b/nameparser/_policy.py index e8b92bd..b91507a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -142,6 +142,23 @@ class Script(StrEnum): #: quantify over this one union (HANGUL simply omitted). _JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) +#: Scripts whose characters cannot BE an initial. A Han ideograph, a +#: hangul syllable and a kana are each a morpheme or a syllable rather +#: than a letter, so a single one never stands in for a name the way +#: "J." stands in for "John". Membership is not "non-Latin": should a +#: Script.CYRILLIC or Script.DEVANAGARI ever be added, it does NOT +#: belong here -- those are alphabets, they have letters, and their +#: initials are real ("А. С. Пушкин", "م. الفارسي"). +#: +#: Enumerated rather than spelled *_SCRIPT_RANGES: the table admits a +#: script that DETERMINES A CONVENTION (see Script), which is a +#: different question from whether that script has initials. The four +#: members coinciding today is what has been implemented, not a +#: property of the enum -- a Thai entry (#317) must not inherit this +#: answer without someone deciding it. +_NO_INITIALS = (Script.HAN, Script.HANGUL, Script.HIRAGANA, + Script.KATAKANA) + def _script_matcher(*scripts: Script, whole: bool = False) -> Callable[[str], bool]: diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index b680eb8..1391630 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -22,6 +22,39 @@ def test_is_initial() -> None: assert not is_initial("b") # bare lowercase letter is not an initial +def test_is_initial_script_repertoire() -> None: + # An initial is a single LETTER standing in for a name. Alphabets + # have letters, so these are real initials ("А. С. Пушкин"). + assert is_initial("А.") # Cyrillic + assert is_initial("Α.") # Greek + assert is_initial("م.") # Arabic + assert is_initial("ה.") # Hebrew + assert is_initial("र.") # Devanagari + assert is_initial("Ա.") # Armenian + # Han ideographs, hangul syllables and kana are morphemes or + # syllables -- a single one never stands in for a name (#320). + assert not is_initial("씨.") + assert not is_initial("様.") + assert not is_initial("김.") + assert not is_initial("さ.") + assert not is_initial("ラ.") + # unchanged: a digit is the visible edge of \w's reach, and the + # shape half still owns it -- only the repertoire narrowed + assert is_initial("2.") + # unchanged: the SHAPE half still requires a single character + assert not is_initial("राम.") + + +def test_strict_suffix_veto_skips_cjk() -> None: + """#320: the initial veto is what stopped a period-written CJK + honorific being recognized. _normalize strips the trailing period, + so '씨.' reaches the vocabulary as '씨' -- the veto was the only + thing rejecting it.""" + lex = Lexicon(suffix_words=frozenset({"씨", "様"})) + assert is_suffix_strict("씨.", lex) + assert is_suffix_strict("様.", lex) + + def test_strict_suffix_initial_veto() -> None: assert is_suffix_strict("PhD", _LEX) assert not is_suffix_strict("V.", _LEX) # initial veto From 45eddb209ad844ab2c1c1f3ecc68782d3f4b472e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 23:38:27 -0700 Subject: [PATCH 2/9] Force new Script members to be classified for initials (#320) Adding a Script member already trips a chain of gates: test_policy's test_every_script_member_has_a_range_entry demands a codepoint range, and satisfying that trips test_regex_sync's two differential-rule gates. None of them asks whether the script has initials, which is a separate question from whether it determines an order or a segmentation -- and #317 (Thai) is a live candidate that would otherwise inherit the CJK answer silently. Every gate in that chain is satisfied by editing _SCRIPT_RANGES, the toml's character classes, or the four-member roster literal test_regex_sync spells out -- none of which requires answering the initials question. This one compares a hand-written classification table against set(Script), so it fires at the first step and stays failing through the rest. Verified by adding a temporary Script.THAI member -- 2 failed (this gate and test_policy); adding a range entry too -- 3 failed (this gate and both regex_sync gates). Co-Authored-By: Claude Opus 5 --- tests/v2/pipeline/test_vocab.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 1391630..bc087d2 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -5,7 +5,7 @@ effective_script, is_initial, is_suffix_lenient, is_suffix_strict, resolve_script_set, single_script, ) -from nameparser._policy import Script, _SCRIPT_RANGES +from nameparser._policy import Script, _NO_INITIALS, _SCRIPT_RANGES _LEX = Lexicon( suffix_acronyms=frozenset({"phd", "ma"}), @@ -55,6 +55,36 @@ def test_strict_suffix_veto_skips_cjk() -> None: assert is_suffix_strict("様.", lex) +def test_every_script_is_classified_for_initials() -> None: + """A member joining Script must be classified here on purpose; + _policy._NO_INITIALS carries the reasoning. + + The classification lives in this table rather than the assertion + being `set(Script) == set(_NO_INITIALS)`: that passes trivially + today, since all four current members are CJK, and the only way to + green it again after adding a script would be to declare that + script initial-less. That prejudges the answer. The point is to + force a decision, not a particular one. + """ + has_initials = { + Script.HAN: False, # ideographs are morphemes + Script.HANGUL: False, # syllable blocks + Script.HIRAGANA: False, # syllables + Script.KATAKANA: False, # syllables + } + assert set(has_initials) == set(Script), ( + "a Script member is unclassified for initials: decide whether " + "a single character of it can stand in for a name, add the row, " + "and put it in _policy._NO_INITIALS if it cannot") + assert {s for s, yes in has_initials.items() if not yes} \ + == set(_NO_INITIALS), ( + "this table and _policy._NO_INITIALS disagree about which " + "scripts have initials: a row here saying False is what puts a " + "script in the constant, so add the missing member to " + "_NO_INITIALS -- or, if the constant is the one that's right, " + "flip the row") + + def test_strict_suffix_initial_veto() -> None: assert is_suffix_strict("PhD", _LEX) assert not is_suffix_strict("V.", _LEX) # initial veto From 1ee9e6e653b07d73c348a68dcc3ec73fa9513a86 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 23:54:41 -0700 Subject: [PATCH 3/9] Pin the initial-repertoire behavior at parse level (#320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Cyrillic assertion is the load-bearing one: it asserts the TAG, because the naive narrowing changes 'Й.' from initial to conjunction without moving field output on a short name, which is how it passed a full green suite unnoticed. The collision is live, not hypothetical -- й ships as a default conjunction (Ukrainian, 60943ed for #267) -- so the test reads й from the module's _LEX rather than from a lexicon built in the test body, which would keep passing if the shipped entry were dropped. Two of the three case notes state a different mechanism than the one this change was expected to have. Measured rather than assumed: - '김민준 씨.' does not turn on segmentation, which divides 김민준 identically either way. effective_script('씨.') is None -- the trailing period defeats the wholly-one-script test -- so with '씨.' still a name piece the script_orders lookup declined for the whole name and the pieces fell back to name_order. - '田中さん, 様.' is FAMILY_COMMA before and after; SUFFIX_COMMA needs more than one word ahead of the comma. Both runs were always in the #312 peel's reach. What moved is is_suffix_strict's answer: the peel scan-back stopped at '様.' as the site rather than stepping over it. Each Korean row names the period-free sibling whose fields it must match and before this did not (ko_honorific_after_comma and ko_honorific_ssi); the Japanese row names ja_honorific_glued_before_an_initial, the intended veto it is the misfire of. corpus_cjk.jsonl is regenerated because it is derived from the case table (test_cjk_corpus_matches_the_case_table pins it). Case classifications are UNDETERMINED pending the differential run. Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 42 +++++++++++++++++++++++++++++ tests/v2/pipeline/test_classify.py | 19 ++++++++++++- tools/differential/corpus_cjk.jsonl | 3 +++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 48b619f..8f8be6e 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -773,6 +773,23 @@ def __post_init__(self) -> None: "Classification agrees with what classify does with " "the same token downstream -- 'V.' is a middle " "initial, not a post-nominal"), + Case("ja_honorific_period_does_not_stop_the_peel", "田中さん, 様.", + {"family": "田中", "suffix": "さん, 様."}, + classification="UNDETERMINED", + notes="#320's real cost: the veto did not merely misfile " + "'様.' -- it made _is_post_nominal say no, so the #312 " + "peel's scan-back stopped AT '様.' as the site instead " + "of stepping over it, found no listed tail there, and " + "abandoned the peel. One period and さん stayed glued " + "to 田中. Exactly the shape of " + "ja_honorific_glued_before_an_initial above, except " + "that here the token stopping the scan is a real " + "honorific rather than an initial, which is what makes " + "it a bug rather than the intended veto. The structure " + "is FAMILY_COMMA before and after -- SUFFIX_COMMA needs " + "more than one word ahead of the comma and 田中さん is " + "one -- so the two runs were always in the peel's " + "reach; only the strict test's answer moved"), Case("ja_sama_glued", "山田太郎様", {"family": "山田太郎", "suffix": "様"}, classification="fix(#308)", @@ -785,6 +802,31 @@ def __post_init__(self) -> None: {"family": "김", "given": "민준", "suffix": "님"}, classification="fix(#308)", notes="the online/formal glued address form, 씨's twin"), + Case("ko_honorific_written_with_a_period", "김민준, 씨.", + {"family": "김민준", "suffix": "씨."}, + classification="UNDETERMINED", + notes="the period-written form of ko_honorific_after_comma " + "('김민준, 씨'), whose fields it must match and before " + "#320 did not. _normalize strips the trailing period, " + "so the vocabulary sees 씨 either way -- the initial " + "veto was the only thing rejecting the written form, " + "and literally the veto: _is_suffix_piece is " + "'vocab:suffix' in tags and 'initial' not in tags, and " + "'씨.' carried both, so the suffix-shaped piece went to " + "the given"), + Case("ko_honorific_with_a_period_no_comma", "김민준 씨.", + {"given": "민준", "family": "김", "suffix": "씨."}, + classification="UNDETERMINED", + notes="the period-written ko_honorific_ssi ('김민준 씨'), and " + "#320 by a different route than the row above: the veto " + "left '씨.' a NAME piece, and effective_script('씨.') is " + "None because the trailing period defeats the " + "wholly-one-script test, so script_orders declined for " + "the whole name and the three pieces fell back to " + "name_order -- given 김, middle 민준, family '씨.'. " + "Hangul segmentation is NOT what moves: it divides " + "김민준 identically either way, and only the order the " + "pieces are read in changes"), Case("ko_honorific_glued_teacher", "김선생님", {"family": "김", "suffix": "선생님"}, classification="fix(#307)", diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py index ef0ace1..afb9647 100644 --- a/tests/v2/pipeline/test_classify.py +++ b/tests/v2/pipeline/test_classify.py @@ -14,7 +14,10 @@ suffix_acronyms_ambiguous=frozenset({"ma"}), particles=frozenset({"de", "la", "van"}), particles_ambiguous=frozenset({"van"}), - conjunctions=frozenset({"and", "y"}), + # й is a REAL default conjunction (Ukrainian, #267), carried here so + # test_cyrillic_initial_outranks_the_conjunction pins a live + # collision rather than a hypothetical one + conjunctions=frozenset({"and", "y", "й"}), bound_given_names=frozenset({"abdul"}), maiden_markers=frozenset({"née"}), ) @@ -55,6 +58,20 @@ def test_initial_tag() -> None: assert "initial" not in _tags(out, "John") +def test_cyrillic_initial_outranks_the_conjunction() -> None: + """#320 regression. 'й' is the Ukrainian conjunction (#267); 'Й.' + is an initial and must not be read as it. Narrowing is_initial to + [A-Za-z] -- the fix #320 originally proposed -- flips this token to + 'conjunction' and strips 'initial' off every Cyrillic, Greek, + Arabic and Hebrew initial. Neither moves field output on a short + name, so this asserts the TAG. _LEX carries й for this: the + collision is with SHIPPED vocabulary, so a lexicon built here + would keep passing if й were dropped from the defaults.""" + out = _classified("Й. Сліпий") + assert "initial" in _tags(out, "Й.") + assert "conjunction" not in _tags(out, "Й.") + + def test_ambiguous_suffix_acronym_needs_periods() -> None: out = _classified("M.A. Ma") assert "vocab:suffix" in _tags(out, "M.A.") diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 0d1d4d3..d43700b 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -45,6 +45,7 @@ "田中さん, PhD" "田中さん, V." "田中さん, 太郎" +"田中さん, 様." "田中博士" "諸葛亮" "阿明" @@ -63,7 +64,9 @@ "김민준 박사 씨" "김민준 박사님" "김민준 씨" +"김민준 씨." "김민준, 씨" +"김민준, 씨." "김민준님" "김민준박사님" "김민준씨" From 943208148ee116ddc052e5170ce9ffc52a243055 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 2 Aug 2026 00:12:38 -0700 Subject: [PATCH 4/9] Classify the three period-honorific rows against 1.4.0 (#320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured, not guessed. The differential harness reports 739 names, 94 intentional diffs, 0 unexplained, exit 0 -- and every Latin row is byte-identical to the pre-#320 run, as the design predicted: _INITIAL is untouched and no Latin token carries a CJK character. The only three rows the run gained are the three names Task 3 added to the CJK corpus, each already claimed by an existing expected_changes.toml rule (comma-family, suffix-routing, cjk-comma-compound), so the toml needed no edit. All three rows are fix(#320). What 1.4.0 returns, via the pinned worker: '김민준, 씨.' -> first '씨.' / last 김민준 '김민준 씨.' -> first 김민준 / last '씨.' '田中さん, 様.' -> first '様.' / last 田中さん For the two comma rows that is exactly what 2.0 produced with Tasks 1-3 reverted, so both sat at parity with 1.4.0 until this change; the veto is what moves them. The spaceless-comma row 김민준 씨. is the one judgement call: it already differed from 1.4.0 beforehand, as given 김 / middle 민준 / family '씨.', because hangul segmentation divides the name either way. But it did not differ as the fields the row now asserts -- the suffix and the family-first reading both arrive with the veto's removal -- so it is classified to #320 rather than to the segmenter, matching how ko_honorific_ssi is classified to #307 without naming the same segmentation it also depends on. Each row's notes now record what 1.4.0 did with the input. Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 8f8be6e..fecf85b 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -775,7 +775,7 @@ def __post_init__(self) -> None: "initial, not a post-nominal"), Case("ja_honorific_period_does_not_stop_the_peel", "田中さん, 様.", {"family": "田中", "suffix": "さん, 様."}, - classification="UNDETERMINED", + classification="fix(#320)", notes="#320's real cost: the veto did not merely misfile " "'様.' -- it made _is_post_nominal say no, so the #312 " "peel's scan-back stopped AT '様.' as the site instead " @@ -789,7 +789,10 @@ def __post_init__(self) -> None: "is FAMILY_COMMA before and after -- SUFFIX_COMMA needs " "more than one word ahead of the comma and 田中さん is " "one -- so the two runs were always in the peel's " - "reach; only the strict test's answer moved"), + "reach; only the strict test's answer moved. 1.4.0 read " + "this first '様.' / last 田中さん, which is exactly what " + "2.0 produced before this change -- the row sat at " + "parity until #320 moved it"), Case("ja_sama_glued", "山田太郎様", {"family": "山田太郎", "suffix": "様"}, classification="fix(#308)", @@ -804,7 +807,7 @@ def __post_init__(self) -> None: notes="the online/formal glued address form, 씨's twin"), Case("ko_honorific_written_with_a_period", "김민준, 씨.", {"family": "김민준", "suffix": "씨."}, - classification="UNDETERMINED", + classification="fix(#320)", notes="the period-written form of ko_honorific_after_comma " "('김민준, 씨'), whose fields it must match and before " "#320 did not. _normalize strips the trailing period, " @@ -813,10 +816,12 @@ def __post_init__(self) -> None: "and literally the veto: _is_suffix_piece is " "'vocab:suffix' in tags and 'initial' not in tags, and " "'씨.' carried both, so the suffix-shaped piece went to " - "the given"), + "the given. 1.4.0 read this first '씨.' / last 김민준 -- " + "the same fields 2.0 gave before this change, so the row " + "was at parity and #320 is what moves it"), Case("ko_honorific_with_a_period_no_comma", "김민준 씨.", {"given": "민준", "family": "김", "suffix": "씨."}, - classification="UNDETERMINED", + classification="fix(#320)", notes="the period-written ko_honorific_ssi ('김민준 씨'), and " "#320 by a different route than the row above: the veto " "left '씨.' a NAME piece, and effective_script('씨.') is " @@ -826,7 +831,14 @@ def __post_init__(self) -> None: "name_order -- given 김, middle 민준, family '씨.'. " "Hangul segmentation is NOT what moves: it divides " "김민준 identically either way, and only the order the " - "pieces are read in changes"), + "pieces are read in changes. 1.4.0 read this first " + "김민준 / last '씨.' -- undivided, no suffix. The only " + "row of the three that already differed from 1.4.0 " + "before this change, but it differed as given 김 / " + "middle 민준 / family '씨.'; the fields above are #320's, " + "not the segmenter's, so the row is classified to it -- " + "as ko_honorific_ssi is classified to #307 without " + "naming the same segmentation it also depends on"), Case("ko_honorific_glued_teacher", "김선생님", {"family": "김", "suffix": "선생님"}, classification="fix(#307)", From 522d7c04f1b7ec664686e18c34826ab2904fd325 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 2 Aug 2026 00:27:22 -0700 Subject: [PATCH 5/9] Record what #320 falsified in six doc places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The initial veto stopped being a claim about shape alone, and several places had been written as though it still was. AGENTS.md's suffix_not_acronyms/is_an_initial bullet is now marked Latin-only. It enumerated a collision that no longer happens for the seven single-character CJK members of suffix_not_acronyms -- 씨, 様, 氏, 군, 양, 님 and 殿. The last of those was missing from the working list this pass started from; enumerating six of seven in a bullet that exists to name a set is the failure being fixed, so it was checked against SUFFIX_NOT_ACRONYMS rather than transcribed. _NO_INITIALS is cited as what it is -- the four scripts that do NOT have initials -- rather than glossed backwards. The two "kept in sync by hand" comments now say what stays in sync. _vocab's records that "verbatim" is a promise about the PATTERN, not the predicate: since #320, _INITIAL.fullmatch(text) and is_initial(text) are different questions and '씨.' answers them differently, so the bare pattern is not the thing to ask. The copy stays as verbatim as it ever was -- config's REGEXES["initial"] still differs by the empty alternative's `?`, exactly as the line above it and test_regex_sync already said, so this is not and never was a byte-identical copy. _render's records that its copy is deliberately NOT composed with the repertoire test -- layering forbids the import, and its only call site is v1's conjunction carve-out, which this pattern reaches only once `normalized in lex.conjunctions` holds. Verified rather than assumed: the default lexicon carries no CJK conjunction or particle, and neither does JA, ZH, RU or TR_AZ. That is a property of shipped data and not an invariant, conjunctions being configurable public API, so the comment also records why the divergence stays harmless if a user breaks it: CJK is caseless, so word.lower() and word.capitalize() agree. _classify's contract line said "Policy is not consulted", which since #320 is true of the Policy dataclass but not of the _policy module -- is_initial reads its _NO_INITIALS constant. Narrowed to Policy FIELDS, which is the point the line was making. docs/modules.rst's public `initial` tag gains the repertoire qualifier. It described an initial-shaped word; the tag now also requires a script that has initials. The release-log entry says what an upgrader sees. Every example in it was run -- the seven honorifics, the "田中さん, 様." peel and its periodless twin, and the three alphabets that keep their initials (А. С. Пушкин, م. الفارسي, Ա. Խաչատրյան) -- and the field claims are per-spelling, the vetoed honorific landing in the family name in "김민준 씨." and the given name only after a comma. It quotes v1's regex as REGEXES["initial"] actually spells it, `?` included, since that is the public name a reader will look up. It also records the finding from the differential: measured against 1.4.0, "김민준, 씨." and "田中さん, 様." were returning exactly what 1.x returns, so #307/#308/#312 shipped with a hole wherever the honorific carried a period. The differential README gains the trap this branch fell into twice. compare.py invokes the worker by script path; inserting `python` before it makes python the command and the script an argument, so uv never reads the PEP 723 metadata and the 1.4 pin is never installed. With nothing to satisfy, uv runs it in the project .venv, where the working tree is installed editable -- 2.x answering every query under a 1.4.0 label. The README says so and says what does NOT explain it, because the wrong story points at a mitigation that fails: sys.path[0] is the script's own directory, which holds no nameparser, so this is not the CWD trap AGENTS.md documents three sections up and PYTHONSAFEPATH=1 does not rescue it (measured: safe path on, still 2.0.0). Only an absolute script path from outside the project fails loudly. It is worse than an ordinary mistake because the corrupted output is the 2.x expected values: every diff vanishes, the run comes out as parity, and the reader draws the opposite of the truth. The fix is to stop trusting a version number nobody made the worker report. Plus the zsh note: `compare.py | tail` leaves $? as tail's status and PIPESTATUS undefined (zsh's is the 1-indexed pipestatus), so a failing run reads as passing. Prose only; no behavior change. Suite 2894 passed / 16 skipped / 11 xfailed, ruff clean, mypy clean over 95 files, sphinx-build -W clean. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 2 +- docs/modules.rst | 3 +- docs/release_log.rst | 1 + nameparser/_pipeline/_classify.py | 4 ++- nameparser/_pipeline/_vocab.py | 10 ++++++ nameparser/_render.py | 12 +++++++ tools/differential/README.md | 52 +++++++++++++++++++++++++++++++ 7 files changed, 81 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9ede3b1..4c16719 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,7 +176,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Cyrillic suffix regexes need `re.I` even when the pattern is suffix-only** — a Latin title-cased word (`Ivanovich`) keeps its suffix lowercase, so `re.I` seemed skippable; but an irregular Cyrillic suffix can be nearly the whole word (`ильич`), so title-casing capitalizes into the suffix itself (`Ильич`). `east_slavic_patronymic_cyrillic` shipped without `re.I` on the Latin reasoning and silently failed on capitalized irregular forms — don't assume Latin's title-case safety transfers to Cyrillic. (#185) -**`suffix_not_acronyms` vs `is_an_initial` tension** — single-letter roman numeral suffixes (`i`, `v`) are in `suffix_not_acronyms` but also match the `is_an_initial` regex (single uppercase letter), so `is_suffix()` rejects them. The lenient test lives in `is_suffix_lenient()`, which accepts `suffix_not_acronyms` members unconditionally and is only safe in unambiguous positions: (1) suffix-comma detection uses it via `are_suffixes_after_comma()`; (2) lastname-comma post-comma parsing uses it inline, only when `nxt is None` and `len(parts)==2` (no `parts[2]` suffix segment). See issues #136, #144. +**`suffix_not_acronyms` vs `is_an_initial` tension** — single-letter roman numeral suffixes (`i`, `v`) are in `suffix_not_acronyms` but also match the `is_an_initial` regex (single uppercase letter), so `is_suffix()` rejects them. The lenient test lives in `is_suffix_lenient()`, which accepts `suffix_not_acronyms` members unconditionally and is only safe in unambiguous positions: (1) suffix-comma detection uses it via `are_suffixes_after_comma()`; (2) lastname-comma post-comma parsing uses it inline, only when `nxt is None` and `len(parts)==2` (no `parts[2]` suffix segment). See issues #136, #144. **The tension is Latin-only since #320** — the veto is now scoped to scripts that HAVE initials — `_vocab.is_initial` ANDs the shape test with "not in `_policy._NO_INITIALS`", that constant listing the four scripts which do NOT (Han, Hangul, Hiragana, Katakana) — so the seven single-character CJK members of `suffix_not_acronyms` (씨, 様, 氏, 군, 양, 님, 殿) no longer collide with it when written with a period: `"김민준 씨."` gives suffix `씨.` where the veto had made it the family name, and `"김민준, 씨."` where it had made it the given name. Latin is untouched — `V.`/`I.` still lose to the veto and still need `is_suffix_lenient()`. **Comparing against v1 needs `PYTHONSAFEPATH=1` AND a directory outside the worktree** — `uv run --isolated --no-project --with 'nameparser==1.4.0'` still puts the checkout's `nameparser/` ahead of the pinned wheel on `sys.path`, so the "v1" side silently imports the branch and every comparison reports parity. Run it as `cd && PYTHONSAFEPATH=1 uv run --isolated --no-project --with 'nameparser==1.4.0' python -c "..."`. This produces false confidence rather than an error, so it invalidates results without ever looking wrong. diff --git a/docs/modules.rst b/docs/modules.rst index 7f50dee..41755f3 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -50,7 +50,8 @@ Results "de"/"van", wherever it lands — combine with ``Role.FAMILY`` for actual family particles), ``conjunction`` (a joining word, "and"/"y"), ``initial`` (an - initial-shaped word, "J."), and ``joined`` (a continuation of the + initial-shaped word in a script that HAS initials — "J." or "А.", + never "씨."), and ``joined`` (a continuation of the previous token within one merged piece, so the suffix view renders "Ph. D." as one credential). Every other tag is namespaced (``vocab:...``) and unstable — never match against those. diff --git a/docs/release_log.rst b/docs/release_log.rst index 6eb8994..fc211f1 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -29,6 +29,7 @@ Release Log - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right on the SEGMENTER path, alongside declining the pack or the segmenter. It is no lever where the VOCABULARY divides the name, the two spellings agreeing exactly there — ``김민준 씨`` and ``김민준씨`` both give family 김, given 민준, suffix 씨, as do ``王小明 先生`` and ``王小明先生`` under the Chinese pack — and Korean data has no pack to decline either, hangul segmentation being on by default. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix``, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, which also matches its spaced form. ``田中さん, PhD`` and ``威廉·莎士比亚さん`` peel too; the first of those does *not* otherwise match its spaced form, since ``田中さん PhD`` leaves PhD in ``suffix`` beside さん while after a comma it reads as a ``title`` — where the credential lands is the comma's business, not the peel's. Previously each of these left the honorific inside the name. A comma no longer switches the peel off; what it does now is say which runs of the name to look in, and those are the two around a family comma, an honorific being as often glued to the given name as to the family. Anything past those two runs is out of reach, which is the one limit worth knowing: ``김, 민준 지훈씨`` peels, while ``김, 민준, 지훈씨`` (a third run) and ``김,, 민준씨`` (a doubled comma, which puts the name in a later run) do not. The reach also rests on the second run being name text, and a one-word part before the comma reads as a family comma even when the part after it is entirely suffix-shaped — so ``田中さん, V.`` keeps さん in the family name where ``田中さん, PhD`` gives it up. Same credential, opposite answer, and unchanged from 1.4.0 in that spelling. The 间隔号 does not switch the peel off either. Both marks say where a name divides into surname and given, and an honorific is not part of the name in either reading. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) + - Fix a period after a CJK honorific stopping it being recognized: ``씨.``, ``様.``, ``氏.``, ``님.``, ``군.``, ``양.`` and ``殿.`` now route to ``suffix`` like their periodless spellings, where the trailing period had left them inside the name — the family name in ``"김민준 씨."``, the given name in ``"김민준, 씨."``. The cause was v1's initial regex, ``^(\w\.|[A-Z])?$`` (``REGEXES["initial"]``, still public v1 API), whose ``\w`` is Unicode-aware and so matched a hangul syllable or a Han ideograph as readily as a letter; the strict suffix test applies that as a veto (``V.`` in ``"John V. Smith"`` is a middle initial, not roman five), and a veto written for Latin was being asked of scripts it was never about. The cost ran past the honorific itself: because the vetoed token read as name text, the glued-honorific peel's scan back for its site stopped at it instead of stepping over it, took it as the site, found no honorific at the end of it and gave up — so ``"田中さん, 様."`` kept ``さん`` inside the family name while ``"田中さん, 様"`` peeled it. Measured against 1.4.0, ``"김민준, 씨."`` and ``"田中さん, 様."`` were returning exactly what 1.x returns, so the honorific work earlier in this release had a hole in it wherever the honorific was written with a period. An initial is a single LETTER standing in for a name, and Han ideographs, hangul syllables and kana are morphemes and syllables rather than letters, so the veto now asks its question only of the scripts where it means something. Alphabets keep their initials untouched — ``"А. С. Пушкин"``, ``"م. الفارسي"`` and ``"Ա. Խաչատրյան"`` are unaffected, and so is the Ukrainian conjunction entry below, where a punctuated ``Й.`` still outranks the conjunction ``й``. The public ``initial`` tag follows the same line: ``씨.`` no longer carries it. **Default-on**, and it reaches ``HumanName`` too (#320) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 78bd988..a52132c 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -2,7 +2,9 @@ Consumes: tokens. Produces: tokens with vocabulary tags added (text/span/role unchanged). -Reads: every Lexicon vocabulary field; Policy is not consulted. +Reads: every Lexicon vocabulary field; no Policy FIELD is consulted +(is_initial does consult the _policy module's _NO_INITIALS constant, +which is not configuration -- nothing here varies by Policy value). Tags emitted -- stable (API): "particle", "conjunction", "initial"; namespaced (unstable): "vocab:title", "vocab:given-title", diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 2e1e771..74ed4db 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -19,6 +19,16 @@ # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus # its empty-string alternative -- WorkToken text is never empty. Kept in # sync by hand; layering forbids importing the config package here. +# "Verbatim" is a promise about the PATTERN, not about the predicate: +# since #320 is_initial is this SHAPE test ANDed with a repertoire test +# (_in_initialless_script, below), so _INITIAL.fullmatch(text) and +# is_initial(text) are no longer the same question -- '씨.' answers yes +# to the first and no to the second. Call is_initial; the bare pattern +# is not the thing to ask. The narrowing lives in the predicate +# precisely so this copy can stay exactly as verbatim as it ever was +# -- REGEXES["initial"] is public v1 API and cannot narrow, and the +# only difference between the two remains the empty alternative noted +# above (config's `?`), which test_regex_sync splices back in. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") # Ported verbatim from v1 (nameparser/config/regexes.py diff --git a/nameparser/_render.py b/nameparser/_render.py index ad236f6..702e8e7 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -38,6 +38,18 @@ # Ported verbatim from v1 (nameparser/config/regexes.py "initial", # minus the empty alternative) -- layering forbids importing the # pipeline here; keep in sync with _pipeline/_vocab.py by hand. +# Deliberately NOT composed with that module's repertoire test (#320): +# layering forbids the import, and nothing here needs it. The only use +# is v1's conjunction carve-out in _cap_word below, which this pattern +# can only reach once `normalized in lex.conjunctions` already holds -- +# and no CJK token reaches that, the shipped vocabulary carrying no CJK +# conjunction or particle in the default lexicon or in any locale pack. +# That is a property of the shipped DATA, not an invariant -- conjunctions +# is public, configurable API -- but the divergence stays harmless if a +# user adds one: CJK is caseless, so the carve-out's word.lower() and the +# fall-through's word.capitalize() return the same string either way. +# So the two copies keep identical PATTERNS and divergent PREDICATES; +# test_regex_sync pins the patterns, which is the promise being kept. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") diff --git a/tools/differential/README.md b/tools/differential/README.md index 54cedaa..f683c0d 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -38,6 +38,58 @@ needs widening. The run must exit 0 before a 2.0 release; the classified summary it prints is the source for the "Behavior Changes" section of `docs/release_log.rst`. +## Do not put `python` in front of the worker + +`compare.py` spawns the worker by **script path**: + +``` +uv run --no-project tools/differential/worker_v1.py +``` + +Inserting `python` before the path -- +`uv run --no-project python tools/differential/worker_v1.py` -- makes +`python` the command and the script a mere argument, so `uv` never +reads the script's PEP 723 inline metadata and the `nameparser==1.4.*` +pin is never installed. With nothing to satisfy, `uv` runs the script +in the project's own `.venv`, where the working tree is installed +editable (`__editable__.nameparser-2.0.0.pth`) -- so the import +resolves to the checkout and **2.x answers every query while the +output is labelled 1.4.0**. Reproduced twice while working on #320. + +It is the same editable working tree that the missing-`--no-project` +case above lands on, by a different road. **`PYTHONSAFEPATH=1` does +not rescue it** -- that is the fix for the sibling CWD trap +(`AGENTS.md`, "Comparing against v1"), and reaching for it here is the +natural wrong turn, since a `.pth`-installed package is on `sys.path` +proper and safe-path never touches it. Measured: safe path on, still +2.0.0. `sys.path[0]` is the SCRIPT's directory +(`tools/differential/`), which contains no `nameparser` at all, so the +CWD is not the route either. Running the same command with an absolute +script path from a directory outside the project is the one variant +that does not lie: it raises `ModuleNotFoundError` instead. + +That is worse than an ordinary mistake, because of what the corrupted +output looks like. It is not garbage and it does not crash: it is +exactly the 2.x expected values, which is exactly what someone asking +"did 1.4.0 agree?" is hoping to see. Every diff vanishes, the run +comes out as parity, and the conclusion drawn is the precise opposite +of the truth. Same outcome as the missing `--no-project` above, and +the same reason both are written down here rather than left to the +reader to rediscover. + +So do not trust a 1.4 version number you did not make the worker +report. Establishing which library actually answered is cheap -- print +`nameparser.__version__` from **inside** the worker and check it +against the pin before comparing anything. Under this trap it prints +the checkout's version, which is the whole tell. + +One shell note while you are here: `compare.py | tail` swallows the +exit code under zsh. `$?` after a pipeline is `tail`'s status, and +`PIPESTATUS` is a bash array zsh does not define at all -- zsh's own +is the 1-indexed `pipestatus`, so `${PIPESTATUS[0]}` is the empty +string and a failing run reads as a passing one. Redirect to a file +and read the file instead of piping. + ## The three corpora `compare.py` reads **every** `corpus*.jsonl` beside it by default From a3c31f6cf4bc6611309898c13ea10e421d4bc9f7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 2 Aug 2026 10:49:13 -0700 Subject: [PATCH 6/9] Ask the roman-numeral fork for the SHAPE, and make the gate behavioral MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from the branch review, both about the same seam between "initial-shaped" and "is an initial". assign's roman-numeral fork asks whether the piece before a trailing single roman letter was an initial -- i.e. whether we are mid-run and the letter is a name part rather than roman five. It asked through the "initial" TAG, which #320 narrowed from initial-SHAPED to actually an initial. The fork only ever needed the shape, and the narrowing cost 'John 씨. V' its family name outright (given John, family ''), while 'John 김. V' turned a middle initial and a family name into a family name and a suffix. Blast radius was exactly that: last piece a bare roman numeral, preceding piece one CJK character plus a period. _vocab now names the two questions separately -- is_initial_shaped is v1's is_an_initial verbatim, is_initial is that ANDed with the repertoire test -- and assign calls the shape half on the token it used to read tags from. One shared definition rather than a hand copy: the repo has precedent for copied patterns (_assign._PERIOD_ABBREV) but each one buys a test_regex_sync obligation. All five probed inputs ('John 씨. V', 'John 김. V', 'John 田. X', 'John 김. V, PhD', 'John 김. 김. V') are back to their 1.4.0/master fields, and the #320 honorific fixes are untouched. The narrowed tag is NOT inert after this, which the review expected it to be. Measured over all three differential corpora plus every case- table row, with _classify's tag restored to the shape test: six field diffs, all of them #320's own fixes. The route is _group's _is_suffix_piece ('vocab:suffix' in tags and 'initial' not in tags) -- but as _assign imports it, in the trailing-suffix peel. Both call sites INSIDE _group (_is_rootname, and _is_suffix_piece reached through it) move nothing. Second finding: test_every_script_is_classified_for_initials compared the table to Script and its False rows to _NO_INITIALS, and never to is_initial. A Script.THAI declared with a private-use range, put in _NO_INITIALS with a False row, passed the whole test while is_initial('ก.') still returned True -- the declaration contradicted and nothing noticed. It also made the test the one on this branch on the wrong side of the no-constant-content rule. It now derives a representative character from that script's own _SCRIPT_RANGES entry and asserts is_initial on it, matching the row. Shape-admitted rather than simply the first codepoint, because several ranges open on an unassigned or punctuation character (HIRAGANA's 0x3040, KATAKANA's 0x30A0) that \w rejects for the shape's reason, which would make the repertoire assertion vacuous; a script with no initial-shaped character anywhere in its spans fails outright. Reproduced: under the scenario above the test now fails with "no character in _SCRIPT_RANGES[thai] is initial-SHAPED". Co-Authored-By: Claude Opus 5 --- nameparser/_pipeline/_assign.py | 12 ++++++-- nameparser/_pipeline/_vocab.py | 19 ++++++++++-- tests/v2/pipeline/test_vocab.py | 54 ++++++++++++++++++++++++++++++++- 3 files changed, 80 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 0407736..9e63cf9 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -31,7 +31,8 @@ import re from nameparser._pipeline._vocab import ( - effective_script, is_suffix_lenient, resolve_script_set, + effective_script, is_initial_shaped, is_suffix_lenient, + resolve_script_set, ) from nameparser._pipeline._group import ( _is_suffix_piece, _is_title_piece, @@ -200,9 +201,16 @@ def _assign_main(seg_idx: int, state: ParseState, if _is_suffix_piece(piece, tags, tokens): k -= 1 continue + # is_initial_shaped, not the "initial" tag: this asks whether + # the preceding piece looks like part of an initial run, which + # is a question about layout, and #320 narrowed the tag to + # initials that can really stand in for a name. Reading the tag + # here made '씨.' stop suppressing the fork and cost 'John 씨. V' + # its family name. if (k == len(rest) and k >= 2 and len(piece) == 1 and _ROMAN.match(tokens[piece[0]].text) - and "initial" not in tokens[pieces[rest[k - 2]][0]].tags): + and not is_initial_shaped( + tokens[pieces[rest[k - 2]][0]].text)): # a trailing single letter is a name part unless it happens # to be a roman numeral -- and V/X/I are ordinary middle # initials, so taking it as a suffix is a call, not a fact diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 74ed4db..eb75481 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -70,6 +70,22 @@ _in_initialless_script = _script_matcher(*_NO_INITIALS, whole=False) +def is_initial_shaped(text: str) -> bool: + """v1's is_an_initial verbatim: the SHAPE half alone -- one word + character plus a period, or a bare ASCII capital. + + Callers asking whether a token is STRUCTURALLY part of an initial + run want this; callers asking whether it can really stand in for a + name want is_initial (#320). The two answers differ only inside + _NO_INITIALS scripts, where '씨.' is initial-SHAPED but is not an + initial. assign's roman-numeral fork is the shape caller: "the + piece before this trailing single roman letter looks like an + initial, so we are mid-run and the letter is a name part" was + always a question about layout, and narrowing it to real initials + dropped the family name out of 'John 씨. V'.""" + return bool(_INITIAL.fullmatch(text)) + + def is_initial(text: str) -> bool: """'A.' / 'j.' / bare capital -- v1's is_an_initial, narrowed to scripts that HAVE initials (#320). v1's \\w is Unicode-aware and @@ -77,8 +93,7 @@ def is_initial(text: str) -> bool: out of the suffix vocabulary -- and, downstream of that, left the glued honorific in a name carrying such a token unpeeled ('田中さん 様.').""" - return bool(_INITIAL.fullmatch(text)) \ - and not _in_initialless_script(text) + return is_initial_shaped(text) and not _in_initialless_script(text) def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool: diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index bc087d2..15a1c7b 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -2,7 +2,7 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( - effective_script, is_initial, is_suffix_lenient, + effective_script, is_initial, is_initial_shaped, is_suffix_lenient, is_suffix_strict, resolve_script_set, single_script, ) from nameparser._policy import Script, _NO_INITIALS, _SCRIPT_RANGES @@ -45,6 +45,20 @@ def test_is_initial_script_repertoire() -> None: assert not is_initial("राम.") +def test_is_initial_shaped_keeps_the_shape_half_reachable() -> None: + """The two halves are separately askable (#320): assign's + roman-numeral fork asks the SHAPE question about the piece before a + trailing 'V', and answering it with the narrowed predicate dropped + the family name out of 'John 씨. V' entirely.""" + for text in ("A.", "j.", "B", "2."): + assert is_initial_shaped(text) is is_initial(text) is True + for text in ("Jo", "b", "raam."): + assert is_initial_shaped(text) is is_initial(text) is False + # the whole difference between them, in both directions + for text in ("씨.", "様.", "김.", "さ.", "ラ."): + assert is_initial_shaped(text) and not is_initial(text) + + def test_strict_suffix_veto_skips_cjk() -> None: """#320: the initial veto is what stopped a period-written CJK honorific being recognized. _normalize strips the trailing period, @@ -55,6 +69,27 @@ def test_strict_suffix_veto_skips_cjk() -> None: assert is_suffix_strict("様.", lex) +def _representative(script: Script) -> str: + """The first codepoint of `script`'s _SCRIPT_RANGES spans that the + SHAPE half admits as an initial. Shape-admitted, not simply the + first codepoint: a range's first codepoint is often unassigned or + punctuation (KATAKANA's 0x30A0 is a hyphen, HIRAGANA's 0x3040 is + unassigned), which \\w does not match -- and testing is_initial on + such a character answers False for the SHAPE's reason, making the + repertoire assertion below vacuously green. Raising when no span + holds one is the point rather than a corner: a script whose + declared ranges contain no initial-shaped character at all has + ranges that do not describe it.""" + for lo, hi in _SCRIPT_RANGES[script]: + for cp in range(lo, hi + 1): + if is_initial_shaped(chr(cp) + "."): + return chr(cp) + raise AssertionError( + f"no character in _SCRIPT_RANGES[{script}] is initial-SHAPED, " + f"so this script's declaration cannot be tested against " + f"is_initial -- check that the ranges are that script's") + + def test_every_script_is_classified_for_initials() -> None: """A member joining Script must be classified here on purpose; _policy._NO_INITIALS carries the reasoning. @@ -65,6 +100,14 @@ def test_every_script_is_classified_for_initials() -> None: green it again after adding a script would be to declare that script initial-less. That prejudges the answer. The point is to force a decision, not a particular one. + + Three bindings, not two: the table covers Script, the table's + False rows are _NO_INITIALS, and -- the one that makes this a + behavioral test rather than a comparison of two constants -- each + row is checked against is_initial on a character DERIVED from that + script's own _SCRIPT_RANGES entry. Without the third, a script + declared initial-less under ranges that are not its own passes all + the way through while is_initial still says yes to its characters. """ has_initials = { Script.HAN: False, # ideographs are morphemes @@ -83,6 +126,15 @@ def test_every_script_is_classified_for_initials() -> None: "script in the constant, so add the missing member to " "_NO_INITIALS -- or, if the constant is the one that's right, " "flip the row") + for script, yes in has_initials.items(): + char = _representative(script) + assert is_initial(char + ".") is yes, ( + f"the declaration for {script} does not reach is_initial: " + f"the row says has_initials={yes}, but is_initial(" + f"{char + '.'!r}) -- on a character taken from " + f"_SCRIPT_RANGES[{script}] -- says {not yes}. Either the " + f"ranges are not this script's, or _NO_INITIALS and the " + f"repertoire predicate have come apart") def test_strict_suffix_initial_veto() -> None: From 08e61f6d87b1da4d4ddee5f6124a293b1164fadd Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 2 Aug 2026 10:53:57 -0700 Subject: [PATCH 7/9] Pin four unpinned #320 shapes, including the strict-knob path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fix moves 24 of 52 probed input shapes and the branch pinned three. Four more rows, all measured before they were written, and all classified against 1.4.0 measured through the pinned worker (which reported 1.4.0, per the README's warning about the invocation that silently answers 2.x). - ja_honorific_with_a_period_no_comma, '田中さん 様.'. The SPACED form is the example is_initial's own docstring cites, and it was nowhere in the suite -- the existing row pins the COMMA form, which reaches the peel by flattening two runs where this one has a single run, so that row can stay green through a change that breaks this one. 1.4.0: first 田中さん, last '様.' -- what 2.0 gave until #320. - ko_honorific_period_under_strict_comma_suffixes, '김민준, 씨.' under Policy(lenient_comma_suffixes=False). The table had no lenient_comma_suffixes row at all, and this is the knob's own shape: a post-comma suffix that is initial-SHAPED. '씨.' was in that class by shape and the knob decided it; it is out of the class now and the honorific parses identically under both settings, which is what the row holds -- _script_segment names this setting as the one that strands 씨 in a neighbouring shape, and the knob's documentation scopes it to 'I' and 'V'. The knob has no v1 spelling, so the facade runner skips the row and the classification compares against 1.4.0's single reading (first '씨.', last 김민준). - ko_honorific_yang_trails / ko_honorific_yang_written_with_a_period, '김민준 양' and '김민준 양.'. 양 is what suffixes.py singles out as the shipped vocabulary's risk class -- a top-tier surname admitted on the strength of trailing position alone -- and only its LEADING reading was pinned. Both spellings now agree, so a future tightening of the 양/군 policy cannot move one without the other. 군 gets no pair: it parses identically and carries no surname reading, so it would pin nothing these two do not. corpus_cjk.jsonl regenerates from the case table (test_regex_sync pins it). The differential run stays at unexplained: 0 -- the three new CJK names classify against existing rules, 田中さん 様. and 김민준 양. under fix(suffix-routing) and 김민준 양 under fix(cjk-honorific-suffix); no rule was added or widened. Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 69 +++++++++++++++++++++++++++++ tools/differential/corpus_cjk.jsonl | 3 ++ 2 files changed, 72 insertions(+) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index fecf85b..e2284c5 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -773,6 +773,21 @@ def __post_init__(self) -> None: "Classification agrees with what classify does with " "the same token downstream -- 'V.' is a middle " "initial, not a post-nominal"), + Case("ja_honorific_with_a_period_no_comma", "田中さん 様.", + {"family": "田中", "suffix": "さん, 様."}, + classification="fix(#320)", + notes="the SPACED form, and the example _vocab.is_initial's " + "own docstring cites as what #320 cost. Same fields as " + "the comma row below, reached without a comma: the peel " + "scans segment 0 either way, and with '様.' no longer " + "vetoed the scan-back steps over it onto 田中さん and " + "peels さん. Worth its own row because the comma form " + "arrives through a different branch -- FAMILY_COMMA " + "flattens TWO runs before scanning, this one has a " + "single run -- so the comma row can stay green through " + "a change that breaks this one. 1.4.0 read this first " + "田中さん / last '様.', which is what 2.0 produced until " + "#320: parity before, a classified change after"), Case("ja_honorific_period_does_not_stop_the_peel", "田中さん, 様.", {"family": "田中", "suffix": "さん, 様."}, classification="fix(#320)", @@ -819,6 +834,29 @@ def __post_init__(self) -> None: "the given. 1.4.0 read this first '씨.' / last 김민준 -- " "the same fields 2.0 gave before this change, so the row " "was at parity and #320 is what moves it"), + Case("ko_honorific_period_under_strict_comma_suffixes", "김민준, 씨.", + {"family": "김민준", "suffix": "씨."}, + policy=Policy(lenient_comma_suffixes=False), + classification="fix(#320)", + notes="the row above under the knob that governs exactly this " + "shape, and the table's only exercise of it. " + "lenient_comma_suffixes=False drops segment's post-comma " + "test to the strict one, so a 'Family, Suffix' input " + "whose suffix is INITIAL-SHAPED reads as a given-name " + "initial instead ('John Smith, V' -> given 'V'). '씨.' " + "is a single character plus a period, so it was in that " + "class by shape, and before #320 the knob decided it: " + "given '씨.' / family 김민준. It is out of the class now, " + "and the honorific parses identically under both " + "settings -- which is the claim this row exists to " + "hold, since the knob's own documentation scopes it to " + "the roman numerals 'I' and 'V' and _script_segment " + "names it as the setting that strands 씨 in a " + "neighbouring shape. No v1 spelling exists for the knob " + "(the facade runner skips this row), so the " + "classification compares against 1.4.0's single " + "reading, first '씨.' / last 김민준 -- the same fields " + "2.0 gave under EITHER setting before this change"), Case("ko_honorific_with_a_period_no_comma", "김민준 씨.", {"given": "민준", "family": "김", "suffix": "씨."}, classification="fix(#320)", @@ -1154,6 +1192,37 @@ def __post_init__(self) -> None: "Classified to #271, not parity: 1.4 gave first 양, " "last 미선, and it is the CJK order flip that swaps " "them"), + Case("ko_honorific_yang_trails", "김민준 양", + {"family": "김", "given": "민준", "suffix": "양"}, + classification="fix(#307)", + notes="the other side of ko_surname_yang_leads: the same " + "token trailing a name is 'Miss', and that is the whole " + "argument shipping it -- suffixes.py singles 양 out " + "(with 군) as the risk class it takes, a top-tier " + "surname admitted to the vocabulary on the strength of " + "position alone. Nothing pinned the trailing reading " + "before this row, so the leading rows carried the pair " + "by themselves. Classified to #307 like ko_honorific_ssi " + "(1.4 gave first 김민준, last 양; the recognition and " + "the order flip both move it) -- the point of the row " + "is the twin below"), + Case("ko_honorific_yang_written_with_a_period", "김민준 양.", + {"family": "김", "given": "민준", "suffix": "양."}, + classification="fix(#320)", + notes="the period-written twin, whose fields must equal the " + "row above and before #320 did not (given 김, middle " + "민준, family '양.' -- the veto kept '양.' a name piece, " + "exactly ko_honorific_with_a_period_no_comma's route). " + "The pair is the point: 양 is the shipped vocabulary's " + "acknowledged risk, so if the 양/군 policy is ever " + "tightened or withdrawn, both spellings have to move " + "together and neither row can be adjusted alone. 군 " + "gets no pair of its own -- it parses identically and " + "is the SAFER half (no surname reading), so it would " + "pin nothing these two do not. Classified to #320 like " + "its 씨 counterpart: 1.4.0 read this first 김민준 / last " + "'양.', and the fields above are the ones this change " + "produced, not the segmenter's"), Case("ko_surname_yang_leads_a_segmentable_given", "양 지훈", {"family": "양", "given": "지훈"}, classification="fix(#271)", diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index d43700b..357a4d9 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -42,6 +42,7 @@ "田中さん" "田中さん II" "田中さん V." +"田中さん 様." "田中さん, PhD" "田中さん, V." "田中さん, 太郎" @@ -65,6 +66,8 @@ "김민준 박사님" "김민준 씨" "김민준 씨." +"김민준 양" +"김민준 양." "김민준, 씨" "김민준, 씨." "김민준님" From 2e9bee4efc3333e68a79b033b6ed5849980c17d9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 2 Aug 2026 11:01:33 -0700 Subject: [PATCH 8/9] Say what #320 actually narrowed, and why MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six prose corrections from the review pass; no behavior change. _types.py's STABLE_TAGS comment is the hand-maintained twin of docs/modules.rst's block, which the branch updated and this one did not -- it still promised the "initial" tag for any initial-SHAPED word, which since #320 is false for CJK. Nothing pins the two copies against each other, so the comment now says so. is_initial's docstring named the wrong mechanism: measured, '씨.' carried vocab:suffix and vocab:suffix-word before the fix as after, and is_suffix_lenient took it either way -- the veto rejected is_suffix_strict alone, via _is_suffix_strict_n. cases.py already had this right, so the docstring was contradicting the row it explains. _NO_INITIALS' rationale argued from two things that do not hold. It justified membership phonologically ("a syllable, not a letter"), but Devanagari is an abugida and Arabic an abjad and the branch's own tests assert both keep their initials -- so the stated criterion would have put Devanagari IN. The operative question is orthographic: does the tradition abbreviate a given name to one character plus a period? That matters concretely for Thai (#317), an abugida the old wording would misfile. And the enum does NOT admit only scripts that determine a convention -- KATAKANA is in it so the classifier can name what it declines, and neither DEFAULT_SCRIPT_ORDERS nor segment_scripts' default mentions it. The conclusion (do not derive _NO_INITIALS from the enum) survives on the better ground that membership is granted on assorted grounds. The release note claimed the fix for "a period", true only of U+002E: _normalize strips the ASCII period alone, so 씨． (U+FF0E), 씨。 (U+3002) and 씨｡ (U+FF61) miss the vocabulary lookup before the veto is reached, and all three parse identically before and after this branch. Scoped, with the still-open _normalize gap named. Also: test_classify's fixture comment and its docstring made opposite claims about whether й shipping is load-bearing (the docstring was right -- _LEX is local); that docstring named four scripts where the sibling test asserts six; cases.py said "fields" where only the field ASSIGNMENT matches; test_vocab called a digit "the" edge of \w when _ is equally inside it; and AGENTS.md's suffix_not_acronyms bullet now says which of the two predicates the tension is about. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 2 +- docs/release_log.rst | 2 +- nameparser/_pipeline/_vocab.py | 10 +++++---- nameparser/_policy.py | 33 ++++++++++++++++++------------ nameparser/_types.py | 6 +++++- tests/v2/cases.py | 6 ++++-- tests/v2/pipeline/test_classify.py | 17 ++++++++------- tests/v2/pipeline/test_vocab.py | 5 +++-- 8 files changed, 50 insertions(+), 31 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4c16719..0124076 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,7 +176,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Cyrillic suffix regexes need `re.I` even when the pattern is suffix-only** — a Latin title-cased word (`Ivanovich`) keeps its suffix lowercase, so `re.I` seemed skippable; but an irregular Cyrillic suffix can be nearly the whole word (`ильич`), so title-casing capitalizes into the suffix itself (`Ильич`). `east_slavic_patronymic_cyrillic` shipped without `re.I` on the Latin reasoning and silently failed on capitalized irregular forms — don't assume Latin's title-case safety transfers to Cyrillic. (#185) -**`suffix_not_acronyms` vs `is_an_initial` tension** — single-letter roman numeral suffixes (`i`, `v`) are in `suffix_not_acronyms` but also match the `is_an_initial` regex (single uppercase letter), so `is_suffix()` rejects them. The lenient test lives in `is_suffix_lenient()`, which accepts `suffix_not_acronyms` members unconditionally and is only safe in unambiguous positions: (1) suffix-comma detection uses it via `are_suffixes_after_comma()`; (2) lastname-comma post-comma parsing uses it inline, only when `nxt is None` and `len(parts)==2` (no `parts[2]` suffix segment). See issues #136, #144. **The tension is Latin-only since #320** — the veto is now scoped to scripts that HAVE initials — `_vocab.is_initial` ANDs the shape test with "not in `_policy._NO_INITIALS`", that constant listing the four scripts which do NOT (Han, Hangul, Hiragana, Katakana) — so the seven single-character CJK members of `suffix_not_acronyms` (씨, 様, 氏, 군, 양, 님, 殿) no longer collide with it when written with a period: `"김민준 씨."` gives suffix `씨.` where the veto had made it the family name, and `"김민준, 씨."` where it had made it the given name. Latin is untouched — `V.`/`I.` still lose to the veto and still need `is_suffix_lenient()`. +**`suffix_not_acronyms` vs `is_an_initial` tension** — single-letter roman numeral suffixes (`i`, `v`) are in `suffix_not_acronyms` but also match the `is_an_initial` regex (single uppercase letter), so `is_suffix()` rejects them. The lenient test lives in `is_suffix_lenient()`, which accepts `suffix_not_acronyms` members unconditionally and is only safe in unambiguous positions: (1) suffix-comma detection uses it via `are_suffixes_after_comma()`; (2) lastname-comma post-comma parsing uses it inline, only when `nxt is None` and `len(parts)==2` (no `parts[2]` suffix segment). See issues #136, #144. **The tension is Latin-only since #320** — the veto is now scoped to scripts that HAVE initials, and v1's single `is_an_initial` is now two predicates, of which the tension is about `_vocab.is_initial` (the one `is_suffix()`/`is_suffix_strict` vetoes with; `is_initial_shaped` is the bare v1 shape test and vetoes nothing) — `_vocab.is_initial` ANDs that shape test with "not in `_policy._NO_INITIALS`", that constant listing the four scripts which do NOT (Han, Hangul, Hiragana, Katakana) — so the seven single-character CJK members of `suffix_not_acronyms` (씨, 様, 氏, 군, 양, 님, 殿) no longer collide with it when written with a period: `"김민준 씨."` gives suffix `씨.` where the veto had made it the family name, and `"김민준, 씨."` where it had made it the given name. Latin is untouched — `V.`/`I.` still lose to the veto and still need `is_suffix_lenient()`. **Comparing against v1 needs `PYTHONSAFEPATH=1` AND a directory outside the worktree** — `uv run --isolated --no-project --with 'nameparser==1.4.0'` still puts the checkout's `nameparser/` ahead of the pinned wheel on `sys.path`, so the "v1" side silently imports the branch and every comparison reports parity. Run it as `cd && PYTHONSAFEPATH=1 uv run --isolated --no-project --with 'nameparser==1.4.0' python -c "..."`. This produces false confidence rather than an error, so it invalidates results without ever looking wrong. diff --git a/docs/release_log.rst b/docs/release_log.rst index fc211f1..848b599 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -29,7 +29,7 @@ Release Log - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right on the SEGMENTER path, alongside declining the pack or the segmenter. It is no lever where the VOCABULARY divides the name, the two spellings agreeing exactly there — ``김민준 씨`` and ``김민준씨`` both give family 김, given 민준, suffix 씨, as do ``王小明 先生`` and ``王小明先生`` under the Chinese pack — and Korean data has no pack to decline either, hangul segmentation being on by default. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix``, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, which also matches its spaced form. ``田中さん, PhD`` and ``威廉·莎士比亚さん`` peel too; the first of those does *not* otherwise match its spaced form, since ``田中さん PhD`` leaves PhD in ``suffix`` beside さん while after a comma it reads as a ``title`` — where the credential lands is the comma's business, not the peel's. Previously each of these left the honorific inside the name. A comma no longer switches the peel off; what it does now is say which runs of the name to look in, and those are the two around a family comma, an honorific being as often glued to the given name as to the family. Anything past those two runs is out of reach, which is the one limit worth knowing: ``김, 민준 지훈씨`` peels, while ``김, 민준, 지훈씨`` (a third run) and ``김,, 민준씨`` (a doubled comma, which puts the name in a later run) do not. The reach also rests on the second run being name text, and a one-word part before the comma reads as a family comma even when the part after it is entirely suffix-shaped — so ``田中さん, V.`` keeps さん in the family name where ``田中さん, PhD`` gives it up. Same credential, opposite answer, and unchanged from 1.4.0 in that spelling. The 间隔号 does not switch the peel off either. Both marks say where a name divides into surname and given, and an honorific is not part of the name in either reading. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) - - Fix a period after a CJK honorific stopping it being recognized: ``씨.``, ``様.``, ``氏.``, ``님.``, ``군.``, ``양.`` and ``殿.`` now route to ``suffix`` like their periodless spellings, where the trailing period had left them inside the name — the family name in ``"김민준 씨."``, the given name in ``"김민준, 씨."``. The cause was v1's initial regex, ``^(\w\.|[A-Z])?$`` (``REGEXES["initial"]``, still public v1 API), whose ``\w`` is Unicode-aware and so matched a hangul syllable or a Han ideograph as readily as a letter; the strict suffix test applies that as a veto (``V.`` in ``"John V. Smith"`` is a middle initial, not roman five), and a veto written for Latin was being asked of scripts it was never about. The cost ran past the honorific itself: because the vetoed token read as name text, the glued-honorific peel's scan back for its site stopped at it instead of stepping over it, took it as the site, found no honorific at the end of it and gave up — so ``"田中さん, 様."`` kept ``さん`` inside the family name while ``"田中さん, 様"`` peeled it. Measured against 1.4.0, ``"김민준, 씨."`` and ``"田中さん, 様."`` were returning exactly what 1.x returns, so the honorific work earlier in this release had a hole in it wherever the honorific was written with a period. An initial is a single LETTER standing in for a name, and Han ideographs, hangul syllables and kana are morphemes and syllables rather than letters, so the veto now asks its question only of the scripts where it means something. Alphabets keep their initials untouched — ``"А. С. Пушкин"``, ``"م. الفارسي"`` and ``"Ա. Խաչատրյան"`` are unaffected, and so is the Ukrainian conjunction entry below, where a punctuated ``Й.`` still outranks the conjunction ``й``. The public ``initial`` tag follows the same line: ``씨.`` no longer carries it. **Default-on**, and it reaches ``HumanName`` too (#320) + - Fix an ASCII period after a CJK honorific stopping it being recognized: ``씨.``, ``様.``, ``氏.``, ``님.``, ``군.``, ``양.`` and ``殿.`` now route to ``suffix`` like their periodless spellings, where the trailing period had left them inside the name — the family name in ``"김민준 씨."``, the given name in ``"김민준, 씨."``. The cause was v1's initial regex, ``^(\w\.|[A-Z])?$`` (``REGEXES["initial"]``, still public v1 API), whose ``\w`` is Unicode-aware and so matched a hangul syllable or a Han ideograph as readily as a letter; the strict suffix test applies that as a veto (``V.`` in ``"John V. Smith"`` is a middle initial, not roman five), and a veto written for Latin was being asked of scripts it was never about. The cost ran past the honorific itself: because the vetoed token read as name text, the glued-honorific peel's scan back for its site stopped at it instead of stepping over it, took it as the site, found no honorific at the end of it and gave up — so ``"田中さん, 様."`` kept ``さん`` inside the family name while ``"田中さん, 様"`` peeled it. Measured against 1.4.0, ``"김민준, 씨."`` and ``"田中さん, 様."`` were returning exactly what 1.x returns, so the honorific work earlier in this release had a hole in it wherever the honorific was written with a period. An initial is a single LETTER standing in for a name, and Han ideographs, hangul syllables and kana are morphemes and syllables rather than letters, so the veto now asks its question only of the scripts where it means something. Alphabets keep their initials untouched — ``"А. С. Пушкин"``, ``"م. الفارسي"`` and ``"Ա. Խաչատրյան"`` are unaffected, and so is the Ukrainian conjunction entry below, where a punctuated ``Й.`` still outranks the conjunction ``й``. The public ``initial`` tag follows the same line: ``씨.`` no longer carries it. Read ``period`` strictly here: the fix is scoped to the ASCII full stop U+002E, because that is the only period ``_normalize`` strips. The fullwidth U+FF0E and the ideographic U+3002 (with its halfwidth twin U+FF61) — the stops a CJK writer is likelier to type — leave the honorific unmatchable by the vocabulary lookup, which runs before the veto is ever consulted, so ``"김민준 씨．"`` still reads the honorific as the family name. That is a separate, still-open gap in ``_normalize`` rather than in the veto: those spellings parse identically before and after this change, and widening the strip is follow-up work. **Default-on**, and it reaches ``HumanName`` too (#320) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index eb75481..3ef2c6d 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -89,10 +89,12 @@ def is_initial_shaped(text: str) -> bool: def is_initial(text: str) -> bool: """'A.' / 'j.' / bare capital -- v1's is_an_initial, narrowed to scripts that HAVE initials (#320). v1's \\w is Unicode-aware and - matched CJK too, which vetoed period-written CJK honorifics ('씨.') - out of the suffix vocabulary -- and, downstream of that, left the - glued honorific in a name carrying such a token unpeeled - ('田中さん 様.').""" + matched CJK too, which made period-written CJK honorifics ('씨.') + fail is_suffix_strict -- the veto in _is_suffix_strict_n, NOT the + vocabulary: suffix_as_written has no veto, so classify tagged '씨.' + 'vocab:suffix' either way, and is_suffix_lenient took it either way + too. Downstream of that one strict-test No, the glued honorific in + a name carrying such a token went unpeeled ('田中さん 様.').""" return is_initial_shaped(text) and not _in_initialless_script(text) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index b91507a..dbe8b76 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -142,20 +142,27 @@ class Script(StrEnum): #: quantify over this one union (HANGUL simply omitted). _JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) -#: Scripts whose characters cannot BE an initial. A Han ideograph, a -#: hangul syllable and a kana are each a morpheme or a syllable rather -#: than a letter, so a single one never stands in for a name the way -#: "J." stands in for "John". Membership is not "non-Latin": should a -#: Script.CYRILLIC or Script.DEVANAGARI ever be added, it does NOT -#: belong here -- those are alphabets, they have letters, and their -#: initials are real ("А. С. Пушкин", "م. الفارسي"). +#: Scripts whose characters cannot BE an initial. The criterion is +#: orthographic CONVENTION, not what a character is: does the writing +#: tradition abbreviate a given name to ONE character plus a period, +#: the way "J." stands in for "John"? Han, hangul and kana have no +#: such convention, so a lone punctuated 씨/様/김 is not a shortened +#: name and the veto has nothing to veto there. Do not restate that +#: phonologically ("letters, not syllables") -- Devanagari is an +#: abugida and Arabic an abjad, neither has letters in that sense, and +#: both abbreviate, so should Script.CYRILLIC or Script.DEVANAGARI +#: ever be added neither belongs here; their initials are real and +#: pinned as such ("А. С. Пушкин", "م. الفارسي"). #: -#: Enumerated rather than spelled *_SCRIPT_RANGES: the table admits a -#: script that DETERMINES A CONVENTION (see Script), which is a -#: different question from whether that script has initials. The four -#: members coinciding today is what has been implemented, not a -#: property of the enum -- a Thai entry (#317) must not inherit this -#: answer without someone deciding it. +#: Enumerated rather than derived from _SCRIPT_RANGES' keys: the +#: Script enum admits a member so that SOME behavior may key on it +#: (see Script), on assorted grounds -- KATAKANA is in it so the +#: classifier can name what it deliberately declines, and neither +#: DEFAULT_SCRIPT_ORDERS nor segment_scripts' default mentions it. +#: Membership therefore settles nothing about abbreviation: the four +#: coinciding today is what has been implemented, not a property of +#: the enum -- and Thai (#317) is an abugida too, so it must not +#: inherit this answer without someone deciding it. _NO_INITIALS = (Script.HAN, Script.HANGUL, Script.HIRAGANA, Script.KATAKANA) diff --git a/nameparser/_types.py b/nameparser/_types.py index 5b9a178..a4298a6 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -82,11 +82,15 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: wherever it lands -- including a given-name "Van" -- so combine it #: with Role.FAMILY (as family_particles does) to get actual family #: particles; "conjunction" a joining word ("and", "y"); "initial" an -#: initial-shaped word ("J.", "Q"); +#: initial-shaped word in a script that HAS initials -- "J." or "А.", +#: never "씨." (#320); #: "joined" a continuation of the previous token within one merged #: piece ("Ph." + "D."), which the suffix view joins with a space #: instead of ", ". Every other tag is namespaced ("vocab:...") and is #: unstable debugging provenance -- never match against those. +#: This prose is the hand-maintained twin of docs/modules.rst's +#: STABLE_TAGS block; nothing pins the two against each other (the +#: test only compares the frozenset), so edit both or neither. STABLE_TAGS = frozenset({"particle", "conjunction", "initial", "joined"}) #: The one sanctioned view-reorder marker (namespaced = unstable API). diff --git a/tests/v2/cases.py b/tests/v2/cases.py index e2284c5..e01e859 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -824,8 +824,10 @@ def __post_init__(self) -> None: {"family": "김민준", "suffix": "씨."}, classification="fix(#320)", notes="the period-written form of ko_honorific_after_comma " - "('김민준, 씨'), whose fields it must match and before " - "#320 did not. _normalize strips the trailing period, " + "('김민준, 씨'), whose field ASSIGNMENT it must match " + "and before #320 did not -- same roles, the suffix " + "VALUE differing by the period it was written with. " + "_normalize strips the trailing period, " "so the vocabulary sees 씨 either way -- the initial " "veto was the only thing rejecting the written form, " "and literally the veto: _is_suffix_piece is " diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py index afb9647..69b7391 100644 --- a/tests/v2/pipeline/test_classify.py +++ b/tests/v2/pipeline/test_classify.py @@ -14,9 +14,10 @@ suffix_acronyms_ambiguous=frozenset({"ma"}), particles=frozenset({"de", "la", "van"}), particles_ambiguous=frozenset({"van"}), - # й is a REAL default conjunction (Ukrainian, #267), carried here so - # test_cyrillic_initial_outranks_the_conjunction pins a live - # collision rather than a hypothetical one + # й is COPIED from the shipped conjunctions (Ukrainian, #267) so the + # collision test_cyrillic_initial_outranks_the_conjunction pins is + # one that really ships and the reader can check against the + # defaults. The copy is local: this file never reads the shipped set conjunctions=frozenset({"and", "y", "й"}), bound_given_names=frozenset({"abdul"}), maiden_markers=frozenset({"née"}), @@ -63,10 +64,12 @@ def test_cyrillic_initial_outranks_the_conjunction() -> None: is an initial and must not be read as it. Narrowing is_initial to [A-Za-z] -- the fix #320 originally proposed -- flips this token to 'conjunction' and strips 'initial' off every Cyrillic, Greek, - Arabic and Hebrew initial. Neither moves field output on a short - name, so this asserts the TAG. _LEX carries й for this: the - collision is with SHIPPED vocabulary, so a lexicon built here - would keep passing if й were dropped from the defaults.""" + Arabic, Hebrew, Devanagari and Armenian initial (the six + test_vocab.test_is_initial_script_repertoire asserts). Neither + moves field output on a short name, so this asserts the TAG. _LEX + copies й from the SHIPPED conjunctions so the collision is a real + one -- but the copy is local, so this test would keep passing if й + were dropped from the defaults.""" out = _classified("Й. Сліпий") assert "initial" in _tags(out, "Й.") assert "conjunction" not in _tags(out, "Й.") diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 15a1c7b..b27a21f 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -38,9 +38,10 @@ def test_is_initial_script_repertoire() -> None: assert not is_initial("김.") assert not is_initial("さ.") assert not is_initial("ラ.") - # unchanged: a digit is the visible edge of \w's reach, and the - # shape half still owns it -- only the repertoire narrowed + # unchanged: a digit is ONE edge of \w's reach and '_' is another, + # and the shape half still owns both -- only the repertoire narrowed assert is_initial("2.") + assert is_initial("_.") # unchanged: the SHAPE half still requires a single character assert not is_initial("राम.") From 82f9815ed44524e3d429466413769baf138e954b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 2 Aug 2026 11:52:01 -0700 Subject: [PATCH 9/9] Tell the roman-numeral story once, where someone will be standing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'John 씨. V' incident was written out three times: is_initial_shaped's docstring, assign's call site, and a3c31f6's message. All accurate, and collectively noise. Keep the copy at the call site -- that is where someone debugging the input actually lands -- and leave the commit message as the full record. The docstring's job is helping a caller pick between the two predicates, so it keeps the choosing rule and the discriminating example and points at the fork rather than retelling it. Co-Authored-By: Claude Opus 5 --- nameparser/_pipeline/_vocab.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 3ef2c6d..86afd03 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -78,11 +78,8 @@ def is_initial_shaped(text: str) -> bool: run want this; callers asking whether it can really stand in for a name want is_initial (#320). The two answers differ only inside _NO_INITIALS scripts, where '씨.' is initial-SHAPED but is not an - initial. assign's roman-numeral fork is the shape caller: "the - piece before this trailing single roman letter looks like an - initial, so we are mid-run and the letter is a name part" was - always a question about layout, and narrowing it to real initials - dropped the family name out of 'John 씨. V'.""" + initial -- see assign's roman-numeral fork, the shape caller, for + what picking the wrong one costs.""" return bool(_INITIAL.fullmatch(text))