From bc04c148e9944be294976cec8338cdff4a567a0e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 7 Sep 2026 15:26:45 -0700 Subject: [PATCH 1/4] fix(suffixes): rai and cha leave SUFFIX_ACRONYMS Both words arrived in the 2019-12-11 bulk wikipedia post-nominal import and neither was ever reviewed against a surname collision. "Aishwarya Rai" read suffix 'Rai' with NO family name at all at every 2.x release; it now reads family 'Rai', which is 1.4.0's own answer, and "Lala Lajpat Rai" reads middle 'Lajpat', family 'Rai'. The criterion, not the word. The four ambiguous acronyms (ma, do, ed, jd) are all TWO letters, and that is not a coincidence: a two-letter acronym has too few letters to be anything but common, so both readings are live and the ambiguous marking is the honest answer. A longer credential that is tenuous or specialized -- RAI is "RETA Authorized Instructor", CHA is Certified Hotel Administrator -- and collides with a common surname has no second reading worth carrying, so it is removed. A caller who needs one back adds it: Lexicon.default().add(suffix_acronyms={"cha"}). The cost is accepted and pinned. "John Smith RAI" reads family 'RAI' with 'Smith' a middle name, and "Ahmad Jayadi, CHA" reads given 'CHA', family 'Ahmad Jayadi' -- with no credential after the comma, C1 reads it as an ordinary family comma. Both have case rows so the reversal is visible if anyone wants it back. One of the two identical 'ch', lines goes with them as a same-commit cleanup: a frozenset literal hid the duplicate, so membership drops by exactly 2 (612 -> 610) and 'ch' itself stays. Five corpus names move, all radar-tier (corpus_issues.jsonl), in three diff shapes with one cause, which is why the ledger rule declares {family, given, middle, suffix} where the deleted one declared two. The fix(#342) NOT WANTED rule leaves expected_since_1.4.0.toml as its own comment instructed: 'Aishwarya Rai' now agrees with 1.4.0 and stops diffing there, so that ledger's fix(#342) heading lists four names and the three 2.x ledgers list five. The _WATCHED_DIFFS row for the name goes with it -- a case row watches it now. Gate: 367 / 263 / 175 / 37 intentional at 1.4.0 / 2.0.0 / 2.1.0 / 2.2.0, 0 unexplained, 0 radar unclassified, corpus sizes unchanged. Closes #342 Co-Authored-By: Claude Fable 5.1 --- nameparser/config/suffixes.py | 27 ++++++- tests/v2/cases.py | 39 ++++++++++ tests/v2/test_ledger_guards.py | 69 +++++++++++++++-- tools/differential/compare.py | 1 - tools/differential/expected_since_1.4.0.toml | 79 +++++++++++--------- tools/differential/expected_since_2.0.0.toml | 44 +++++++++++ tools/differential/expected_since_2.1.0.toml | 44 +++++++++++ tools/differential/expected_since_2.2.0.toml | 44 +++++++++++ 8 files changed, 299 insertions(+), 48 deletions(-) diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index a27b9fd7..dfd07a21 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -213,6 +213,30 @@ # suffix only when written with periods ('M.A.' yes, 'Ma' no), so # 'Jack Ma' keeps its family name. # + # The other half of the criterion, added 2026-09-07 with #342. + # Marking is for a SHORT credential in wide use that is also a + # name -- the four entries here are all two letters, which is not + # a coincidence: a two-letter acronym has too few letters to be + # anything but common, so both readings are live and a flag is + # the honest answer. A LONGER credential that is tenuous or + # specialized and collides with a common name is REMOVED from + # SUFFIX_ACRONYMS instead of marked, because its BARE spelling + # has no second reading worth carrying: 'rai' and 'cha' left the + # set that day (RAI is "RETA Authorized Instructor", CHA is + # Certified Hotel Administrator or Certified Healthcare Auditor; + # Rai is a common surname across Hindi- and Bengali-speaking + # regions and Cha the Korean 차). Removal need not take the + # DOTTED spelling with it: "John Smith R.A.I." still reads suffix + # 'R.A.I.', because rules.md#S3 splits an interior-period token + # on its periods and the chunk 'i' is a Roman numeral in + # SUFFIX_WORDS -- "John Smith C.H.A." has no such chunk and reads + # family 'C.H.A.'. A caller who needs one back adds it -- + # Lexicon.default().add(suffix_acronyms={"cha"}) -- which is the + # answer this library gives for every locale-specific + # vocabulary. The cost is stated and accepted: with the entry + # gone, "John Smith RAI" reads family 'RAI'. See + # decisions.md#suffix-acronym-collisions. + # # NOT 'ms' or 'sa', though #296's audit table put them here for the # leading-title collision (bare "Ms" the honorific, "M.S." the # degree): the gate is position-blind and the collision is not. @@ -387,8 +411,6 @@ 'cgr', 'cgsp', 'ch', - 'ch', - 'cha', 'chba', 'chdm', 'che', @@ -807,7 +829,6 @@ 'qsd', 'qsp', 'ra', - 'rai', 'rba', 'rci', 'rcp', diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 650e59b5..59f9ef52 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -402,6 +402,45 @@ def _check_cjk_shape_purity(self) -> None: notes="the other half of the same rule: three pieces means " "peeling 'MA' still leaves given+family, so the " "credential reading wins (v1 parity)"), + # -- #342: rai and cha left SUFFIX_ACRONYMS. The criterion is in + # decisions.md#suffix-acronym-collisions: a TWO-LETTER credential + # in wide use that is also a name earns the ambiguous marking + # (ma, do, ed, jd), while a three-letter credential that is + # tenuous or specialized and collides with a common surname is + # REMOVED, a caller adding it back with + # Lexicon.default().add(suffix_acronyms={"cha"}). These rows pin + # the FORK, not the entries (mechanisms.md#VOCABULARY-EXERCISES-FORKS). + Case("bare_surname_is_not_a_credential", "Aishwarya Rai", + {"given": "Aishwarya", "family": "Rai"}, + classification="fix(#342)", + notes="#342's own subject: 'rai' arrived in the 2019 bulk " + "wikipedia post-nominal import and was never reviewed " + "against the surname, so 2.0.0 through 2.2.0 read " + "suffix 'Rai' with NO family name at all. 1.4.0 read " + "family 'Rai' and this restores that reading, which is " + "why the 1.4.0 ledger loses a rule here rather than " + "gaining one. Named in the release note"), + Case("removed_credential_loses_its_suffix_reading", "John Smith RAI", + {"given": "John", "middle": "Smith", "family": "RAI"}, + classification="fix(#342)", + notes="the accepted cost, pinned so the reversal is visible. " + "RAI is a real if specialized credential and a full " + "name in front of it used to be enough to read it as " + "one; with the entry gone the all-caps shape carries " + "no signal the parser reads, so the word families and " + "'Smith' becomes a middle name. The shape-plus-position " + "heuristic that would recover it is a parking-lot " + "bullet of decisions.md#suffix-acronym-collisions"), + Case("removed_credential_after_a_comma_reads_as_the_given_name", + "Ahmad Jayadi, CHA", + {"given": "CHA", "family": "Ahmad Jayadi"}, + classification="fix(#342)", + notes="the comma form moves the OTHER way and is why the " + "ledger rule declares four fields rather than two. " + "With 'cha' gone the comma is an ordinary family " + "comma (C1): the pre-comma run is the family and the " + "post-comma word is the given name. 'John Smith, RAI' " + "is the same shape and moves with it"), Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", {"given": "John", "middle": "Q", "family": "Smith", "suffix": "MA"}, diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 8f945351..e000fa3a 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -808,14 +808,36 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # has no third token for the regex to require. "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": ("'Smitty' Jones", "Jones Jr.", "'Smitty' Dr. Jones"), - # #451's two NOT-WANTED rules, literal-anchored to one corpus name - # apiece: a three-token name with a rootname before or after is a + # #451's remaining NOT-WANTED rule, literal-anchored to one corpus + # name: a three-token name with a rootname before or after is a # different diff shape (or, for 'Carod i Rovira' and 'Lluis Carod # i', no diff at all -- #397's still-open enhancement, not a - # 1.4-to-2.x regression), and 'Rai'/'Jane Rai Smith' have no - # 'aishwarya' to anchor on. - "fix(#342)": ("Aishwarya Rai Bachchan", "Rai", "Jane Rai Smith"), + # 1.4-to-2.x regression). "fix(#397)": ("Carod i Rovira", "Josep Carod i Rovira", "Lluis Carod i"), + # #342's rule is a literal alternation of five names, so a + # widening reaching only names the corpora lack would leave + # _CORPUS_CLAIMS unmoved. These probes are the wall, and every one + # of them is a name MEASURED not to move under the removal (which + # is the roster's actual condition -- a name the rule has no + # business claiming, not merely one outside the corpus). Two test + # the right anchor, where a third word or a re-ordered comma + # leaves 'Rai' with a family name beside it and nothing to take; + # 'Ahmad Jayadi' and 'John Smith' test the ALTERNATIVES' trailing + # words, each being a mover with its last word dropped, so an + # alternative TRUNCATED to its leading tokens is caught. No probe + # tests the left anchor and none can: dropping the '^' widens the + # rule to names ENDING in an alternative, and every such name is + # itself a mover -- 'Mr Aishwarya Rai' moves -- so no name that + # widening reaches can satisfy the roster's condition. + # 'Sejal Chaturvedi, CSM' is the wall against a SHAPE widening + # instead: the trailing-acronym shape "^.+\s[A-Z]{2,}$" reaches + # three of the five above and would look like a faithful + # generalization, but it also reaches this corpus name + # (corpus_issues.jsonl), which parses identically with and without + # 'rai'/'cha' in SUFFIX_ACRONYMS -- so the widening is caught. + "fix(#342)": ("Aishwarya Rai Bachchan", "Rai, Aishwarya", + "Jane Rai Smith", "Ahmad Jayadi", "John Smith", + "Sejal Chaturvedi, CSM"), # #451's four replacements for the fields-only catch-all. Each is # anchored to a two-token name, so the probes are a third token and # each other's vocabulary: the four exist BECAUSE one rule could not @@ -1528,6 +1550,17 @@ class _LatinCopy(NamedTuple): # move. One set, identical in all four ledgers. frozenset({"Baba Ramdev", "Guru Nanak", "Lama Zopa", "Swami Vivekananda"}), + # #342's movers, one corpus name per alternative -- a list of + # names, not a copy of any wordlist, so there is no vocabulary + # for it to drift from. The rule's subject is what two REMOVED + # entries stop doing, and the vocabulary it would be suspected of + # copying (SUFFIX_ACRONYMS) no longer holds either word, so a + # member drawn from it could not name these names at all. A + # member spelled as the shape -- a trailing all-caps token -- + # would reach 'John Smith XYZ' and every future acronym + # regression. One set, identical in all four ledgers. + frozenset({"Ahmad Jayadi, CHA", "Aishwarya Rai", "John Smith RAI", + "John Smith, RAI", "Lala Lajpat Rai"}), # fix(#445)'s movers, one corpus name per alternative -- a list of # names, not a copy of any wordlist, so there is no vocabulary for # it to drift from. Two sets because the ledgers group the nine @@ -2020,6 +2053,12 @@ def _claim(rule: dict) -> _Claim: # before it reached the gate. "fix(#346) a renunciate title and one name word leave the name a given name": _Claim(4, ('family', 'given'), "a3399ee7b21e", None), + # #342's alternation. Five corpus names and four roles: the + # comma forms move `given` where the bare forms move `middle`, + # so a widening taking one shape alone would change the roles + # here before it reached the gate. + "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word": + _Claim(5, ('family', 'given', 'middle', 'suffix'), "c3d76812da97", None), "fix(A2) content-free input names nobody, so every role empties": _Claim(5, ('given',), "1af8d718688b", None), "fix(#335) a marker-led clause leaves the one name word its bare reading": @@ -2087,8 +2126,6 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('family', 'given'), "d8ee9cd5da5f", None), "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(288, ('family', 'given'), "10c78dd0f2d2", None), - "fix(#342) NOT WANTED: a bare trailing 'Rai' is read as a post-nominal suffix and the family is lost": - _Claim(1, ('family', 'suffix'), "694fd06a2e9a", None), "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost": _Claim(1, ('family', 'suffix'), "498602f3cfd0", None), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": @@ -2234,6 +2271,12 @@ def _claim(rule: dict) -> _Claim: # before it reached the gate. "fix(#346) a renunciate title and one name word leave the name a given name": _Claim(4, ('family', 'given'), "a3399ee7b21e", None), + # #342's alternation. Five corpus names and four roles: the + # comma forms move `given` where the bare forms move `middle`, + # so a widening taking one shape alone would change the roles + # here before it reached the gate. + "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word": + _Claim(5, ('family', 'given', 'middle', 'suffix'), "c3d76812da97", None), # The compound rule, at the two baselines where 'abdul Smith # Jr V' already diffs {family, given} under fix(#401) and the # widened diff leaves that rule's `fields`. Three roles here @@ -2409,6 +2452,12 @@ def _claim(rule: dict) -> _Claim: # before it reached the gate. "fix(#346) a renunciate title and one name word leave the name a given name": _Claim(4, ('family', 'given'), "a3399ee7b21e", None), + # #342's alternation. Five corpus names and four roles: the + # comma forms move `given` where the bare forms move `middle`, + # so a widening taking one shape alone would change the roles + # here before it reached the gate. + "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word": + _Claim(5, ('family', 'given', 'middle', 'suffix'), "c3d76812da97", None), # The four one-name CJK rules, literal-anchored, at the two # baselines where the render is the whole of what moved. A # reach of 1 is one _CORPUS_CLAIMS cannot police on its own -- @@ -2438,6 +2487,12 @@ def _claim(rule: dict) -> _Claim: # before it reached the gate. "fix(#346) a renunciate title and one name word leave the name a given name": _Claim(4, ('family', 'given'), "a3399ee7b21e", None), + # #342's alternation. Five corpus names and four roles: the + # comma forms move `given` where the bare forms move `middle`, + # so a widening taking one shape alone would change the roles + # here before it reached the gate. + "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word": + _Claim(5, ('family', 'given', 'middle', 'suffix'), "c3d76812da97", None), # The four one-name CJK rules, literal-anchored, at the two # baselines where the render is the whole of what moved. A # reach of 1 is one _CORPUS_CLAIMS cannot police on its own -- diff --git a/tools/differential/compare.py b/tools/differential/compare.py index 1d771028..d68618a0 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -1986,7 +1986,6 @@ class _ShapeMismatch(NamedTuple): _WATCHED_DIFFS: dict[str, dict[str, tuple[str, ...]]] = { "expected_since_1.4.0.toml": { "1 & 2, 3 4 5, Mr.": ("_initials",), - "Aishwarya Rai": ("family", "suffix"), "Anh do": ("_initials",), "Anna Müller (geb. Schmidt)": ("maiden", "nickname"), "Anna Müller geb. Schmidt": ("family", "maiden", "middle"), diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 9ac500d6..728194d7 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -1035,43 +1035,48 @@ name_regex = "(?i)^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*\\svd\\s*$" fields = ["family", "suffix"] [[change]] -issue = "fix(#342) NOT WANTED: a bare trailing 'Rai' is read as a post-nominal suffix and the family is lost" -# 'Aishwarya Rai': the tree reads given 'Aishwarya', family '', suffix -# 'Rai'. 'rai' is a bare entry in SUFFIX_ACRONYMS -# (nameparser/config/suffixes.py -- not SUFFIX_ACRONYMS_AMBIGUOUS, -# not SUFFIX_WORDS), so the credential reading outranks the surname -# reading with nothing to arbitrate them. -# #342 (OPEN, milestone v2.2, bug) reports this: Rai, in the issue's -# own words, is "a common surname across Hindi- and Bengali-speaking -# regions," and the bulk 2019-12-11 wikipedia post-nominal import -# (af5bdab) that added it was never reviewed against that collision. -# -# Whether 1.4 itself read the trailing word as `family` is outside -# what this ledger's worker can check directly -# (tools/differential/README.md's warning against reading a v1 release -# from a cached environment); the diff this rule explains at this -# baseline moves exactly {family, suffix}, which is consistent with -# that reading and is what the fields below claim. -# -# Classified because the cause is known and recorded, NOT because the -# reading is wanted -- decisions.md#P5's 2026-08-22 #369 entry -# attributes that exact wording, quoted verbatim, to the previous -# fix(#369) ledger rules (since rewritten, because #369 itself was -# fixed). A [[never]] exclusion was considered and rejected: an -# excluded name reports UNEXPLAINED and exits non-zero, which would -# redden the 1.4.0 gate and block every release until #342 lands -- -# and #342 lists two candidate fixes that trade different corpus -# names against each other, so which one lands is not yet decided. -# -# DELETE THIS RULE when #342 lands. The gate then reports it as -# 'EXPLAINED NOTHING', which is the reminder. -# -# Literal-anchored to the one corpus name that reaches this shape. A -# rule keyed on the SUFFIX_ACRONYMS collision itself, rather than the -# literal name, would claim every trailing acronym-shaped surname and -# pre-excuse regressions on names nobody has looked at. -name_regex = "(?i)^aishwarya\\s+rai$" -fields = ["family", "suffix"] +issue = "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word" +# 'rai' and 'cha' left SUFFIX_ACRONYMS on 2026-09-07 +# (nameparser/config/suffixes.py). Both arrived in the 2019-12-11 +# bulk wikipedia post-nominal import (af5bdab), neither was reviewed +# against a surname collision, and decisions.md#suffix-acronym-collisions +# states the criterion that removed them rather than marking them +# ambiguous: marking is for a short credential in wide use that is +# also a name, removal for a longer one that is tenuous and collides. +# +# Five corpus names, all from the radar-tier corpus_issues.jsonl, in +# THREE diff shapes with one cause -- the word stopped being +# credential vocabulary, so the positional read takes it: +# 'Aishwarya Rai' {family, suffix} suffix -> family +# (2.x only -- parity at 1.4.0) +# 'Lala Lajpat Rai' {middle, family, suffix} the family shifts right +# 'John Smith RAI' {middle, family, suffix} same +# 'John Smith, RAI' {given, family, suffix} an ordinary family comma (C1) +# 'Ahmad Jayadi, CHA' {given, family, suffix} same +# The comma pair is why `fields` declares four roles rather than the +# two the pre-2.3 fix(#342) rule declared: with no credential after +# the comma, C1 reads the pre-comma run as the family and the +# post-comma word as the GIVEN name, so `given` moves where it did +# not before. The declaration is the exact union of the five, which +# is what the OVER-DECLARED check recomputes at the end of every run. +# +# 'Aishwarya Rai' is in this rule at 2.0.0/2.1.0/2.2.0 and NOT +# explained by it at 1.4.0, where the name now reaches PARITY: 1.4.0 +# read family 'Rai' too, which is what the deleted +# `fix(#342) NOT WANTED` rule existed to classify. The regex is the +# same in all four files anyway -- reach is not explanation, and a +# per-ledger regex would say a name left the corpus. +# +# An anchored alternation of the five NAMES rather than a shape. The +# shape is "a trailing word that used to be an acronym entry", which +# would be a copy of SUFFIX_ACRONYMS and would stand ready to explain +# every future trailing-acronym regression. The members are a list of +# names and copy no wordlist, which is what _NOT_A_VOCABULARY_COPY in +# tests/v2/test_ledger_guards.py records; _CORPUS_CLAIMS pins the +# reach at 5 with its digest, and _MUST_NOT_MATCH carries the +# boundaries the paragraph above argues. +name_regex = "^(?:Ahmad Jayadi, CHA|Aishwarya Rai|John Smith RAI|John Smith, RAI|Lala Lajpat Rai)$" +fields = ["family", "given", "middle", "suffix"] [[change]] issue = "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost" diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index d8104140..9cba00dc 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -86,6 +86,50 @@ issue = "fix(#346) a renunciate title and one name word leave the name a given n name_regex = "^(?:Baba Ramdev|Guru Nanak|Lama Zopa|Swami Vivekananda)$" fields = ["family", "given"] +[[change]] +issue = "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word" +# 'rai' and 'cha' left SUFFIX_ACRONYMS on 2026-09-07 +# (nameparser/config/suffixes.py). Both arrived in the 2019-12-11 +# bulk wikipedia post-nominal import (af5bdab), neither was reviewed +# against a surname collision, and decisions.md#suffix-acronym-collisions +# states the criterion that removed them rather than marking them +# ambiguous: marking is for a short credential in wide use that is +# also a name, removal for a longer one that is tenuous and collides. +# +# Five corpus names, all from the radar-tier corpus_issues.jsonl, in +# THREE diff shapes with one cause -- the word stopped being +# credential vocabulary, so the positional read takes it: +# 'Aishwarya Rai' {family, suffix} suffix -> family +# (2.x only -- parity at 1.4.0) +# 'Lala Lajpat Rai' {middle, family, suffix} the family shifts right +# 'John Smith RAI' {middle, family, suffix} same +# 'John Smith, RAI' {given, family, suffix} an ordinary family comma (C1) +# 'Ahmad Jayadi, CHA' {given, family, suffix} same +# The comma pair is why `fields` declares four roles rather than the +# two the pre-2.3 fix(#342) rule declared: with no credential after +# the comma, C1 reads the pre-comma run as the family and the +# post-comma word as the GIVEN name, so `given` moves where it did +# not before. The declaration is the exact union of the five, which +# is what the OVER-DECLARED check recomputes at the end of every run. +# +# 'Aishwarya Rai' is in this rule at 2.0.0/2.1.0/2.2.0 and NOT +# explained by it at 1.4.0, where the name now reaches PARITY: 1.4.0 +# read family 'Rai' too, which is what the deleted +# `fix(#342) NOT WANTED` rule existed to classify. The regex is the +# same in all four files anyway -- reach is not explanation, and a +# per-ledger regex would say a name left the corpus. +# +# An anchored alternation of the five NAMES rather than a shape. The +# shape is "a trailing word that used to be an acronym entry", which +# would be a copy of SUFFIX_ACRONYMS and would stand ready to explain +# every future trailing-acronym regression. The members are a list of +# names and copy no wordlist, which is what _NOT_A_VOCABULARY_COPY in +# tests/v2/test_ledger_guards.py records; _CORPUS_CLAIMS pins the +# reach at 5 with its digest, and _MUST_NOT_MATCH carries the +# boundaries the paragraph above argues. +name_regex = "^(?:Ahmad Jayadi, CHA|Aishwarya Rai|John Smith RAI|John Smith, RAI|Lala Lajpat Rai)$" +fields = ["family", "given", "middle", "suffix"] + [[change]] issue = "fix(#271/#272/#298) native-script CJK: family-first order, hangul segmentation, the kana license and the dots" # The 2.1 East Asian defaults, in the fields they move name pieces diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 63ab7ae3..68e20774 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -144,6 +144,50 @@ issue = "fix(#346) a renunciate title and one name word leave the name a given n name_regex = "^(?:Baba Ramdev|Guru Nanak|Lama Zopa|Swami Vivekananda)$" fields = ["family", "given"] +[[change]] +issue = "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word" +# 'rai' and 'cha' left SUFFIX_ACRONYMS on 2026-09-07 +# (nameparser/config/suffixes.py). Both arrived in the 2019-12-11 +# bulk wikipedia post-nominal import (af5bdab), neither was reviewed +# against a surname collision, and decisions.md#suffix-acronym-collisions +# states the criterion that removed them rather than marking them +# ambiguous: marking is for a short credential in wide use that is +# also a name, removal for a longer one that is tenuous and collides. +# +# Five corpus names, all from the radar-tier corpus_issues.jsonl, in +# THREE diff shapes with one cause -- the word stopped being +# credential vocabulary, so the positional read takes it: +# 'Aishwarya Rai' {family, suffix} suffix -> family +# (2.x only -- parity at 1.4.0) +# 'Lala Lajpat Rai' {middle, family, suffix} the family shifts right +# 'John Smith RAI' {middle, family, suffix} same +# 'John Smith, RAI' {given, family, suffix} an ordinary family comma (C1) +# 'Ahmad Jayadi, CHA' {given, family, suffix} same +# The comma pair is why `fields` declares four roles rather than the +# two the pre-2.3 fix(#342) rule declared: with no credential after +# the comma, C1 reads the pre-comma run as the family and the +# post-comma word as the GIVEN name, so `given` moves where it did +# not before. The declaration is the exact union of the five, which +# is what the OVER-DECLARED check recomputes at the end of every run. +# +# 'Aishwarya Rai' is in this rule at 2.0.0/2.1.0/2.2.0 and NOT +# explained by it at 1.4.0, where the name now reaches PARITY: 1.4.0 +# read family 'Rai' too, which is what the deleted +# `fix(#342) NOT WANTED` rule existed to classify. The regex is the +# same in all four files anyway -- reach is not explanation, and a +# per-ledger regex would say a name left the corpus. +# +# An anchored alternation of the five NAMES rather than a shape. The +# shape is "a trailing word that used to be an acronym entry", which +# would be a copy of SUFFIX_ACRONYMS and would stand ready to explain +# every future trailing-acronym regression. The members are a list of +# names and copy no wordlist, which is what _NOT_A_VOCABULARY_COPY in +# tests/v2/test_ledger_guards.py records; _CORPUS_CLAIMS pins the +# reach at 5 with its digest, and _MUST_NOT_MATCH carries the +# boundaries the paragraph above argues. +name_regex = "^(?:Ahmad Jayadi, CHA|Aishwarya Rai|John Smith RAI|John Smith, RAI|Lala Lajpat Rai)$" +fields = ["family", "given", "middle", "suffix"] + [[change]] issue = "fix(#369) a given-name title licenses the bound given-name join with one word to spare" # 'Sheik abdul salam': rules.md#P5 -- "or after a given-name title, diff --git a/tools/differential/expected_since_2.2.0.toml b/tools/differential/expected_since_2.2.0.toml index 72bf3bf2..fe0f9a87 100644 --- a/tools/differential/expected_since_2.2.0.toml +++ b/tools/differential/expected_since_2.2.0.toml @@ -133,6 +133,50 @@ issue = "fix(#346) a renunciate title and one name word leave the name a given n name_regex = "^(?:Baba Ramdev|Guru Nanak|Lama Zopa|Swami Vivekananda)$" fields = ["family", "given"] +[[change]] +issue = "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word" +# 'rai' and 'cha' left SUFFIX_ACRONYMS on 2026-09-07 +# (nameparser/config/suffixes.py). Both arrived in the 2019-12-11 +# bulk wikipedia post-nominal import (af5bdab), neither was reviewed +# against a surname collision, and decisions.md#suffix-acronym-collisions +# states the criterion that removed them rather than marking them +# ambiguous: marking is for a short credential in wide use that is +# also a name, removal for a longer one that is tenuous and collides. +# +# Five corpus names, all from the radar-tier corpus_issues.jsonl, in +# THREE diff shapes with one cause -- the word stopped being +# credential vocabulary, so the positional read takes it: +# 'Aishwarya Rai' {family, suffix} suffix -> family +# (2.x only -- parity at 1.4.0) +# 'Lala Lajpat Rai' {middle, family, suffix} the family shifts right +# 'John Smith RAI' {middle, family, suffix} same +# 'John Smith, RAI' {given, family, suffix} an ordinary family comma (C1) +# 'Ahmad Jayadi, CHA' {given, family, suffix} same +# The comma pair is why `fields` declares four roles rather than the +# two the pre-2.3 fix(#342) rule declared: with no credential after +# the comma, C1 reads the pre-comma run as the family and the +# post-comma word as the GIVEN name, so `given` moves where it did +# not before. The declaration is the exact union of the five, which +# is what the OVER-DECLARED check recomputes at the end of every run. +# +# 'Aishwarya Rai' is in this rule at 2.0.0/2.1.0/2.2.0 and NOT +# explained by it at 1.4.0, where the name now reaches PARITY: 1.4.0 +# read family 'Rai' too, which is what the deleted +# `fix(#342) NOT WANTED` rule existed to classify. The regex is the +# same in all four files anyway -- reach is not explanation, and a +# per-ledger regex would say a name left the corpus. +# +# An anchored alternation of the five NAMES rather than a shape. The +# shape is "a trailing word that used to be an acronym entry", which +# would be a copy of SUFFIX_ACRONYMS and would stand ready to explain +# every future trailing-acronym regression. The members are a list of +# names and copy no wordlist, which is what _NOT_A_VOCABULARY_COPY in +# tests/v2/test_ledger_guards.py records; _CORPUS_CLAIMS pins the +# reach at 5 with its digest, and _MUST_NOT_MATCH carries the +# boundaries the paragraph above argues. +name_regex = "^(?:Ahmad Jayadi, CHA|Aishwarya Rai|John Smith RAI|John Smith, RAI|Lala Lajpat Rai)$" +fields = ["family", "given", "middle", "suffix"] + [[change]] issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # 'Scott E. Werner', 'Amy E Maid', 'Jose E. Maria Santos': From 880e58bc0cef97843b1ab48e7119590f6c31b7f9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 7 Sep 2026 15:41:22 -0700 Subject: [PATCH 2/4] feat(suffixes): ba joins the ambiguous acronyms BA is a common credential and Ba a real surname -- Vietnamese, and Senegalese Fula -- which is the ma/Ma shape exactly, so 'ba' takes the ambiguous marking rather than the removal the commit before this one gave rai and cha. One line of data; the four sites that read the marking are what the rows pin. "Anna Ba" reads family 'Ba' with a suffix-or-name flag, where every release since 2.0.0 read suffix 'Ba' and no family name at all (1.4.0 read family 'Ba' unflagged, so this is 1.4.0's answer plus the flag). "John Smith BA" is unchanged in its ROLES and newly flagged: a full name in front leaves the credential reading available, which is S2's words-to-spare guard doing the work the marking exists to reach. "John Smith B.A." stays an unflagged suffix -- the period gate counts a per-letter dotted spelling unambiguously -- which rules.md#S2's "John Smith M.A." example already pins; no second row for it. The comma cost is the one already accepted for 'ed': "Smith, BA" reads given 'BA', family 'Smith', with no flag, because the comma decided it. The comma_ambiguous_acronym row ("Smith, Ed") pins that shape; a second member of the set earns no row of its own. No corpus name carries 'ba' as a trailing word, so nothing moves: gate unchanged at 367 / 263 / 175 / 37 intentional, 0 unexplained, 0 radar unclassified. Refs #342 Co-Authored-By: Claude Fable 5.1 --- nameparser/config/suffixes.py | 3 ++- tests/v2/cases.py | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index dfd07a21..5ccc886e 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -215,7 +215,7 @@ # # The other half of the criterion, added 2026-09-07 with #342. # Marking is for a SHORT credential in wide use that is also a - # name -- the four entries here are all two letters, which is not + # name -- the five entries here are all two letters, which is not # a coincidence: a two-letter acronym has too few letters to be # anything but common, so both readings are live and a flag is # the honest answer. A LONGER credential that is tenuous or @@ -245,6 +245,7 @@ # and read as a credential anyway. Both words are genuine duals -- # title and unambiguous suffix -- and position decides, as for # 'sr' and 'lt' (decisions.md#C1). + 'ba', 'do', 'ed', 'jd', diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 59f9ef52..878307fb 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -405,7 +405,7 @@ def _check_cjk_shape_purity(self) -> None: # -- #342: rai and cha left SUFFIX_ACRONYMS. The criterion is in # decisions.md#suffix-acronym-collisions: a TWO-LETTER credential # in wide use that is also a name earns the ambiguous marking - # (ma, do, ed, jd), while a three-letter credential that is + # (ba, do, ed, jd, ma), while a three-letter credential that is # tenuous or specialized and collides with a common surname is # REMOVED, a caller adding it back with # Lexicon.default().add(suffix_acronyms={"cha"}). These rows pin @@ -441,6 +441,30 @@ def _check_cjk_shape_purity(self) -> None: "comma (C1): the pre-comma run is the family and the " "post-comma word is the given name. 'John Smith, RAI' " "is the same shape and moves with it"), + # -- #342: 'ba' is the other half of the same decision. BA is a + # common credential and Ba a real surname (Vietnamese; Senegalese + # Fula), which is the ma/Ma shape exactly, so it takes the + # marking rather than the removal. Two rows, the two halves of + # S2's words-to-spare guard; the release note names both texts. + Case("bare_ba_is_a_surname", "Anna Ba", + {"given": "Anna", "family": "Ba"}, + ambiguities=("suffix-or-name",), + classification="parity", + notes="the marking's point: with only two pieces, 'one of " + "them is a credential' is the less likely reading, so " + "S2's words-to-spare guard keeps the family name and " + "the parse reports which reading it took. 2.0.0 " + "through 2.2.0 read suffix 'Ba' with no family name; " + "1.4.0 read family 'Ba' unflagged, so this row " + "restores 1.4.0's roles and adds the flag"), + Case("ba_is_a_suffix_when_a_family_name_remains", "John Smith BA", + {"given": "John", "family": "Smith", "suffix": "BA"}, + ambiguities=("suffix-or-name",), + notes="the words-to-spare half: a full name in front leaves " + "the credential reading available, so the role does " + "NOT move -- only the flag is new. The twin of " + "ambiguous_acronym_is_a_suffix_when_a_family_name_remains " + "above"), Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", {"given": "John", "middle": "Q", "family": "Smith", "suffix": "MA"}, From 31ce5293687c501bf5cc25e9abdede81376aa04b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 7 Sep 2026 15:56:02 -0700 Subject: [PATCH 3/4] docs(design+release): suffix-acronym collisions, recorded The #suffix-acronym-collisions entry carries the bundle's decisions in the doc's own shapes. The criterion is the part worth keeping: C-i decides ship-or-mark, and this adds the third answer and says when to reach for it -- the four acronyms marked before it are all TWO letters, which is not a coincidence, while a longer credential that is tenuous and collides has no second reading worth carrying and is removed. The "rai earns the marking rather than moving" sentence in #vocabulary-collisions gets a dated supersession note rather than a rewrite: it was right that Rai is borne and wrong about which answer that earns. #454 closes by design. mc has no vowel and is not a borne name, and the Scottish prefix never detaches from the name it belongs to, so "Donald Mc" is not a name anyone writes; "Mc Donald" and "John van Mc" hold on the never-given particle membership (#360), which is the one that actually bears on the example. A case row pins the reading whose whole point is that it must not change. What #454 got right is fixed. rules.md#P6's Accepted clause promised that a comma-less name keeps its positional reading, with a de-shaped example that could never falsify it; measured, it fails on exactly the two words in both the particle and the unambiguous suffix vocabulary, and the clause now says so and names them. No example line moves and corpus_rules.jsonl regenerates byte-identical. The two places that already conceded the defect in prose -- the 'Donald mc' bullet in decisions.md and the matching comment in expected_since_1.4.0.toml -- now point at the repair instead of at a standing defect. Derek's 2026-08-23 amendment from #342 rides in verbatim on the comma-suffix-arc Declined entry, and the Excluded (MAIDEN_MARKERS) "z domu" bullet repoints from #291 to #434. The Open block for contested vocabulary memberships is now empty and says so: all three of its entries were answered inside a fortnight. The #vocabulary-collisions size snapshot keeps its date and gains a 2026-09-07 recompute -- 610 acronyms with 5 ambiguous, and 613 was already one high when it was written. docs/customize.rst stops telling callers to leave an uncertain word out and tells them to mark it ambiguous, which is what C-i says, with removal offered as the other direction and named for what it is. The 2.3.0 release notes gain two Behavior Changes bullets. #454 gets none: this log has no precedent for recording a by-design disposition. Closes #454 Co-Authored-By: Claude Fable 5.1 --- docs/customize.rst | 13 ++++++-- docs/design/decisions.md | 35 +++++++++++++++----- docs/design/rules.md | 12 +++++++ docs/release_log.rst | 4 +++ tests/v2/cases.py | 19 +++++++++++ tools/differential/expected_since_1.4.0.toml | 9 +++-- 6 files changed, 77 insertions(+), 15 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 8a027897..6122dc76 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -190,9 +190,16 @@ plausible as ordinary name words on their own (an acronym suffix that doubles as a nickname, a particle that doubles as a given name). They don't add new vocabulary by themselves; they narrow how an existing entry is read when it appears alone. If you're not sure whether a word -you're adding is one of these ambiguous cases, leave it out — an -unrecognized word usually still parses reasonably, while a wrongly -disambiguated one silently picks the less likely reading. (That +you're adding is one of these ambiguous cases, mark it ambiguous — a +wrong ambiguous marking only adds a flag to a reading that was going +to be taken anyway, while a wrong unambiguous claim silently picks the +less likely reading and can lose a real person's surname. The other +direction is to leave the word out of ``suffix_acronyms`` altogether, +which is the right answer for a credential nobody is asking for: a +long, specialized acronym that collides with a common surname earns +removal rather than a marking, and a caller who needs it adds it back +with ``Lexicon.default().add(suffix_acronyms={"cha"})``. That is what +the default vocabulary did with ``rai`` and ``cha`` in 2.3. (The same conservatism is why ``dean`` above isn't in the default vocabulary in the first place: "Dean" is also a common given name, and a default that swallowed it as a title would misparse "Dean Martin" for diff --git a/docs/design/decisions.md b/docs/design/decisions.md index e57afe91..3a3b7ee8 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -10,7 +10,7 @@ Entry conventions: - **Harvested entries** (content landed from a session's report) keep their provenance: a "measurements are that session's, spot-checked at landing" framing line, the contributor's measured-vs-remembered markings where they matter, and — the hard-won one — a flagged uncertainty is resolved by reading the source artifact, never by inference from the neighboring argument. - **`Declined:`** — proposals rejected WITH the evidence that killed them. Resolved-as-no is a decision; without a home for it, the next person re-derives the rejected proposal and its measurement. - **`Excluded:`** — standing prohibitions with indefinite lifetime, keyed by vocabulary set: entries that must stay OUT of a wordlist, each with its reason. Distinct from Declined because the failure mode differs — nobody re-derives a rejected proposal, but someone sweeping a wordlist ships the excluded entry as a bug. -- **`Open:`** — unresolved questions as issue links with one-line handles. The ISSUE is canonical; this block never restates it. Two keyings: under a rule ID for questions about the rule, and — like `Excluded:` — keyed to a VOCABULARY SET for contested memberships, the category neither `deviates:` nor `Excluded:` covers: the rule is right and a word's set membership is the question (rai in the suffix acronyms, swami absent from the given-name titles). Place the block beside the set's `Excluded:` entries so a wordlist editor meets both. +- **`Open:`** — unresolved questions as issue links with one-line handles. The ISSUE is canonical; this block never restates it. Two keyings: under a rule ID for questions about the rule, and — like `Excluded:` — keyed to a VOCABULARY SET for contested memberships, the category neither `deviates:` nor `Excluded:` covers: the rule is right and a word's set membership is the question — a word whose set is questioned, rather than a rule that needs changing. Place the block beside the set's `Excluded:` entries so a wordlist editor meets both. - **Weighing entries** for contested questions: the options considered, each option's intended effect, and the accepted costs of the option chosen. The costs accepted here are the artifacts rules.md lists under the rule's `Accepted:` consequences; the two link by rule ID. ### A2 — the empty name @@ -217,6 +217,8 @@ The mechanism shipped twice before anyone wrote down its criterion. Sizes as of uv run python -c "from nameparser import Parser; L=Parser().lexicon; print({s: len(getattr(L,s)) for s in ('particles','particles_ambiguous','suffix_words','suffix_acronyms','suffix_acronyms_ambiguous','titles')})" +Recomputed 2026-09-07 with that recipe, after this section's own #342 decision landed: particles 70 with 37 ambiguous, suffix_acronyms 610 with 5 ambiguous, titles 758. All THREE of the snapshot's figures were audited against the released 2.2.0 wheel, not the acronym one alone — run the recipe under a PEP 723 script pinning nameparser==2.2.0, from a directory outside the checkout so the tree cannot shadow the wheel, and it reports particles 70 with 37 ambiguous, suffix_acronyms 612 and titles 706. So the snapshot's particles and its titles were off at their own date exactly as its 613 was, and the one that matters to a reader subtracting the two rows is the particles: they have NOT moved since 2.2.0, where "67 with 39" against today's "70 with 37" reads as three additions and two demotions that never happened. What did move: rai and cha left the acronyms and ba joined the ambiguous subset (#suffix-acronym-collisions), and the titles gained 52 over the wheel. The snapshot above stays as written, being evidence from its own date; the argument survives either set of digits, since the two shares still differ by orders of magnitude. + - 2026-08-16 (collision keystone; #348, #360, #342, #385) — the 58%-vs-0.65% gap is BASE RATE, not disagreement. Both sets apply the same test; most particles are short words that double as names (van, bin, le, do, bar, mac) while most credential acronyms are not (abpp, acp). Recorded because the gap reads as an inconsistency and is not one — a reviewer who "harmonizes" the two shares will break one of them. - **C-i, vocabulary vs. name.** A word belongs in its set's ambiguous subset iff it is borne as an ordinary name IN THE POSITION THE VOCABULARY CLAIM ACTS ON. Existence of a bearer anywhere is not the test, and the first draft of this criterion (recorded 2026-08-16, corrected 2026-08-17) got that wrong. Under uncertainty, default to AMBIGUOUS. This generalizes the evidence standard already stated in NON_GIVEN_NAME_PARTICLES' docstring — a wrong unambiguous claim misparses a real person, a wrong ambiguous marking only adds a flag — from "which set" to "which subset". Applies uniformly to particles, suffix_acronyms and titles. - **C-ii, vocabulary vs. vocabulary.** Where two sets claim a word and NEITHER reading is a name, precedence is a frequency judgment recorded per word. vd is the live case: never-given particle AND credential acronym (the British Volunteer Decoration), neither of them a name. Decision: the Dutch van der reading, as the more common. That is what unblocks #380, whose trailing-orphan half is a separate decision recorded under its own rule. @@ -246,7 +248,7 @@ The mechanism shipped twice before anyone wrote down its criterion. Sizes as of genuinely collide where `das` does not. Corrected on #360, which carried the wrong table publicly. - The concrete demonstration is "do", which three vocabularies claimed — titles, particles/ambiguous, suffix_acronyms/ambiguous. Two mark it ambiguous; the third, TITLES, was the one that actually decided "Do Quang Minh" (title="Do", given="Quang") and reported nothing — until #296's audit (2026-08-23) took it out of TITLES, since when the name reads given "Do" and reports the fork. Same word behind #385's "Anh Do". -- Applications, each still its own work: #360 (mc, ste — neither is a borne given name, so both leave the ambiguous half); #342 (rai — Rai IS a borne surname, so it earns the marking rather than moving); #385 (do — resolved at decisions.md#R2). +- Applications, each still its own work: #360 (mc, ste — neither is a borne given name, so both leave the ambiguous half); #342 (rai — Rai IS a borne surname, so it earns the marking rather than moving); #385 (do — resolved at decisions.md#R2). (#342's clause — "rai — Rai IS a borne surname, so it earns the marking rather than moving" — is SUPERSEDED 2026-09-07: Rai is borne, and what that earned was removal from the vocabulary rather than the marking. See #suffix-acronym-collisions for the length criterion that decides between the two.) - C-ii's per-word framing versus a rule stated for a SHAPE: measured 2026-08-16, the words that are both particle and suffix vocabulary are vd, do and mc — three, not the one this criterion adjudicated. rules.md#P6 states its precedence for the shape, so do and mc inherit vd's answer without being weighed. Recorded rather than papered over: stating a per-word judgement as a general clause is how an unexamined word acquires a decision, and the two are named here so the next reader knows which one was actually argued. - Caution when applying C-i to the particle set: TITLES ∩ ambiguous — {do, freiherr, st} until #296's audit took 'do' out of TITLES (2026-08-23; `Do Quang Minh` reads given 'Do' with the fork now), {freiherr, st} since — is load-bearing, per the Excluded note in the W2 section. Emptying it makes the particle-or-given emitter dead code. @@ -411,16 +413,27 @@ Excluded (the never-given / ambiguous particle line, nameparser/config/particles - Encoding rationale (#293, predating #360's membership questions): the data layer stores the NEVER-GIVEN set and derives the ambiguous one, because that is safe-by-default for new particles — a one-place addition — and the v1 shim translates by one-directional complement. And the constants are FROZEN specifically to kill the cached-Lexicon.default()-vs-fresh- Constants desync that runtime module-constant mutation caused. - Load-bearing dependency: TITLES ∩ ambiguous — {do, freiherr, st} until #296's audit, {freiherr, st} since — is what keeps the particle-or-given ambiguity emitter reachable at all; moving them all would leave the shipped vocabulary unable to reach it, which is why test_the_chained_emitter_is_reachable_by_construction (tests/v2/test_parser.py) pins reachability by construction rather than by shipped vocabulary — an empty intersection fails no test and does not mean the emitter is unreachable. -Open (contested vocabulary memberships — the rule is right, the word's set is questioned; the issue is canonical): -[#342](https://github.com/derek73/python-nameparser/issues/342) -rai in SUFFIX_ACRONYMS vs. the South Asian surname. (#346, #343 and -#344 were the other two entries here and were answered together on -2026-09-06 — see #indic-honorifics.) +Open (contested vocabulary memberships — the rule is right, the word's set is questioned; the issue is canonical): none. The block held three entries and all three were answered inside a fortnight — #346, #343 and #344 together on 2026-09-06 (see #indic-honorifics) and #342 on 2026-09-07 (see #suffix-acronym-collisions). The heading stays with nothing under it on purpose, the same reason an empty ledger roster section is a statement and a missing one is nobody having looked. sa, se and om are named as the next candidates by #342's own comment and carry no issue, so they are not entries here. Excluded (SUFFIX_ACRONYMS / SUFFIX_WORDS — the esq dual membership, deliberate; AGENTS.md's gotcha carries the full algebra): - esq is in BOTH sets and must not be "deduplicated". The load-bearing membership is the acronym one (it carries the multi-dot spellings: removing it costs "John Smith E.S.Q." its family name); the word membership is inert as shipped but is what keeps "Esq" matching for a caller who edits suffix_acronyms themselves. esq is the ONLY member of SUFFIX_ACRONYMS ∩ SUFFIX_WORDS — that singleton is why the two sets cannot carry a disjointness assert, which is the standing cost this entry defends. Deliberately no changed-parse count — the count is a property of the measuring grid, not of the code. +### suffix-acronym-collisions — the trailing-position collision class, decided (2026-09-07, #342/#454) + +Closes #342 (a wordlist question) and #454 (a rules.md question) together, because they are the same question asked of two different words and answering one without the other would leave the criterion half-stated. No parser code moves in either. Two existing forks carry the whole thing and both were confirmed on the pre-bundle tree with a throwaway override before any wordlist was edited: `Parser(lexicon=Lexicon.default().remove(suffix_acronyms={"rai","cha"}).add(suffix_acronyms_ambiguous={"ba"})).parse(text)`. + +- **The criterion, and it is about LENGTH as much as about collision.** #vocabulary-collisions C-i decides whether a word is borne as an ordinary name in the position the vocabulary claim acts on, and answers ship or mark. This entry adds the third answer and says when to reach for it. The four entries that carried the marking before this one — ma, do, ed, jd — are all TWO letters, and that is not a coincidence: two letters is too few for an acronym to be anything but common, so both readings are genuinely live and a flag is the honest output. A LONGER credential that is tenuous or specialized, and collides with a common name, has no second reading worth carrying, and the right answer is to REMOVE it from the vocabulary rather than mark it — a caller who needs it adds it back, which is the answer this file already gives for every locale-specific vocabulary. ba, added here, is the two-letter case; rai and cha are the other one. +- **rai and cha left SUFFIX_ACRONYMS.** RAI is "RETA Authorized Instructor"; CHA is Certified Hotel Administrator or Certified Healthcare Auditor. Rai is a common surname across Hindi- and Bengali-speaking regions and Cha is the Korean 차. Both entries arrived in the 2019-12-11 bulk wikipedia post-nominal import (af5bdab) and neither was reviewed against a surname. Measured 2026-09-07: "Aishwarya Rai" reads family Rai where 2.0.0 through 2.2.0 read suffix Rai with no family name at all; "Lala Lajpat Rai" reads middle Lajpat, family Rai; "Kim Cha" reads family Cha. This SUPERSEDES the #vocabulary-collisions sentence "rai — Rai IS a borne surname, so it earns the marking rather than moving", which has a dated supersession note there rather than a rewrite; the sentence was right that Rai is borne and wrong about which of the three answers that earns. +- **The accepted cost, stated rather than discovered.** "John Smith RAI" reads family RAI with Smith a middle name, and "Ahmad Jayadi, CHA" reads given CHA, family "Ahmad Jayadi" — with no credential after the comma, C1 reads an ordinary family comma. Both are case rows, so the reversal is one measurement away for anyone who wants it back. The judgment behind accepting it: a wrong credential reading destroys a real person's surname on every record they appear in, and a missed credential reading leaves the letters in a name field where a human can still see them. +- **The roman-numeral accident, recorded because it looks like a survival and is not.** "John Smith R.A.I." still reads suffix R.A.I. after the removal — but by ACCIDENT, not by vocabulary: rules.md#S3 splits a token with interior periods on those periods and reads the chunks (`_pipeline._vocab.period_joined_vocab`), and the last chunk `i` is a Roman numeral in SUFFIX_WORDS, so "John Smith X.Y.I." and "John Smith R.A.V." read suffix while "John Smith R.A.X." does not — measured 2026-09-07, `period_joined_vocab('R.A.I.')` returning "suffix" and the chunks being r, a, i, never `ra`. "John Smith C.H.A." shows the other half: it read suffix BEFORE the removal, the lookup fold stripping the periods to reach the cha entry, and reads family C.H.A. after. Nothing is pinned on the accident; it is evidence for the parking lot below. +- **ba joined SUFFIX_ACRONYMS_AMBIGUOUS.** BA is a common credential and Ba a real surname (Vietnamese; Senegalese Fula) — the ma/Ma shape exactly. Measured: "Anna Ba" reads family Ba with a suffix-or-name flag (1.4.0's reading plus the flag), "John Smith BA" keeps suffix BA and is newly flagged, "John Smith B.A." is an unflagged suffix through the period gate, and "Smith, BA" reads given BA — the "Smith, Ed" cost, already Accepted under S2 and not reopened here. No corpus name carries a trailing ba, so nothing moves in the differential. +- **#454 is BY DESIGN, and the words are mc and vd.** mc has no vowel and is not a borne name; the Scottish prefix never detaches from the name it belongs to, so "Donald Mc" is not a name anyone writes and its suffix reading costs nothing real. vd needs a surname after it, so a bare trailing vd is the decoration. Neither joins the ambiguous subset and the `fix(suffix-routing)` ledger rule — a two-token name ending in a credential acronym keeps it in `suffix` — stays. What #454's example actually turns on is a different membership: "Mc Donald" reads family "Mc Donald" and "John van Mc" reads family "van Mc" because mc is a never-given PARTICLE (#360), which is why the trailing bare shape is the only one at issue. +- **What #454 got right, and it is a rules.md defect that is now fixed.** rules.md#P6's Accepted clause said that under the default order a comma-less name keeps its positional reading, with "Jong Anke de" → family "de" as its only example. Measured, that holds for de and for do and fails for exactly the two words that are in BOTH the particle vocabulary and the UNAMBIGUOUS suffix vocabulary: "Donald mc" and "Smith vd" lose the family entirely to `suffix`. The clause is repaired to say so, and the two places that already conceded it in prose — the `'Donald mc'` bullet of #differential-ledger's fields-only arc and the corresponding comment in expected_since_1.4.0.toml — are amended from "recorded here rather than fixed" to point at the repair. No example line moves: the clause's own example is de-shaped and stays true, which is why no test caught the defect and why regenerating corpus_rules.jsonl is a no-op here. Recompute the membership with `L = Parser().lexicon; sorted(L.particles & (L.suffix_acronyms - L.suffix_acronyms_ambiguous))`, which gives ['mc', 'vd'] on 2026-09-07, and `sorted(L.particles & L.suffix_acronyms_ambiguous)`, which gives ['do']. +- **Parking lot: the shape-plus-position credential heuristic.** #490's idea without the "collides with vocabulary" part, and what would recover "John Smith RAI" without any wordlist at all. Two shapes: an ALL-CAPS acronym standing in the suffix position of a MIXED-CASE name ("John Smith XYZ", "John Smith, XYZ"), where the case contrast is the signal; and a DOTTED acronym in that position regardless of case ("John Smith X.Y.Z.", "john smith x.y.z."), where the periods are. Measured 2026-09-07, none of the four is read as a suffix today — the bare form gives family XYZ, the comma form given XYZ with family "John Smith", and the second two family X.Y.Z. — while the roman-numeral accident above reads three dotted forms as suffixes on a fork that is about generations, not credentials. That contrast is the evidence the dotted shape is unhandled: the parser already produces the wanted answer for X.Y.I. and R.A.V. by coincidence and the unwanted one for R.A.X. A follow-up issue carries it; nothing in this entry depends on it. +- **Not decided here:** sa stays where #296's audit put it (title and suffix dual, position decides); se and om stay unambiguous, there being no surname evidence worth standing behind and OM being the Order of Merit; the S2 comma-form boundary ("Smith, Ed" → given Ed) is not reopened. #342's own comment lists sa, se and om beside ba and cha as model recall rather than corpus-attested, and per the #360 lesson they need a human before they move. +- **Measurement (2026-09-07).** Five corpus names move, every one from the radar-tier corpus_issues.jsonl, in three diff shapes with one cause: Aishwarya Rai moves {family, suffix}, Lala Lajpat Rai and John Smith RAI move {middle, family, suffix}, and John Smith, RAI and Ahmad Jayadi, CHA move {given, family, suffix}. Recompute by parsing every name in the tools/differential/corpus*.jsonl glob twice — once with the shipped lexicon, once with `Lexicon.default().add(suffix_acronyms={"rai","cha"})`, which RESTORES the two entries this bundle removed so the diff read forwards is the removal's — and diffing the seven name fields plus `ambiguities`; the ba step of the same sweep moves nothing. Aishwarya Rai is the one name whose diff DISAPPEARS at 1.4.0, that release having read family Rai too, which is why the `fix(#342) NOT WANTED` ledger rule was deleted rather than rewritten and why the 1.4.0 gate lists four names under the replacement where the three 2.x ledgers list five. Read today's intentional counts off the `corpus:` line of `uv run python tools/differential/compare.py --baseline X`; they rose by three at 1.4.0 and by five at each 2.x baseline. + ### indic-honorifics — the renunciate class and the Indic honorific vocabulary (2026-09-06, #346/#344/#343) Closes #346, #344 and #343 as one bundle: each CLASS is decided once, and each script's membership in it is then argued word by word, so the three lists overlap without matching. No parser code moves in it. Four existing forks carry the whole thing — the renunciate fold (rules.md#H1), the spaced trailing suffix path, title vocabulary beating the one-letter-plus-period initial shape, and the unvalidated given_name_titles — and all four were confirmed on the pre-bundle tree with a throwaway override before any wordlist was edited — the recipe is `Parser(lexicon=Lexicon.default().add(given_name_titles={...}, titles={...}, suffix_words={...})).parse(text)`, and the fourth fork is the one a reader will not expect: `Lexicon` deliberately does not validate `given_name_titles` against `titles` (see #given-name-titles), so an entry in the given-name set alone is never recognized as a title at all. In the shipped data that is free, TITLES being GIVEN_NAME_TITLES | {...}. @@ -560,7 +573,7 @@ Excluded (Policy.script_orders defaults): Script.KATAKANA is deliberately absent Excluded (MAIDEN_MARKERS, per nameparser/config/maiden_markers.py): -- Polish "z domu" — a two-token marker; pending the multi-token matching decision, tracked in #291 since 2026-07-27. +- Polish "z domu" — the exclusion is LIFTED and the phrase SHIPS; see the INCLUDED entry below for the argument. It was provisional from the day it was written, 2026-07-27, pending the multi-token matching decision — first tracked in #291, repointed to #434 on 2026-09-07 per Derek's amendment on #342, markers having no run predicate, so the multi-token question does not dissolve there the way the suffix one did — and #434 shipped whole-phrase marker matching on 2026-08-26, which is what lifted it. Kept rather than deleted because a provisional exclusion that was answered reads, once deleted, as an exclusion nobody ever considered. - Contrast entry — unaccented "nee" SHIPS as a marker despite #274 flagging "is nee safe as a default (it's also a rare surname)" as open; the question resolved silently with the shipped set. Recorded here because the included risky member deserves its analysis as much as the excluded ones; M1's (Nee) boundary covers only the enclosure path, not this marker path. @@ -771,7 +784,7 @@ Declined: - A literal alternation of all 14 absorbed names (2026-08-27), on #413's literal-anchored precedent — kept as the fallback if a reading turned out contested, and not needed: every family had an existing rule in another ledger, a decision to cite, or an open issue. - Extending #451 to the trailing-numeral ambiguity (2026-08-27) — `'Mohamad X'` loses its family to `_ROMAN`, and by C-i's "under uncertainty, default to AMBIGUOUS" a trailing `X` IS borne as an ordinary name in that position (Malcolm X). Numerals reach `suffix` through a pattern rather than a vocabulary set, so there is no ambiguous SUBSET for `x` to join — but the fork does not go unreported: measured, `parse('Mohamad X')` and `parse('John V')` both carry `AmbiguityKind.SUFFIX_OR_NAME`, which `_assign` raises from the fork itself because "V/X/I are ordinary middle initials, so taking it as a suffix is a call, not a fact". The contrast worth keeping is with the acronym half: `parse('Aishwarya Rai')` loses its family and reports NOTHING, because `rai` is unambiguous vocabulary. So the numeral fork already flags its own uncertainty and the vocabulary path does not — which is an argument about `SUFFIX_ACRONYMS_AMBIGUOUS`, not about `_ROMAN`, and it belongs to #342 rather than here. `'Malcolm X'` is in no corpus (`'Mohamad X'` is, and is what this bundle classified); no open issue on the numeral half. - `explains = N` per rule (2026-08-28, #452's own first sketch) — catches a rule explaining fewer NAMES even while its fields stay exercised, which the field check does not. Declined on maintenance cost rather than merit: 179 numbers across the three ledgers, every one of them liable to move whenever a corpus does rather than only when a rule does. Measured on the precedent — #414's harvest moved 17 of the 30 `_CORPUS_CLAIMS` entries that existed then and added 14 more, parsing the roster at `7a10689^` and `7a10689`. (That roster's own comment said "Twenty of these moved" and was wrong; measured three ways — that commit, the whole of PR #415, and the backtick-harvest commit after it — the figure is 17 every time, and the comment now says so with the recipe.) The field check needs no number and cannot go stale. This is the design to revisit if a name-count shrink ever bites. -- Extending it to `'Donald mc'` (2026-08-27) — `mc` is `SUFFIX_ACRONYMS` AND `PARTICLES`, the collision C-ii decided for `vd`, and C-ii reaches this shape only through P6's attachment, which is scoped to the comma form. The authority is rules.md#S2's Accepted — "an unambiguous suffix is consumed even when that leaves no family name at all" — and NOT P6's scope note, which was this entry's first citation and does not support it. P6 promises the comma-less shapes "keep their positional reading", and for the words that are both particle and suffix vocabulary it does not hold: measured, `'Jong de'` and `'Smith do'` keep theirs (family 'de', family 'do') while `'Donald mc'` and `'Smith vd'` lose the family entirely to `suffix`. So `'Donald mc'` is out of #451's scope on S2's statement, and P6's own Accepted clause is falsified by two of the three words it sweeps in — its only example, `"Jong Anke de"`, is `de`-shaped, which is why no test catches it. Recorded here rather than fixed; it is a rules.md defect, not a ledger one. +- Extending it to `'Donald mc'` (2026-08-27) — `mc` is `SUFFIX_ACRONYMS` AND `PARTICLES`, the collision C-ii decided for `vd`, and C-ii reaches this shape only through P6's attachment, which is scoped to the comma form. The authority is rules.md#S2's Accepted — "an unambiguous suffix is consumed even when that leaves no family name at all" — and NOT P6's scope note, which was this entry's first citation and does not support it. P6 promises the comma-less shapes "keep their positional reading", and for the words that are both particle and suffix vocabulary it does not hold: measured, `'Jong de'` and `'Smith do'` keep theirs (family 'de', family 'do') while `'Donald mc'` and `'Smith vd'` lose the family entirely to `suffix`. So `'Donald mc'` is out of #451's scope on S2's statement, and P6's own Accepted clause is falsified by two of the three words it sweeps in — its only example, `"Jong Anke de"`, is `de`-shaped, which is why no test catches it. Recorded here as a rules.md defect rather than a ledger one, and FIXED 2026-09-07 under #454: P6's Accepted clause now states the exception, naming mc and vd as the two words the positional reading does not hold for. The measurement above is unchanged and is what the repaired clause says. See #suffix-acronym-collisions. ### differential-ledger, the corpus-tier arc (2026-09-01, #468/#469) @@ -987,6 +1000,10 @@ Declined: "Smith, A.P." has the suffix steal the given initials; "John Leed" and "Mary Nicet" lose family names; and the period-gate escape is equivalent to removal because nobody writes "L.E.E.D.". This decline is why the seven removable entries were REMOVED in 2.0 rather than split, and it is the missing history behind C1's #291 marker. + + Amended 2026-08-23 (#291 closed working-as-designed): the decline stands as measured — splitting into single words does cost `Smith, A.P.`, `John Leed` and `Mary Nicet` — but the inference drawn from it did not. "A multi-word *entry* is inert" was read as "a multi-word *credential* is unreachable", and that is false: the run predicate `is_wholly_suffix` reassembles adjacent suffix tokens, so `parse("John Smith, MD PhD").suffix` has been `'MD PhD'` since 1.4.0, and `psm i`/`psm ii` — two of the seven entries called dead — parse today from their component words. The third option neither the issue nor the spec considered is to leave the shipped vocabulary alone and let callers add the component words to a `Lexicon`, which is what was chosen. + + Also declined with it: `SUFFIX_PHRASES` and segment-level matching (unnecessary once the run predicate is measured), and amendment A6's glued-peel question (no phrase-matching unit, so `_is_post_nominal`'s token-level test stays correct by construction). - The trailing-abbreviation structural fallback — with a measurement LIMIT rather than a measurement: the differential corpora structurally cannot evidence it, because they hold only names someone wrote down, and an unrecognized abbreviation is by definition outside the vocabulary. A green run there proves nothing (the reusable harness fact is in mechanisms.md's field notes). - SUFFIX_PHRASES matching in assignment — measured cost: it renders suffix="LEED, AP", because the suffix view comma-joins suffix words unless they carry the stable "joined" tag, which grouping applies within a piece and — since #436/#437, 2026-09-06 — post_rules applies between two post-nominals the writer did not comma-separate. The general form: multi-word vocabulary must merge where the render tag is applied, not where the role is assigned. diff --git a/docs/design/rules.md b/docs/design/rules.md index 9d914629..9de7c9cb 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -481,6 +481,18 @@ P6. Rationale: a particle ending the name has nothing to link listing, and a bare "Jong de" may be a given name beside a particle — so nothing there names the family, the positional reading stands, and the same input reports family "de" here. + That is the reading for a particle the UNAMBIGUOUS suffix + vocabulary does not also claim, which is all but two of them. + Where the word is both a particle and UNAMBIGUOUS suffix + vocabulary — vd and mc — nothing has named the family, so the + attachment above never fires and S2 takes the trailing word as + a post-nominal, even though that leaves no family name at all: + a bare "Donald mc" reports suffix "mc" with an empty family, + and so does "Smith vd". The comma is the contrast, and it is + where the precedence stated above acts, "Berg, Jan vd" reading + family "vd Berg". `do` follows `de` rather than `mc`, sitting + in the AMBIGUOUS acronym half, where S2's words-to-spare test + leaves a two-word name its family. "Jong Anke de" → family="de" Accepted: the precedence over S2 is stated for the shape, so it sweeps in every word that is both particle and suffix diff --git a/docs/release_log.rst b/docs/release_log.rst index b5241b03..bb3649a5 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -14,6 +14,10 @@ Release Log - **Fix Parser.revise() splitting a space-separated suffix value into comma-separated entries.** ``Parser().revise(n, suffix="MD PhD").suffix`` is ``MD PhD`` and was ``MD, PhD``, and a name's own rendered suffix now revises back to itself for all but one of the 38 differential-corpus names that failed to on 2026-09-06 (368 of the 1117 carry a suffix) (recipe in the ``C1`` entry of ``docs/design/decisions.md``; the one left is a Korean honorific glued to an initial, which the value's own parse peels where the whole name kept it glued -- a word read on its own, not an entry boundary). A suffix value's entries are now derived from the value's own commas by the same rule a whole name uses: a comma parts two credentials and a space joins them, so ``revise(n, suffix="MD, PhD")`` is still two entries. ``revise(n, suffix="Ph. D.")`` renders ``Ph. D.`` where the 2.2.0 note below accepted ``Ph., D.``; the head-position merge that note describes still does not fire, the pair joining under the entry rule instead. One limit: a delimiter configured through ``extra_suffix_delimiters`` parts a value only where the value's own words read as a name with a tail segment, so in a run of post-nominals it stays a word; write a comma at the boundary instead. ``ParsedName.replace()`` is unchanged (closes #511) + - **Remove rai and cha from the default post-nominal acronyms, so a trailing Rai or CHA keeps the family name.** ``HumanName("Aishwarya Rai")`` gives last ``Rai``, where 2.0.0 through 2.2.0 gave suffix ``Rai`` and no last name at all -- 1.4.0's reading, restored -- and ``Lala Lajpat Rai`` gives middle ``Lajpat``, last ``Rai``. Both entries came in with a 2019 bulk import of Wikipedia post-nominals and were never reviewed against the surnames they collide with. The cost is that a genuine credential written after a full name is no longer recognized: ``John Smith RAI`` gives middle ``Smith``, last ``RAI``, and the comma forms swap ends -- ``John Smith, RAI`` gives first ``RAI``, last ``John Smith``, and ``Ahmad Jayadi, CHA`` first ``CHA``, last ``Ahmad Jayadi``. A caller who needs either back adds it: ``Lexicon.default().add(suffix_acronyms={"cha"})``. Five names move in the differential corpora, all of them radar-tier. See the ``suffix-acronym-collisions`` entry of ``docs/design/decisions.md`` (closes #342) + + - **Mark ba as an acronym that is also an ordinary name, so a bare trailing Ba keeps the family name.** ``HumanName("Anna Ba")`` gives last ``Ba`` and reports a suffix-or-name ambiguity, where 2.0.0 through 2.2.0 gave suffix ``Ba`` and no last name. A full name in front leaves the credential reading intact: ``John Smith BA`` still gives suffix ``BA``, now flagged, and the dotted ``John Smith B.A.`` is an unflagged suffix, the periods settling it. The comma form costs what ``Smith, Ed`` costs -- ``Smith, BA`` gives first ``BA`` -- which S2 already accepted for the other ambiguous acronyms. BA is a common credential and Ba a real surname in Vietnamese and Senegalese Fula, which is the ``ma``/``Ma`` shape exactly. No corpus name moves (#342) + **Additions** - **Add the renunciate titles to the given-name title list, so a renunciate's one name is a given name.** ``HumanName("Swami Vivekananda")`` gives first ``Vivekananda`` with an empty last name, where every release since 1.4.0 gave last ``Vivekananda``; ``Guru Nanak``, ``Baba Ramdev`` and ``Lama Zopa`` move the same way, and so do the Devanagari and Bengali spellings added below. Two name words behind the title are unchanged -- ``Swami Vivekananda Saraswati`` keeps last ``Saraswati`` -- and a surname-retaining title is untouched: ``Rabbi Cohen`` still gives last ``Cohen``. ``venerable`` is deliberately not in the list, the traditions using it splitting on whether the family name survives. See the ``indic-honorifics`` entry of ``docs/design/decisions.md`` (closes #346) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 878307fb..f0a1e050 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -465,6 +465,25 @@ def _check_cjk_shape_purity(self) -> None: "NOT move -- only the flag is new. The twin of " "ambiguous_acronym_is_a_suffix_when_a_family_name_remains " "above"), + Case("by_design_trailing_mc_reads_as_a_credential", "Donald Mc", + {"given": "Donald", "suffix": "Mc"}, + notes="#454, closed by design 2026-09-07. This is the one " + "row whose point is that it must NOT change. 'mc' is " + "SUFFIX_ACRONYMS and PARTICLES both, and it is not a " + "borne name: it has no vowel, and the Scottish prefix " + "never detaches from the name it belongs to, so " + "'Donald Mc' is not a name anyone writes and its " + "suffix reading costs nothing real. rules.md#P6's " + "attachment is scoped to shapes where something has " + "already named the family, which this has not, so S2 " + "takes the trailing word -- 'an unambiguous suffix is " + "consumed even when that leaves no family name at " + "all'. The neighbours that DO hold: 'Mc Donald' reads " + "family 'Mc Donald' and 'John van Mc' family 'van " + "Mc', both because 'mc' is a never-given particle " + "(#360), which is the membership that actually bears " + "on #454's example. See " + "decisions.md#suffix-acronym-collisions"), Case("ambiguous_acronym_suffix_with_middle", "John Q Smith MA", {"given": "John", "middle": "Q", "family": "Smith", "suffix": "MA"}, diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 728194d7..a7f49a90 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -2666,9 +2666,12 @@ issue = "fix(suffix-routing) a two-token name ending in a credential acronym kee # for the words that are both particle and suffix vocabulary it does # not hold. Measured, 'Jong de' and 'Smith do' keep theirs (family # 'de', family 'do') while 'Donald mc' and 'Smith vd' lose the family -# entirely to `suffix`. decisions.md#P6's own arc entry records that as -# a rules.md defect rather than a ledger one; this rule classifies the -# movement on S2's statement and settles nothing about P6. +# entirely to `suffix`. decisions.md#P6's own arc entry recorded that +# as a rules.md defect rather than a ledger one, and #454 fixed it on +# 2026-09-07: P6's Accepted clause now names vd and mc as the two words +# the positional reading does not hold for, which is what this rule has +# been classifying all along. This rule is unchanged by that repair and +# still rests on S2's statement. # # The two neighbouring shapes both already have rules above, and the # contrast is the point: 'Mc Donald' has the particle LEADING and folds From 75cd103b74ae52da482580bb4edb6dc06b89ec6a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 7 Sep 2026 18:34:10 -0700 Subject: [PATCH 4/4] review round: PR #515 findings Three review agents on the whole PR, plus the maintainer's clarification of the criterion: the decision weighs how common a word is as a borne name against how common it is as a credential -- the name reading dominating removes the entry (rai, cha), rough balance earns the ambiguous marking (ba, the ma shape), the credential dominating keeps it unambiguous. Length is a correlate, not the test; every statement of the criterion now says so. Prose: customize.rst no longer claims an ambiguous marking "only adds a flag" (it moves the comma and bracketed forms, and this PR's own "Smith, BA" is the counterexample); the ba release note states the comma and bracketed costs with their before-values; the wordlist comment stops stating the R.A.I. accident as a rule of removal and carries the ba/se/om/mc/vd reasons where a sweep meets them; AGENTS.md's collision digits are recomputed; rules.md#P6 stops pointing at the emptied Open block. Tests and rosters: Donald Mc is fix(suffix-routing), not parity, and Aishwarya Rai is parity, not fix -- both checked against the 1.4.0 wheel; "Smith, BA" and "John Smith C.H.A." are pinned; _WATCHED_DIFFS gains the two sole-watched arrivals the removal created; the guard roster gains the comma-truncation probes. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 2 +- docs/customize.rst | 23 +++++++++---- docs/design/decisions.md | 8 ++--- docs/design/rules.md | 5 +-- docs/release_log.rst | 2 +- nameparser/config/suffixes.py | 60 ++++++++++++++++++++++------------ tests/v2/cases.py | 54 ++++++++++++++++++++++++------ tests/v2/test_ledger_guards.py | 19 +++++++---- tools/differential/compare.py | 26 ++++++++++++--- 9 files changed, 141 insertions(+), 58 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 16a45834..d34c1521 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -253,7 +253,7 @@ The library has two layers: `nameparser/config/` (data) and `nameparser/parser.p **Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. Since #312 it also crosses the 间隔号, which still stops the surname split standing right beside it: it answers where a name DIVIDES into surname and given, and the peel never asks that question. Whether it also crosses the FAMILY comma is tolerated rather than settled: the 2026-09-01 demotion (rules.md#W3) narrowed that half from contract to best-effort, since no CJK writing system's own convention puts a comma between family and given at all — so `김, 민준씨` reads today exactly as the spaced `김 민준씨` does (family 김, given 민준, suffix 씨) while the split stands down as before, but that reading is watched on the differential's radar tier rather than pinned as contract. Its site is accordingly the name-bearing segment runs — `segments[:2]` under a family comma, and `segments[0]` as before otherwise, the family comma being the one structure that splits the name itself across two runs, with the honorific as often glued to the given side as to the family. That is the whole reach and nothing past it (`김, 민준 지훈씨` peels; `김, 민준, 지훈씨` and `김,, 민준씨` do not, both landing in a third run), and whether `segments[1]` is name text at all is now ASKED rather than inferred from the structure — `segment` does not guarantee it, since a one-word part before the comma reads as FAMILY_COMMA even when the part after it is entirely suffix-shaped, and the peel walking into such a run took `V.` for its site, found no listed tail and abandoned (#319). The question is `segment`'s own suffix-comma predicate, lifted into `_vocab.is_wholly_suffix` so the two stages cannot drift: a wholly suffix-shaped second run is declined and the scan stays in `segments[0]`, so `田中さん, V.` and `田中さん, Ph. D.` give さん up as `田中さん, PhD` always did. The test is necessary but NOT sufficient, and the second condition is not decoration: every honorific tail is also a suffix word, so a glued honorific is itself part of what makes its run read as suffix-shaped, and declining a run that holds the ONLY site loses the peel outright. `segments[0]` must therefore offer a peel site of its own before the second run is declined — `이, J.씨` and `선생님, J.씨` pass the suffix test and are scanned anyway, keeping the pre-#319 reading, while `김민준씨, J.씨` has a site on both sides and peels the person's own 씨 rather than the junk one behind the comma. Uniform in the PEEL, that is — where the credential itself lands is `assign`'s question and still differs by spelling (`V.` → `given`, `PhD` and `Ph. D.` → `suffix`). Not `_is_post_nominal` pluralized: the run predicate says yes both to what the token predicate vetoes (`V.`, `V`, `I` — the class the defect was reported as) and to what the token predicate never sees at all, since `period_joined_vocab` and the delimiter routes are the run predicate's alone (`Msc.Ed.` and `J.씨` reach it that way, and `田中さん, Msc.Ed.` moves with the rest). `Policy(lenient_comma_suffixes=False)` drops this call to the strict token test too — so those three read as name text again and keep the pre-#319 answer, while `Ph. D.` peels under the knob regardless, its merged `phd` passing the strict test. `田中さん, 太郎` is unchanged, and not because of its comma — the honorific there is not at the end of the name, 太郎 is. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. -**A constant's membership is a question you may reopen.** Proposing that a word be ADDED, REMOVED or MOVED between vocabulary sets is ordinary design work — a shipped entry is not evidence that anyone judged it. `SUFFIX_ACRONYMS` arrived in `af5bdab` as a bulk Wikipedia import never reviewed against surname collisions: 575 of its 579 alphabetic entries leave `family` empty in `"John "` against four ambiguous-gated exceptions, and `rai` (#342), `ba`, `cha`, `sa`, `se`, `om` and `mc` are all borne as surnames (measured 2026-08-23). When a fix starts to look like new machinery, check the vocabulary first. Criterion: `decisions.md#vocabulary-collisions`, with #360's positional qualifier. +**A constant's membership is a question you may reopen.** Proposing that a word be ADDED, REMOVED or MOVED between vocabulary sets is ordinary design work — a shipped entry is not evidence that anyone judged it. `SUFFIX_ACRONYMS` arrived in `af5bdab` as a bulk Wikipedia import never reviewed against surname collisions: 572 of its 577 alphabetic entries leave `family` empty in `"John "` against five ambiguous-gated exceptions (recomputed 2026-09-07; the fifth is `ba`), and `sa`, `se` and `om` are borne as surnames (measured 2026-08-23), as were `rai`, `cha`, `ba` and `mc` before `decisions.md#suffix-acronym-collisions` decided all four — `rai` and `cha` removed, `ba` marked ambiguous, `mc` left alone as no borne name at all. When a fix starts to look like new machinery, check the vocabulary first. Criterion: `decisions.md#vocabulary-collisions`, with #360's positional qualifier. **Sweep the forms a change can reach: comma shapes, then `name_order`.** No comma, a FULL name before the comma, and a ONE-WORD name before the comma are three paths, and the third is the miss — #429, #430 and #432 are all that path disagreeing with the full-name path on inputs the full-name path reads correctly. For orders the sweep already exists (`tests/v2/test_cases.py` runs every row under all three) but carries one assertion, R2's family partition, so it checks nothing a new change moves; coverage there has been vacuous before (PR #394's review found the suite passed with `name_order` discarded from grouping). Parse your change's names in each comma shape and each order, and read the ones you did not predict. diff --git a/docs/customize.rst b/docs/customize.rst index 6122dc76..70bd2475 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -190,14 +190,23 @@ plausible as ordinary name words on their own (an acronym suffix that doubles as a nickname, a particle that doubles as a given name). They don't add new vocabulary by themselves; they narrow how an existing entry is read when it appears alone. If you're not sure whether a word -you're adding is one of these ambiguous cases, mark it ambiguous — a -wrong ambiguous marking only adds a flag to a reading that was going -to be taken anyway, while a wrong unambiguous claim silently picks the -less likely reading and can lose a real person's surname. The other +you're adding is one of these ambiguous cases, weigh how often it is a +name against how often it is the credential. Marking it ambiguous is +not free in either direction: an ambiguous acronym counts as a suffix +only when written with its periods, so the bare spelling reads as a +name and the parse reports the fork, and the comma form moves with it +— ``Smith, BA`` reads first ``BA`` rather than suffix ``BA``, and a +bracketed ``John Smith (BA)`` falls through to nickname parsing. A +wrong unambiguous claim takes the credential reading silently and can +lose a real person's surname. For ``particles_ambiguous`` the default +runs the other way: a particle that is not borne as a given name +belongs in the never-given half, which is where ``mc`` and ``ste`` +were moved (#360). The other direction is to leave the word out of ``suffix_acronyms`` altogether, -which is the right answer for a credential nobody is asking for: a -long, specialized acronym that collides with a common surname earns -removal rather than a marking, and a caller who needs it adds it back +which is the right answer when the name reading is the far more common +one: an acronym whose credential is tenuous or specialized beside a +common surname earns removal rather than a marking, and a caller who +needs it adds it back with ``Lexicon.default().add(suffix_acronyms={"cha"})``. That is what the default vocabulary did with ``rai`` and ``cha`` in 2.3. (The same conservatism is why ``dean`` above isn't in the default vocabulary in diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 3a3b7ee8..d372d244 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -10,7 +10,7 @@ Entry conventions: - **Harvested entries** (content landed from a session's report) keep their provenance: a "measurements are that session's, spot-checked at landing" framing line, the contributor's measured-vs-remembered markings where they matter, and — the hard-won one — a flagged uncertainty is resolved by reading the source artifact, never by inference from the neighboring argument. - **`Declined:`** — proposals rejected WITH the evidence that killed them. Resolved-as-no is a decision; without a home for it, the next person re-derives the rejected proposal and its measurement. - **`Excluded:`** — standing prohibitions with indefinite lifetime, keyed by vocabulary set: entries that must stay OUT of a wordlist, each with its reason. Distinct from Declined because the failure mode differs — nobody re-derives a rejected proposal, but someone sweeping a wordlist ships the excluded entry as a bug. -- **`Open:`** — unresolved questions as issue links with one-line handles. The ISSUE is canonical; this block never restates it. Two keyings: under a rule ID for questions about the rule, and — like `Excluded:` — keyed to a VOCABULARY SET for contested memberships, the category neither `deviates:` nor `Excluded:` covers: the rule is right and a word's set membership is the question — a word whose set is questioned, rather than a rule that needs changing. Place the block beside the set's `Excluded:` entries so a wordlist editor meets both. +- **`Open:`** — unresolved questions as issue links with one-line handles. The ISSUE is canonical; this block never restates it. Two keyings: under a rule ID for questions about the rule, and — like `Excluded:` — keyed to a VOCABULARY SET for contested memberships, the category neither `deviates:` nor `Excluded:` covers: the rule is right and a word's set membership is the question (the shape: `rai` in the suffix acronyms and `swami` absent from the given-name titles, both since answered — see #suffix-acronym-collisions and #indic-honorifics). Place the block beside the set's `Excluded:` entries so a wordlist editor meets both. - **Weighing entries** for contested questions: the options considered, each option's intended effect, and the accepted costs of the option chosen. The costs accepted here are the artifacts rules.md lists under the rule's `Accepted:` consequences; the two link by rule ID. ### A2 — the empty name @@ -248,7 +248,7 @@ Recomputed 2026-09-07 with that recipe, after this section's own #342 decision l genuinely collide where `das` does not. Corrected on #360, which carried the wrong table publicly. - The concrete demonstration is "do", which three vocabularies claimed — titles, particles/ambiguous, suffix_acronyms/ambiguous. Two mark it ambiguous; the third, TITLES, was the one that actually decided "Do Quang Minh" (title="Do", given="Quang") and reported nothing — until #296's audit (2026-08-23) took it out of TITLES, since when the name reads given "Do" and reports the fork. Same word behind #385's "Anh Do". -- Applications, each still its own work: #360 (mc, ste — neither is a borne given name, so both leave the ambiguous half); #342 (rai — Rai IS a borne surname, so it earns the marking rather than moving); #385 (do — resolved at decisions.md#R2). (#342's clause — "rai — Rai IS a borne surname, so it earns the marking rather than moving" — is SUPERSEDED 2026-09-07: Rai is borne, and what that earned was removal from the vocabulary rather than the marking. See #suffix-acronym-collisions for the length criterion that decides between the two.) +- Applications, each still its own work: #360 (mc, ste — neither is a borne given name, so both leave the ambiguous half); #342 (rai — Rai IS a borne surname, so it earns the marking rather than moving); #385 (do — resolved at decisions.md#R2). (#342's clause — "rai — Rai IS a borne surname, so it earns the marking rather than moving" — is SUPERSEDED 2026-09-07: Rai is borne, and what that earned was removal from the vocabulary rather than the marking. See #suffix-acronym-collisions for the relative-frequency criterion that decides between the two.) - C-ii's per-word framing versus a rule stated for a SHAPE: measured 2026-08-16, the words that are both particle and suffix vocabulary are vd, do and mc — three, not the one this criterion adjudicated. rules.md#P6 states its precedence for the shape, so do and mc inherit vd's answer without being weighed. Recorded rather than papered over: stating a per-word judgement as a general clause is how an unexamined word acquires a decision, and the two are named here so the next reader knows which one was actually argued. - Caution when applying C-i to the particle set: TITLES ∩ ambiguous — {do, freiherr, st} until #296's audit took 'do' out of TITLES (2026-08-23; `Do Quang Minh` reads given 'Do' with the fork now), {freiherr, st} since — is load-bearing, per the Excluded note in the W2 section. Emptying it makes the particle-or-given emitter dead code. @@ -423,12 +423,12 @@ Excluded (SUFFIX_ACRONYMS / SUFFIX_WORDS — the esq dual membership, deliberate Closes #342 (a wordlist question) and #454 (a rules.md question) together, because they are the same question asked of two different words and answering one without the other would leave the criterion half-stated. No parser code moves in either. Two existing forks carry the whole thing and both were confirmed on the pre-bundle tree with a throwaway override before any wordlist was edited: `Parser(lexicon=Lexicon.default().remove(suffix_acronyms={"rai","cha"}).add(suffix_acronyms_ambiguous={"ba"})).parse(text)`. -- **The criterion, and it is about LENGTH as much as about collision.** #vocabulary-collisions C-i decides whether a word is borne as an ordinary name in the position the vocabulary claim acts on, and answers ship or mark. This entry adds the third answer and says when to reach for it. The four entries that carried the marking before this one — ma, do, ed, jd — are all TWO letters, and that is not a coincidence: two letters is too few for an acronym to be anything but common, so both readings are genuinely live and a flag is the honest output. A LONGER credential that is tenuous or specialized, and collides with a common name, has no second reading worth carrying, and the right answer is to REMOVE it from the vocabulary rather than mark it — a caller who needs it adds it back, which is the answer this file already gives for every locale-specific vocabulary. ba, added here, is the two-letter case; rai and cha are the other one. +- **The criterion, and it is a comparison of RELATIVE FREQUENCY.** #vocabulary-collisions C-i decides whether a word is borne as an ordinary name in the position the vocabulary claim acts on, and answers ship or mark. This entry adds the third answer and says when to reach for it. Being borne at all is only C-i's entry ticket; what chooses among the three answers is how common the word is as a borne name in the TRAILING position weighed against how common it is as a credential. Three outcomes. The NAME reading dominating removes the entry from the vocabulary altogether — a caller who needs it adds it back, which is the answer this file already gives for every locale-specific vocabulary — and that is rai and cha, far more common as surnames than RAI and CHA are as credentials. Rough balance earns the ambiguous marking, both readings being genuinely live so that a flag is the honest output, and that is ba, about as common a surname as BA is a credential. The CREDENTIAL dominating leaves the entry unambiguous, which is where the great majority of the set sits. LENGTH is a CORRELATE and not the test: a short acronym is more often a common credential and more often a name, which is why the four entries carrying the marking before this one — ma, do, ed, jd — are all two letters, and reading that as the rule is what this bullet exists to stop. - **rai and cha left SUFFIX_ACRONYMS.** RAI is "RETA Authorized Instructor"; CHA is Certified Hotel Administrator or Certified Healthcare Auditor. Rai is a common surname across Hindi- and Bengali-speaking regions and Cha is the Korean 차. Both entries arrived in the 2019-12-11 bulk wikipedia post-nominal import (af5bdab) and neither was reviewed against a surname. Measured 2026-09-07: "Aishwarya Rai" reads family Rai where 2.0.0 through 2.2.0 read suffix Rai with no family name at all; "Lala Lajpat Rai" reads middle Lajpat, family Rai; "Kim Cha" reads family Cha. This SUPERSEDES the #vocabulary-collisions sentence "rai — Rai IS a borne surname, so it earns the marking rather than moving", which has a dated supersession note there rather than a rewrite; the sentence was right that Rai is borne and wrong about which of the three answers that earns. - **The accepted cost, stated rather than discovered.** "John Smith RAI" reads family RAI with Smith a middle name, and "Ahmad Jayadi, CHA" reads given CHA, family "Ahmad Jayadi" — with no credential after the comma, C1 reads an ordinary family comma. Both are case rows, so the reversal is one measurement away for anyone who wants it back. The judgment behind accepting it: a wrong credential reading destroys a real person's surname on every record they appear in, and a missed credential reading leaves the letters in a name field where a human can still see them. - **The roman-numeral accident, recorded because it looks like a survival and is not.** "John Smith R.A.I." still reads suffix R.A.I. after the removal — but by ACCIDENT, not by vocabulary: rules.md#S3 splits a token with interior periods on those periods and reads the chunks (`_pipeline._vocab.period_joined_vocab`), and the last chunk `i` is a Roman numeral in SUFFIX_WORDS, so "John Smith X.Y.I." and "John Smith R.A.V." read suffix while "John Smith R.A.X." does not — measured 2026-09-07, `period_joined_vocab('R.A.I.')` returning "suffix" and the chunks being r, a, i, never `ra`. "John Smith C.H.A." shows the other half: it read suffix BEFORE the removal, the lookup fold stripping the periods to reach the cha entry, and reads family C.H.A. after. Nothing is pinned on the accident; it is evidence for the parking lot below. - **ba joined SUFFIX_ACRONYMS_AMBIGUOUS.** BA is a common credential and Ba a real surname (Vietnamese; Senegalese Fula) — the ma/Ma shape exactly. Measured: "Anna Ba" reads family Ba with a suffix-or-name flag (1.4.0's reading plus the flag), "John Smith BA" keeps suffix BA and is newly flagged, "John Smith B.A." is an unflagged suffix through the period gate, and "Smith, BA" reads given BA — the "Smith, Ed" cost, already Accepted under S2 and not reopened here. No corpus name carries a trailing ba, so nothing moves in the differential. -- **#454 is BY DESIGN, and the words are mc and vd.** mc has no vowel and is not a borne name; the Scottish prefix never detaches from the name it belongs to, so "Donald Mc" is not a name anyone writes and its suffix reading costs nothing real. vd needs a surname after it, so a bare trailing vd is the decoration. Neither joins the ambiguous subset and the `fix(suffix-routing)` ledger rule — a two-token name ending in a credential acronym keeps it in `suffix` — stays. What #454's example actually turns on is a different membership: "Mc Donald" reads family "Mc Donald" and "John van Mc" reads family "van Mc" because mc is a never-given PARTICLE (#360), which is why the trailing bare shape is the only one at issue. +- **#454 is BY DESIGN, and the words are mc and vd.** mc has no vowel and is not a borne name — which reverses #342's comment, where mc is listed among the entries "borne as surnames" and is the one member of that list its model-recall hedge does not cover; #360 measured it: mc and ste are contractions of Mac and Sainte, borne in neither position, which is why both left the ambiguous particle half. The Scottish prefix never detaches from the name it belongs to, so "Donald Mc" is not a name anyone writes and its suffix reading costs nothing real. vd needs a surname after it, so a bare trailing vd is the decoration. Neither joins the ambiguous subset and the `fix(suffix-routing)` ledger rule — a two-token name ending in a credential acronym keeps it in `suffix` — stays. What #454's example actually turns on is a different membership: "Mc Donald" reads family "Mc Donald" and "John van Mc" reads family "van Mc" because mc is a never-given PARTICLE (#360), which is why the trailing bare shape is the only one at issue. - **What #454 got right, and it is a rules.md defect that is now fixed.** rules.md#P6's Accepted clause said that under the default order a comma-less name keeps its positional reading, with "Jong Anke de" → family "de" as its only example. Measured, that holds for de and for do and fails for exactly the two words that are in BOTH the particle vocabulary and the UNAMBIGUOUS suffix vocabulary: "Donald mc" and "Smith vd" lose the family entirely to `suffix`. The clause is repaired to say so, and the two places that already conceded it in prose — the `'Donald mc'` bullet of #differential-ledger's fields-only arc and the corresponding comment in expected_since_1.4.0.toml — are amended from "recorded here rather than fixed" to point at the repair. No example line moves: the clause's own example is de-shaped and stays true, which is why no test caught the defect and why regenerating corpus_rules.jsonl is a no-op here. Recompute the membership with `L = Parser().lexicon; sorted(L.particles & (L.suffix_acronyms - L.suffix_acronyms_ambiguous))`, which gives ['mc', 'vd'] on 2026-09-07, and `sorted(L.particles & L.suffix_acronyms_ambiguous)`, which gives ['do']. - **Parking lot: the shape-plus-position credential heuristic.** #490's idea without the "collides with vocabulary" part, and what would recover "John Smith RAI" without any wordlist at all. Two shapes: an ALL-CAPS acronym standing in the suffix position of a MIXED-CASE name ("John Smith XYZ", "John Smith, XYZ"), where the case contrast is the signal; and a DOTTED acronym in that position regardless of case ("John Smith X.Y.Z.", "john smith x.y.z."), where the periods are. Measured 2026-09-07, none of the four is read as a suffix today — the bare form gives family XYZ, the comma form given XYZ with family "John Smith", and the second two family X.Y.Z. — while the roman-numeral accident above reads three dotted forms as suffixes on a fork that is about generations, not credentials. That contrast is the evidence the dotted shape is unhandled: the parser already produces the wanted answer for X.Y.I. and R.A.V. by coincidence and the unwanted one for R.A.X. A follow-up issue carries it; nothing in this entry depends on it. - **Not decided here:** sa stays where #296's audit put it (title and suffix dual, position decides); se and om stay unambiguous, there being no surname evidence worth standing behind and OM being the Order of Merit; the S2 comma-form boundary ("Smith, Ed" → given Ed) is not reopened. #342's own comment lists sa, se and om beside ba and cha as model recall rather than corpus-attested, and per the #360 lesson they need a human before they move. diff --git a/docs/design/rules.md b/docs/design/rules.md index 9de7c9cb..4fc7fd37 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -497,8 +497,9 @@ P6. Rationale: a particle ending the name has nothing to link Accepted: the precedence over S2 is stated for the shape, so it sweeps in every word that is both particle and suffix vocabulary — today vd, do and mc. Only vd's reading was - weighed; mc inherits it, which is the shape's cost and is - tracked with the other contested memberships. `do` sits in the + weighed; mc inherits it, which is the shape's cost and was + decided rather than tracked — #454 closed by design 2026-09-07 + (decisions.md#suffix-acronym-collisions). `do` sits in the AMBIGUOUS acronym half and was already read as a name word there, so the precedence decides nothing for it — and because the report names the reading OVERRIDDEN, `do` reports diff --git a/docs/release_log.rst b/docs/release_log.rst index bb3649a5..4c61008d 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -16,7 +16,7 @@ Release Log - **Remove rai and cha from the default post-nominal acronyms, so a trailing Rai or CHA keeps the family name.** ``HumanName("Aishwarya Rai")`` gives last ``Rai``, where 2.0.0 through 2.2.0 gave suffix ``Rai`` and no last name at all -- 1.4.0's reading, restored -- and ``Lala Lajpat Rai`` gives middle ``Lajpat``, last ``Rai``. Both entries came in with a 2019 bulk import of Wikipedia post-nominals and were never reviewed against the surnames they collide with. The cost is that a genuine credential written after a full name is no longer recognized: ``John Smith RAI`` gives middle ``Smith``, last ``RAI``, and the comma forms swap ends -- ``John Smith, RAI`` gives first ``RAI``, last ``John Smith``, and ``Ahmad Jayadi, CHA`` first ``CHA``, last ``Ahmad Jayadi``. A caller who needs either back adds it: ``Lexicon.default().add(suffix_acronyms={"cha"})``. Five names move in the differential corpora, all of them radar-tier. See the ``suffix-acronym-collisions`` entry of ``docs/design/decisions.md`` (closes #342) - - **Mark ba as an acronym that is also an ordinary name, so a bare trailing Ba keeps the family name.** ``HumanName("Anna Ba")`` gives last ``Ba`` and reports a suffix-or-name ambiguity, where 2.0.0 through 2.2.0 gave suffix ``Ba`` and no last name. A full name in front leaves the credential reading intact: ``John Smith BA`` still gives suffix ``BA``, now flagged, and the dotted ``John Smith B.A.`` is an unflagged suffix, the periods settling it. The comma form costs what ``Smith, Ed`` costs -- ``Smith, BA`` gives first ``BA`` -- which S2 already accepted for the other ambiguous acronyms. BA is a common credential and Ba a real surname in Vietnamese and Senegalese Fula, which is the ``ma``/``Ma`` shape exactly. No corpus name moves (#342) + - **Mark ba as an acronym that is also an ordinary name, so a bare trailing Ba keeps the family name.** ``HumanName("Anna Ba")`` gives last ``Ba`` and reports a suffix-or-name ambiguity, where 2.0.0 through 2.2.0 gave suffix ``Ba`` and no last name. The SPACED full-name form keeps the credential reading: ``John Smith BA`` still gives suffix ``BA``, now flagged, and the dotted ``John Smith B.A.`` is an unflagged suffix, the periods settling it. The COMMA forms move, and this is the marking's real cost: ``Smith, BA`` gives first ``BA``, and ``John Smith, BA`` gives first ``BA``, last ``John Smith``, where 2.0.0 through 2.2.0 gave suffix ``BA`` for both -- what ``Smith, Ed`` costs, which S2 already accepted for the other ambiguous acronyms. A bracketed or quoted ``John Smith (BA)`` falls through to nickname parsing, as the 2.0 note for ``ma``/``do`` below recorded for that pair. Write ``B.A.`` to keep the credential reading. BA is a common credential and Ba a real surname in Vietnamese and Senegalese Fula, which is the ``ma``/``Ma`` shape exactly. No corpus name moves (#342) **Additions** diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 5ccc886e..6d1ebec8 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -214,27 +214,37 @@ # 'Jack Ma' keeps its family name. # # The other half of the criterion, added 2026-09-07 with #342. - # Marking is for a SHORT credential in wide use that is also a - # name -- the five entries here are all two letters, which is not - # a coincidence: a two-letter acronym has too few letters to be - # anything but common, so both readings are live and a flag is - # the honest answer. A LONGER credential that is tenuous or - # specialized and collides with a common name is REMOVED from - # SUFFIX_ACRONYMS instead of marked, because its BARE spelling - # has no second reading worth carrying: 'rai' and 'cha' left the - # set that day (RAI is "RETA Authorized Instructor", CHA is - # Certified Hotel Administrator or Certified Healthcare Auditor; - # Rai is a common surname across Hindi- and Bengali-speaking - # regions and Cha the Korean 차). Removal need not take the - # DOTTED spelling with it: "John Smith R.A.I." still reads suffix - # 'R.A.I.', because rules.md#S3 splits an interior-period token - # on its periods and the chunk 'i' is a Roman numeral in - # SUFFIX_WORDS -- "John Smith C.H.A." has no such chunk and reads - # family 'C.H.A.'. A caller who needs one back adds it -- - # Lexicon.default().add(suffix_acronyms={"cha"}) -- which is the - # answer this library gives for every locale-specific - # vocabulary. The cost is stated and accepted: with the entry - # gone, "John Smith RAI" reads family 'RAI'. See + # Being borne at all is only the entry ticket; what decides among + # the three answers is a comparison of FREQUENCIES -- how common + # the word is as a borne name in the TRAILING position against how + # common it is as a credential. Roughly balanced earns the marking + # here, and the parse reports the fork: 'ba' is the entry that + # earned it that day, BA being a common credential and Ba a real + # surname (Vietnamese; Senegalese Fula) about as common as the + # credential, which is the ma/Ma shape exactly. Where the NAME + # reading dominates, the entry is REMOVED from SUFFIX_ACRONYMS + # instead of marked: 'rai' and 'cha' left the set that day, both + # far more common as surnames than their credentials are as + # credentials (RAI is "RETA Authorized Instructor", CHA is + # Certified Hotel Administrator or Certified Healthcare Auditor, + # both tenuous or specialized; Rai is a common surname across + # Hindi- and Bengali-speaking regions and Cha the Korean 차). + # Where the CREDENTIAL dominates, the entry stays unambiguous. + # LENGTH is a correlate and not the test -- a short acronym is + # more often a common credential AND more often a name -- so do + # not read the letter counts here as a rule. + # + # Removal takes the DOTTED spelling with it too, except by + # accident: "John Smith R.A.I." still reads suffix 'R.A.I.' only + # because rules.md#S3 splits an interior-period token on its + # periods and the chunk 'i' happens to be a Roman numeral in + # SUFFIX_WORDS. "John Smith R.A.X." reads family, and so does + # "John Smith C.H.A." after the removal. Do not count on a dotted + # spelling surviving a removal. A caller who needs an entry back + # adds it -- Lexicon.default().add(suffix_acronyms={"cha"}) -- + # which is the answer this library gives for every + # locale-specific vocabulary. The cost is stated and accepted: + # with the entry gone, "John Smith RAI" reads family 'RAI'. See # decisions.md#suffix-acronym-collisions. # # NOT 'ms' or 'sa', though #296's audit table put them here for the @@ -245,6 +255,14 @@ # and read as a credential anyway. Both words are genuine duals -- # title and unambiguous suffix -- and position decides, as for # 'sr' and 'lt' (decisions.md#C1). + # + # NOT 'se' or 'om' either, weighed 2026-09-07 with #342 and left + # alone: no surname evidence worth standing behind, and OM is the + # Order of Merit. And NOT 'mc' or 'vd', which are also PARTICLES: + # #454 closed by design -- neither is a borne name, so a bare + # trailing one is the decoration, and rules.md#P6's Accepted + # clause names them as the two words whose positional reading + # does not hold. 'ba', 'do', 'ed', diff --git a/tests/v2/cases.py b/tests/v2/cases.py index f0a1e050..3ecf38a2 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -403,23 +403,28 @@ def _check_cjk_shape_purity(self) -> None: "peeling 'MA' still leaves given+family, so the " "credential reading wins (v1 parity)"), # -- #342: rai and cha left SUFFIX_ACRONYMS. The criterion is in - # decisions.md#suffix-acronym-collisions: a TWO-LETTER credential - # in wide use that is also a name earns the ambiguous marking - # (ba, do, ed, jd, ma), while a three-letter credential that is - # tenuous or specialized and collides with a common surname is - # REMOVED, a caller adding it back with - # Lexicon.default().add(suffix_acronyms={"cha"}). These rows pin + # decisions.md#suffix-acronym-collisions and it compares + # FREQUENCIES: how common the word is as a borne name in the + # trailing position against how common it is as a credential. + # Rough balance earns the ambiguous marking (ba, do, ed, jd, ma); + # the name reading dominating REMOVES the entry, a caller adding + # it back with Lexicon.default().add(suffix_acronyms={"cha"}); + # the credential dominating leaves it unambiguous. Length is a + # correlate, not the test. These rows pin # the FORK, not the entries (mechanisms.md#VOCABULARY-EXERCISES-FORKS). Case("bare_surname_is_not_a_credential", "Aishwarya Rai", {"given": "Aishwarya", "family": "Rai"}, - classification="fix(#342)", + classification="parity", notes="#342's own subject: 'rai' arrived in the 2019 bulk " "wikipedia post-nominal import and was never reviewed " "against the surname, so 2.0.0 through 2.2.0 read " "suffix 'Rai' with NO family name at all. 1.4.0 read " "family 'Rai' and this restores that reading, which is " - "why the 1.4.0 ledger loses a rule here rather than " - "gaining one. Named in the release note"), + "why it classifies parity rather than fix, and why " + "the 1.4.0 ledger replaces a NOT WANTED rule where the " + "three 2.x ledgers gain a new one, and its heading " + "explains four names where theirs explain five. Named " + "in the release note"), Case("removed_credential_loses_its_suffix_reading", "John Smith RAI", {"given": "John", "middle": "Smith", "family": "RAI"}, classification="fix(#342)", @@ -441,6 +446,18 @@ def _check_cjk_shape_purity(self) -> None: "comma (C1): the pre-comma run is the family and the " "post-comma word is the given name. 'John Smith, RAI' " "is the same shape and moves with it"), + Case("removed_credential_loses_the_dotted_spelling_too", + "John Smith C.H.A.", + {"given": "John", "middle": "Smith", "family": "C.H.A."}, + classification="fix(#342)", + notes="the removal reaches the DOTTED spelling through S3's " + "period fold, which strips the periods to reach the " + "entry -- so with 'cha' gone this reads family. The " + "contrast is 'John Smith R.A.I.', which still reads " + "suffix and is an accident of 'i' being a SUFFIX_WORDS " + "numeral rather than a survival of the vocabulary, " + "deliberately unpinned " + "(decisions.md#suffix-acronym-collisions)"), # -- #342: 'ba' is the other half of the same decision. BA is a # common credential and Ba a real surname (Vietnamese; Senegalese # Fula), which is the ma/Ma shape exactly, so it takes the @@ -457,6 +474,18 @@ def _check_cjk_shape_purity(self) -> None: "through 2.2.0 read suffix 'Ba' with no family name; " "1.4.0 read family 'Ba' unflagged, so this row " "restores 1.4.0's roles and adds the flag"), + Case("comma_ambiguous_acronym_ba", "Smith, BA", + {"given": "BA", "family": "Smith"}, + classification="parity", + notes="the marking's comma cost, and it is 'Smith, Ed' " + "arriving for a second word: with 'ba' ambiguous, S2 " + "declines the post-comma credential and C1 reads it as " + "the given name. 2.0.0 through 2.2.0 read family " + "'Smith', suffix 'BA'; 1.4.0 read given 'BA', which is " + "what this restores and why it classifies parity. The " + "row is here because the release note names the text -- " + "comma_ambiguous_acronym pins the fork and stays green " + "when 'ba' leaves the ambiguous subset"), Case("ba_is_a_suffix_when_a_family_name_remains", "John Smith BA", {"given": "John", "family": "Smith", "suffix": "BA"}, ambiguities=("suffix-or-name",), @@ -467,7 +496,12 @@ def _check_cjk_shape_purity(self) -> None: "above"), Case("by_design_trailing_mc_reads_as_a_credential", "Donald Mc", {"given": "Donald", "suffix": "Mc"}, - notes="#454, closed by design 2026-09-07. This is the one " + classification="fix(suffix-routing)", + notes="#454, dispositioned by design with this bundle. " + "1.4.0 read last 'Mc'; the 1.4.0 ledger's " + "fix(suffix-routing) rule -- a two-token name ending " + "in a credential acronym keeps it in `suffix` -- is " + "what explains the divergence. This is the one " "row whose point is that it must NOT change. 'mc' is " "SUFFIX_ACRONYMS and PARTICLES both, and it is not a " "borne name: it has no vowel, and the Scottish prefix " diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index e000fa3a..48de98fb 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -819,12 +819,14 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # _CORPUS_CLAIMS unmoved. These probes are the wall, and every one # of them is a name MEASURED not to move under the removal (which # is the roster's actual condition -- a name the rule has no - # business claiming, not merely one outside the corpus). Two test - # the right anchor, where a third word or a re-ordered comma + # business claiming, not merely one outside the corpus). Three + # test the right anchor, where a third word or a re-ordered comma # leaves 'Rai' with a family name beside it and nothing to take; - # 'Ahmad Jayadi' and 'John Smith' test the ALTERNATIVES' trailing - # words, each being a mover with its last word dropped, so an - # alternative TRUNCATED to its leading tokens is caught. No probe + # 'Ahmad Jayadi', 'Ahmad Jayadi,', 'John Smith' and 'John Smith,' + # test the ALTERNATIVES' trailing words, each being a mover with + # its last word dropped -- with the comma and without it, so an + # alternative TRUNCATED to its leading tokens is caught wherever + # the truncation falls. No probe # tests the left anchor and none can: dropping the '^' widens the # rule to names ENDING in an alternative, and every such name is # itself a mover -- 'Mr Aishwarya Rai' moves -- so no name that @@ -836,7 +838,8 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # (corpus_issues.jsonl), which parses identically with and without # 'rai'/'cha' in SUFFIX_ACRONYMS -- so the widening is caught. "fix(#342)": ("Aishwarya Rai Bachchan", "Rai, Aishwarya", - "Jane Rai Smith", "Ahmad Jayadi", "John Smith", + "Jane Rai Smith", "Ahmad Jayadi", "Ahmad Jayadi,", + "John Smith", "John Smith,", "Sejal Chaturvedi, CSM"), # #451's four replacements for the fields-only catch-all. Each is # anchored to a two-token name, so the probes are a third token and @@ -2056,7 +2059,9 @@ def _claim(rule: dict) -> _Claim: # #342's alternation. Five corpus names and four roles: the # comma forms move `given` where the bare forms move `middle`, # so a widening taking one shape alone would change the roles - # here before it reached the gate. + # here before it reached the gate. Reach, not explanation -- + # at this baseline 'Aishwarya Rai' reaches the rule and is + # explained by parity instead, so the gate heading reads 4. "fix(#342) rai and cha left the credential acronyms, so a trailing Rai or CHA is a name word": _Claim(5, ('family', 'given', 'middle', 'suffix'), "c3d76812da97", None), "fix(A2) content-free input names nobody, so every role empties": diff --git a/tools/differential/compare.py b/tools/differential/compare.py index d68618a0..72e2f467 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -1941,9 +1941,14 @@ class _ShapeMismatch(NamedTuple): #: 'Carod i' diffs under the default order at 1.4.0 only, where its #: contest row stands, and 'MD, PHD' carries a contest row at every one #: of the three baselines it diffs at since #501 pinned its 2.x pair. -#: That is why the population is 50 names where the tests/-only scan -#: says 52. -#: The counts: 36 / 31 / 30 / 5 rows, 102 in all, over those 50 names +#: That is why the population is 51 names where the tests/-only scan +#: says 53. +#: Recounted 2026-09-07 with #342, which moved three names across the +#: literal clause at once: 'Aishwarya Rai' gained a case row and left +#: the population, while 'Lala Lajpat Rai' and 'John Smith, RAI' are +#: named nowhere under tests/ outside test_ledger_guards.py and +#: entered it, so the scan went 52 -> 53 and the roster 50 -> 51. +#: The counts: 37 / 33 / 32 / 7 rows, 109 in all, over those 51 names #: -- and the roster is now exactly the population, the five contest #: rows beyond it having gone to _RECORDED_DIFFS with #501 and five #: more with #498, which left the population by gaining a @@ -1960,9 +1965,12 @@ class _ShapeMismatch(NamedTuple): #: four clauses above with the literal set from ast.walk over #: tests/**/*.py EXCLUDING test_ledger_guards.py, as the POPULATION #: clause says -- run over every file it yields 33 / 23 / 22 / 4 rows -#: rather than 36 / 31 / 30 / 5, since _CROSS_RULE_WINNERS' keys and +#: rather than 37 / 33 / 32 / 7, since _CROSS_RULE_WINNERS' keys and #: a few guard literals then score as watchers, and #498's fourteen -#: keys are exactly that kind of literal: the two halves of this +#: keys are exactly that kind of literal -- as are #342's two +#: 2026-09-07 arrivals, both named in _NOT_A_VOCABULARY_COPY, which is +#: why the every-file figures stood still while eight rows landed and +#: one left: the two halves of this #: sentence moved for different reasons on 2026-09-05, the second #: because five rows left this dict and the first because those five #: are watched at 2.x too, where they now score as watched by the @@ -2002,12 +2010,14 @@ class _ShapeMismatch(NamedTuple): "Jack M.A.": ("family", "suffix"), "Jane van der Berg 旧姓 Jones": ("family", "maiden"), "Janey née Jones": ("family", "given", "maiden", "middle"), + "John Smith, RAI": ("family", "given", "suffix"), "John V": ("family", "suffix"), "John of the Doe": ("_initials",), "Jong van der": ("_initials",), "Jong, van der": ("_initials",), "Jose e Maria Santos": ("_initials",), "Juan Garcia y Lopez": ("_initials",), + "Lala Lajpat Rai": ("family", "middle", "suffix"), "Mesnil Garcia van": ("_initials",), "Mohamad X": ("family", "suffix"), "Ph. D., Jr.": ("family", "given"), @@ -2043,10 +2053,12 @@ class _ShapeMismatch(NamedTuple): "Jane van der Berg 旧姓 Jones": ("family", "maiden"), "Janey née Jones": ("family", "given"), "Joe E. Smith": ("_initials",), + "John Smith, RAI": ("family", "given", "suffix"), "John, Smith, Dr.": ("_ambiguities",), "Jong van der": ("_initials",), "Jong, van der": ("_initials",), "Jose E. Maria Santos": ("_initials",), + "Lala Lajpat Rai": ("family", "middle", "suffix"), "MD, DO, DDS": ("given", "title"), "Mesnil Garcia van": ("_initials",), "Ph. D., Jr.": ("family", "suffix", "title"), @@ -2082,10 +2094,12 @@ class _ShapeMismatch(NamedTuple): "Jane van der Berg 旧姓 Jones": ("family", "maiden"), "Janey née Jones": ("family", "given"), "Joe E. Smith": ("_initials",), + "John Smith, RAI": ("family", "given", "suffix"), "John, Smith, Dr.": ("_ambiguities",), "Jong van der": ("_initials",), "Jong, van der": ("_initials",), "Jose E. Maria Santos": ("_initials",), + "Lala Lajpat Rai": ("family", "middle", "suffix"), "MD, DO, DDS": ("given", "title"), "Mesnil Garcia van": ("_initials",), "Ph. D., Jr.": ("family", "suffix", "title"), @@ -2105,7 +2119,9 @@ class _ShapeMismatch(NamedTuple): "E Anne D,Leonardo": ("_initials",), "JOSE E MARIA SANTOS": ("_initials",), "Joe E. Smith": ("_initials",), + "John Smith, RAI": ("family", "given", "suffix"), "Jose E. Maria Santos": ("_initials",), + "Lala Lajpat Rai": ("family", "middle", "suffix"), "Smith, John E, III, Jr": ("_initials",), }, }