[MC-462-C] fix(compare): make LexStat authoritative on un-clusterable forms (#653)

#651 hardened the cognate job by *predicting* which forms LingPy would
reject (a sound-class pre-check). The prediction had false negatives: a
"/" variant separator (e.g. "qap/gaza girtin") survives tokenising as an
unknown segment that LexStat rejects, so the job still crashed with
"Could not convert item ID: <row>" on larger speaker selections (observed
at row 3235 with a 14-speaker run).

Replace the guess with two robust layers:

1. _lingpy_safe_form now also folds "/" into the "_" word boundary, so
   variant forms tokenise and still cluster (no longer dropped).
2. LexStat itself is the authority: build the wordlist, and if it raises
   "Could not convert item ID: <row>", drop that row, rebuild with fresh
   CONTIGUOUS ids, and retry. This cannot crash on any unforeseen
   character (deleting a key in place leaves a gap that corrupts LingPy's
   row numbering, hence the full rebuild). The sound-class pre-check
   helper is removed.

Skipped forms are still recorded/surfaced via the #652 collector. No
change to persisted enrichments or displayed forms. Verified against a
full real workspace (14 speakers / 3464 forms) that previously crashed at
row 3235: now completes, the "/"-variant form clusters, and only the lone
"?" placeholder is skipped.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: TrueNorth49

python/compare/cognate_compute.py

@@ -10,6 +10,7 @@
 import argparse
 import csv
 import json
+import re
 import sys
 import unicodedata
 from dataclasses import dataclass, field
@@ -781,46 +782,25 @@ def _group_label(index: int) -> str:
 def _lingpy_safe_form(ipa_form: str) -> str:
     """Make a form safe for LingPy's tokenizer without altering stored data.
 
-    LingPy's ``ipa2tokens`` raises ``ValueError: Input must not contain
-    spaces`` on multi-word forms (e.g. Kurdish light-verb constructions
-    like ``"qap girtin"``), which LexStat surfaces as the opaque
-    ``Could not convert item ID: <row>`` error and aborts the whole
-    cognate job. Collapse internal whitespace to ``_`` -- LingPy's own
-    word-boundary symbol -- so the form tokenizes cleanly.
+    Two real-world form shapes abort the whole cognate job with the opaque
+    ``Could not convert item ID: <row>`` error:
+
+    - **multi-word forms** (e.g. light-verb constructions like
+      ``"qap girtin"``) -- ``ipa2tokens`` rejects spaces outright;
+    - **``/`` variant separators** (e.g. ``"qap/gaza girtin"``) -- the
+      ``/`` survives tokenisation as an unknown segment that LexStat then
+      refuses to convert.
+
+    Collapse internal whitespace *and* ``/`` to ``_`` -- LingPy's own
+    word-boundary symbol -- so both shapes tokenize cleanly and still
+    cluster.
 
     This value is fed ONLY to the in-memory LexStat wordlist. It is never
     persisted to parse-enrichments.json nor shown in the UI: the cognate
     output is speaker groupings only, and similarity scores use the
     original ``record.ipa``. The displayed form is unchanged.
     """
-    return "_".join(str(ipa_form or "").split())
-
-
-def _lingpy_clusterable(safe_form: str, ipa2tokens, tokens2class) -> bool:
-    """Return True if LingPy can tokenise ``safe_form`` into known sounds.
-
-    A few stored form values are un-clusterable even after whitespace
-    sanitising -- e.g. a bare ``"?"`` placeholder that tokenises to
-    only-unknown sound classes. LexStat aborts the entire job on such a
-    form (``Could not convert item ID: <row>``). Detecting them up front
-    lets us build a gap-free wordlist and exclude only the offending rows
-    -- deleting keys after construction instead corrupts LingPy's row
-    numbering and triggers a misleading "contains N fields" error.
-
-    Excluded rows keep their stored/displayed form untouched; they simply
-    receive no cognate grouping (a ``"?"`` placeholder is not cognate with
-    anything). Any tokeniser exception is treated as un-clusterable.
-    """
-    if not safe_form:
-        return False
-    try:
-        tokens = ipa2tokens(safe_form)
-        if not tokens:
-            return False
-        classes = tokens2class(tokens, "sca")
-    except Exception:
-        return False
-    return any(cls != "0" for cls in classes)
+    return "_".join(str(ipa_form or "").replace("/", " ").split())
 
 
 def _compute_cognate_sets_with_lingpy(
@@ -839,7 +819,6 @@ def _compute_cognate_sets_with_lingpy(
     """
     try:
         from lingpy import LexStat  # type: ignore
-        from lingpy.sequence.sound_classes import ipa2tokens, tokens2class  # type: ignore
     except ImportError as exc:
         raise RuntimeError(
             "LingPy is not installed. Install it with: pip install lingpy"
@@ -855,29 +834,48 @@ def _compute_cognate_sets_with_lingpy(
     if not rows:
         return {}
 
-    # Build the LexStat wordlist with CONTIGUOUS integer row IDs. Forms
-    # are whitespace-sanitised (so multi-word forms still cluster) and any
-    # form LingPy cannot tokenise into known sounds is skipped up front --
-    # excluding them after construction would leave gaps in the row
-    # numbering and crash LingPy. Skipped rows keep their displayed form
-    # and simply receive no cognate grouping.
-    lex_data: Dict[int, List[str]] = {0: ["doculect", "concept", "ipa"]}
-    index_meta: Dict[int, Tuple[str, str]] = {}
-
-    row_index = 0
+    # Sanitise forms (whitespace/"/" -> "_", so multi-word and variant
+    # forms still cluster) and drop empties. Each surviving candidate
+    # carries its original form so we can report it if LexStat rejects it.
+    candidates: List[Tuple[str, str, str, str, str]] = []
     skipped: List[Dict[str, str]] = []
     for speaker, concept_label, ipa_form, concept_id in rows:
         safe_form = _lingpy_safe_form(ipa_form)
-        if not _lingpy_clusterable(safe_form, ipa2tokens, tokens2class):
-            skipped.append({
-                "concept_id": concept_id,
-                "speaker": speaker,
-                "form": str(ipa_form or ""),
-            })
+        original = str(ipa_form or "")
+        if not safe_form:
+            skipped.append({"concept_id": concept_id, "speaker": speaker, "form": original})
             continue
-        row_index += 1
-        lex_data[row_index] = [speaker, concept_label, safe_form]
-        index_meta[row_index] = (concept_id, speaker)
+        candidates.append((speaker, concept_label, safe_form, concept_id, original))
+
+    # Let LexStat itself be the authority on what it can ingest. A few
+    # values survive sanitising yet LexStat still refuses to convert them
+    # (e.g. a bare "?" that tokenises to only-unknown sounds), aborting
+    # with "Could not convert item ID: <row>". When that happens, drop the
+    # offending row and rebuild the wordlist with fresh CONTIGUOUS ids,
+    # then retry -- deleting a key in place instead leaves a gap that
+    # corrupts LingPy's row numbering. Bounded by the candidate count.
+    lexstat = None
+    index_meta: Dict[int, Tuple[str, str]] = {}
+    for _attempt in range(len(candidates) + 1):
+        lex_data: Dict[int, List[str]] = {0: ["doculect", "concept", "ipa"]}
+        index_meta = {}
+        for idx, (speaker, concept_label, safe_form, concept_id, _orig) in enumerate(candidates, start=1):
+            lex_data[idx] = [speaker, concept_label, safe_form]
+            index_meta[idx] = (concept_id, speaker)
+        if not index_meta:
+            break
+        try:
+            lexstat = LexStat(lex_data, check=False)
+            break
+        except ValueError as exc:
+            match = re.search(r"convert item ID:\s*(\d+)", str(exc))
+            if match is None:
+                raise
+            bad = int(match.group(1))
+            if not 1 <= bad <= len(candidates):
+                raise
+            speaker, concept_label, safe_form, concept_id, original = candidates.pop(bad - 1)
+            skipped.append({"concept_id": concept_id, "speaker": speaker, "form": original})
 
     if skipped:
         _warn(
@@ -893,10 +891,9 @@ def _compute_cognate_sets_with_lingpy(
     if skipped_out is not None:
         skipped_out.extend(skipped)
 
-    if not index_meta:  # nothing clusterable
+    if lexstat is None or not index_meta:  # nothing clusterable
         return {}
 
-    lexstat = LexStat(lex_data, check=False)
     lexstat.get_scorer()
     lexstat.cluster(method="lexstat", threshold=float(threshold), ref="cogid")
 

python/compare/test_cognate_lingpy_safe.py

@@ -37,10 +37,16 @@ def test_safe_form_collapses_internal_space():
 
 
 def test_safe_form_handles_multiple_and_repeated_spaces():
-    assert _lingpy_safe_form("qap/gaza girtin") == "qap/gaza_girtin"
     assert _lingpy_safe_form("a   b\tc") == "a_b_c"
 
 
+def test_safe_form_collapses_slash_variant_separator():
+    # A "/" variant separator survives ipa2tokens as an unknown segment
+    # that LexStat rejects; fold it into the "_" word boundary too.
+    assert _lingpy_safe_form("qap/gaza girtin") == "qap_gaza_girtin"
+    assert _lingpy_safe_form("a/b") == "a_b"
+
+
 def test_safe_form_leaves_single_word_unchanged():
     assert _lingpy_safe_form("xwārdin") == "xwārdin"
 
@@ -161,3 +167,24 @@ def test_skipped_out_stays_empty_when_all_clusterable():
     _compute_cognate_sets_with_lingpy(forms_by_concept, concepts, 0.6, skipped_out=skipped)
 
     assert skipped == []
+
+
+def test_slash_variant_form_clusters_and_is_not_skipped():
+    pytest.importorskip("lingpy")
+
+    # "/"-variant forms previously survived sanitising but were rejected by
+    # LexStat ("Could not convert item ID"). They must now cluster, not skip.
+    forms_by_concept = {
+        "627": [
+            FormRecord("Badr01", "627", "", "qap/gaza girtin", "", 0.0, 1.0),
+            FormRecord("Fail03", "627", "", "qap girtin", "", 0.0, 1.0),
+        ],
+    }
+    concepts = [ConceptSpec(concept_id="627")]
+
+    skipped: list = []
+    result = _compute_cognate_sets_with_lingpy(forms_by_concept, concepts, 0.6, skipped_out=skipped)
+
+    assert skipped == []
+    speakers = {s for group in result.get("627", {}).values() for s in group}
+    assert speakers == {"Badr01", "Fail03"}