diff --git a/CHANGELOG.md b/CHANGELOG.md index ec04d9b..fab154b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,9 +24,15 @@ releases may contain breaking changes. - Project scaffolding: package skeleton, vendored LIFT 0.13 RELAX NG schema, test corpus with provenance, corpus-prep and large-file-generator tooling. -- Full object model: all 35 LIFT 0.13 elements as typed dataclasses; - `sil_lift.load()` / `Lexicon.load()` full-document reader that keeps LIFT - residue per node in `Extras`; LIFT-version guard. +- Full object model: all 35 LIFT 0.13 elements as typed dataclasses. + Multilingual fields are `Multitext`, a `Mapping` from language code to + `Text` that coerces plain strings on assignment; `len()` and the views + count languages, deletion takes every form for a language, and the `forms` + list stays the full truth for what no mapping can represent — a form with + no lang, and a second form for a language already present (which validation + reports as `duplicate-form-lang`). `sil_lift.load()` / `Lexicon.load()` + full-document reader that keeps LIFT residue per node in `Extras`; + LIFT-version guard. - `Lexicon.save()` writer with byte-fidelity passthrough — unchanged documents and untouched entries are written byte-identically; touched entries re-serialize canonically with all out-of-schema content preserved. Fidelity diff --git a/docs/en/guides/read-edit-write.md b/docs/en/guides/read-edit-write.md index c7fe813..ff7b7bb 100644 --- a/docs/en/guides/read-edit-write.md +++ b/docs/en/guides/read-edit-write.md @@ -12,7 +12,7 @@ lex = sil_lift.load("dictionary.lift") ## The model -Every LIFT element is a typed dataclass: `Entry`, `Sense`, `Example`, `Pronunciation`, `Variant`, `Relation`, `Etymology`, `Reversal`, and so on. Multilingual text is a `Multitext`, which behaves like a mapping from language code to `Text`: +Every LIFT element is a typed dataclass: `Entry`, `Sense`, `Example`, `Pronunciation`, `Variant`, `Relation`, `Etymology`, `Reversal`, and so on. Multilingual text is a `Multitext`, which is a `Mapping` from language code to `Text`: ```python entry = lex.find(id="abat") @@ -20,8 +20,11 @@ entry = lex.find(id="abat") str(entry.lexical_unit["seh"]) # "abat" entry.lexical_unit["en"] = "grove" # plain strings are coerced "en" in entry.citation # False +list(entry.lexical_unit.keys()) # ["seh", "en"] ``` +`keys()`, `values()` and `items()` are views, one key per language. A schema-valid document has nothing more, but real files sometimes carry a second form for a language already present, or a form with no `lang` at all — neither is reachable by key. `forms` holds every form in file order, and [`validate`](validate.md) reports the duplicate as `duplicate-form-lang`. Deletion works on the language rather than the form, so `del entry.lexical_unit["en"]` removes every English form. + `Text` is structured — an ordered list of `str` and `Span` fragments — because `` can contain nested `` markup. `str(text)` flattens to plain text; the fragments keep the markup for round-tripping. Glosses are _form-shaped_ in LIFT (each `` carries its own language), so a sense has `glosses: list[Form]` plus a helper: diff --git a/src/sil_lift/_text.py b/src/sil_lift/_text.py index beb8849..d14b325 100644 --- a/src/sil_lift/_text.py +++ b/src/sil_lift/_text.py @@ -10,6 +10,7 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass, field from datetime import date, datetime from typing import TYPE_CHECKING @@ -84,21 +85,46 @@ class Form: @dataclass(slots=True, repr=False) -class Multitext: +class Multitext(Mapping[str, Text]): """An insertion-ordered collection of forms, one per language. - Behaves like a ``Mapping[str, Text]`` keyed by language (``mt["en"]``), - with assignment coercing plain strings (``mt["en"] = "dog"``). The - underlying ``forms`` list is the full truth — forms with a ``None`` lang - (schema-invalid input) are reachable there but not via mapping keys. + A ``Mapping[str, Text]`` keyed by language — ``mt["en"]``, ``"en" in mt``, + ``mt.get(...)``, ``mt.keys()`` and the other views — plus the two mutators + LIFT editing needs: assignment coercing plain strings (``mt["en"] = "dog"``) + and deletion. The rest of ``MutableMapping`` is deliberately not inherited; + ``clear`` and ``popitem`` have no clear meaning for a form list that can + also hold forms no key reaches. + + The ``forms`` list is the full truth, and holds what no mapping can + represent: a form with a ``None`` lang, and a second form for a language + already present. Both are schema-invalid — the LIFT 0.13 spec's own example + documents carry a lang-less form, and a repeated language is what validation + reports as ``duplicate-form-lang``, read off ``forms`` rather than off the + mapping. Neither is reachable by key, yielded by a view, or counted by + ``len()``. + + Where a language is repeated, the mapping is its first form: that is the one + ``mt["en"]`` reads and the one assignment updates, leaving any later form for + the language alone, since a ``Form`` carries annotations and residue the + mapping cannot show a caller. Deletion takes every form for the language, so + ``del mt["en"]`` leaves ``"en" not in mt``. + + Two further deviations from ``Mapping``, both serving the fidelity contract. + ``bool(mt)`` asks "is there anything to serialize" rather than + ``len(mt) != 0``, so a multitext holding only residue or only a lang-less + form is truthy while empty. Equality is the dataclass's: form lists must + match exactly, which is stricter than ``Mapping`` equality, where a form no + key reaches would not count. """ forms: list[Form] = field(default_factory=list) extra: Extras = field(default_factory=Extras) def _find(self, lang: str) -> Form | None: + # A None lang is not a key. Matching one would answer __getitem__ and + # __contains__ for a form that no view yields and len() does not count. for form in self.forms: - if form.lang == lang: + if form.lang is not None and form.lang == lang: return form return None @@ -117,36 +143,43 @@ def __setitem__(self, lang: str, value: Text | str) -> None: form.text = text def __delitem__(self, lang: str) -> None: - form = self._find(lang) - if form is None: + if self._find(lang) is None: raise KeyError(lang) - self.forms.remove(form) - - def get(self, lang: str, default: Text | None = None) -> Text | None: - form = self._find(lang) - return default if form is None else form.text - - def __contains__(self, lang: object) -> bool: - return isinstance(lang, str) and self._find(lang) is not None + # Every form for the language, so the key is gone afterwards. Sliced in + # place because callers hold `forms` directly. + self.forms[:] = [form for form in self.forms if form.lang != lang] + # Both read forms directly rather than through keys(): the inherited views + # are built on these two, so consulting a view here would not terminate. def __iter__(self) -> Iterator[str]: - return iter(self.keys()) + # One key per language — the form __getitem__ answers with — so the + # views, len() and dict(self) agree whatever forms holds. The snapshot + # lets a caller delete through the mapping while iterating it; walking + # forms live would skip the language after each removal. + seen: set[str] = set() + for form in tuple(self.forms): + if form.lang is not None and form.lang not in seen: + seen.add(form.lang) + yield form.lang def __len__(self) -> int: - return len(self.forms) + return sum(1 for _ in self) def __bool__(self) -> bool: + # Not derived from len(): emptiness here means "nothing to serialize", + # which residue and a lang-less form each defeat on their own. return bool(self.forms) or bool(self.extra) - def keys(self) -> list[str]: - return [form.lang for form in self.forms if form.lang is not None] - - def values(self) -> list[Text]: - return [form.text for form in self.forms if form.lang is not None] - - def items(self) -> list[tuple[str, Text]]: - return [(form.lang, form.text) for form in self.forms if form.lang is not None] - def __repr__(self) -> str: - inner = ", ".join(f"{form.lang!r}: {str(form.text)!r}" for form in self.forms) - return f"Multitext({{{inner}}})" + pairs = [(form.lang, str(form.text)) for form in self.forms] + langs = [lang for lang, _ in pairs] + # Dict-shaped only while the forms are one per language: a repeated or + # lang-less form would render as a dict literal that cannot exist and + # whose keys contradict keys(). Falling back to pairs keeps every form + # visible, and the shape is the signal that forms holds more than the + # mapping reaches. + if None not in langs and len(set(langs)) == len(langs): + inner = ", ".join(f"{lang!r}: {text!r}" for lang, text in pairs) + return f"Multitext({{{inner}}})" + inner = ", ".join(f"({lang!r}, {text!r})" for lang, text in pairs) + return f"Multitext([{inner}])" diff --git a/tests/test_property_roundtrip.py b/tests/test_property_roundtrip.py index 5a77d29..9f83732 100644 --- a/tests/test_property_roundtrip.py +++ b/tests/test_property_roundtrip.py @@ -36,7 +36,8 @@ # Attribute values: XML parsers normalize \t\n in attributes to spaces, so keep # tokens to characters that round-trip verbatim. _TOKEN = st.text(alphabet=_CHARS_INCL_NON_BMP, min_size=1, max_size=15) -_LANG = st.sampled_from(["en", "fr", "th", "sg", "es", "qaa-x-test"]) +_LANGS = ["en", "fr", "th", "sg", "es", "qaa-x-test"] +_LANG = st.sampled_from(_LANGS) _WHEN = st.one_of( st.none(), @@ -66,7 +67,10 @@ def _texts(draw: st.DrawFn) -> Text: @st.composite def _multitexts(draw: st.DrawFn) -> Multitext: - langs = draw(st.lists(_LANG, unique=True, max_size=3)) + # Langs are neither unique nor always present: a repeated language and a + # lang-less form are both schema-invalid shapes the reader accepts, so the + # round-trip has to hold for form lists the mapping cannot represent. + langs = draw(st.lists(st.sampled_from([*_LANGS, None]), max_size=3)) return Multitext([Form(lang, draw(_texts())) for lang in langs]) diff --git a/tests/test_reader.py b/tests/test_reader.py index 4a5e266..c134d0b 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -7,6 +7,7 @@ from sil_lift import LiftParseError, Span CORPUS_DIR = Path(__file__).parent / "corpus" +NEGATIVE_DIR = CORPUS_DIR / "negative" LOADABLE = sorted( p @@ -101,6 +102,29 @@ def test_subsenses_spot_check() -> None: assert str(sense_2.gloss("en") or "") == "master" +def test_repeated_form_lang_reads_as_one_key_over_both_forms() -> None: + lexicon = sil_lift.load(NEGATIVE_DIR / "duplicate-form-lang.lift") + (entry,) = lexicon.entries + lexical_unit = entry.lexical_unit + assert [(form.lang, str(form.text)) for form in lexical_unit.forms] == [ + ("en", "colour"), + ("en", "color"), + ] + assert list(lexical_unit.keys()) == ["en"] + assert str(lexical_unit["en"]) == "colour" # the first form for a language + assert len(lexical_unit) == len(dict(lexical_unit)) == 1 + + +def test_lang_less_form_reads_as_no_key_at_all() -> None: + lexicon = sil_lift.load(NEGATIVE_DIR / "schema-invalid.lift") + (entry,) = lexicon.entries + lexical_unit = entry.lexical_unit + assert [(form.lang, str(form.text)) for form in lexical_unit.forms] == [(None, "x")] + assert list(lexical_unit.keys()) == [] + assert len(lexical_unit) == 0 + assert lexical_unit # truthy: there is still a form to serialize + + def test_reversal_main_chain() -> None: lexicon = sil_lift.load(CORPUS_DIR / "spec-examples" / "0.13" / "reversals-hierarchy.lift") (entry,) = lexicon.entries @@ -178,7 +202,7 @@ def test_all_flex_fields_spot_check() -> None: assert span.class_ == "Hyperlink" (illustration,) = sense.illustrations assert illustration.href == "Desert.jpg" - assert illustration.label.keys() == ["th", "en", "fr"] + assert list(illustration.label.keys()) == ["th", "en", "fr"] other = lexicon.find(id="คาม ๒_dc4106ac-13fd-4ae0-a32b-b737f413d515") assert other is not None diff --git a/tests/test_text.py b/tests/test_text.py new file mode 100644 index 0000000..0edf386 --- /dev/null +++ b/tests/test_text.py @@ -0,0 +1,142 @@ +"""Tests for Multitext's mapping surface. + +`test_reader` covers what the reader builds from schema-invalid files and +`test_writer` covers what survives a save; these pin the mapping semantics +themselves — which key a repeated language resolves to, what the mutators +reach, and the two places Multitext deviates from `Mapping` on purpose. +Multitexts are built directly rather than parsed so each case states the form +list it is about. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +import pytest + +from sil_lift import Form, Multitext, Text + + +def _multitext(*pairs: tuple[str | None, str]) -> Multitext: + return Multitext(forms=[Form(lang, Text([text])) for lang, text in pairs]) + + +def test_is_a_mapping_and_the_views_are_views() -> None: + multitext = _multitext(("en", "dog"), ("fr", "chien")) + assert isinstance(multitext, Mapping) + # A view, not a list: set operations are part of what Mapping promises. + assert multitext.keys() & {"fr", "de"} == {"fr"} + assert multitext.keys() == {"en", "fr"} + assert list(multitext.items()) == [("en", multitext["en"]), ("fr", multitext["fr"])] + + +def test_keys_are_languages_in_insertion_order() -> None: + multitext = _multitext(("th", "a"), ("en", "b"), ("fr", "c")) + assert list(multitext.keys()) == ["th", "en", "fr"] + multitext["de"] = "d" + assert list(multitext.keys()) == ["th", "en", "fr", "de"] + + +def test_a_repeated_language_is_one_key_answering_with_the_first_form() -> None: + multitext = _multitext(("en", "first"), ("en", "second"), ("fr", "deux")) + assert list(multitext.keys()) == ["en", "fr"] + assert [str(text) for text in multitext.values()] == ["first", "deux"] + assert str(multitext["en"]) == "first" + assert len(multitext) == len(dict(multitext)) == 2 + # forms stays the full truth, which is where duplicate-form-lang reads from. + assert [str(form.text) for form in multitext.forms] == ["first", "second", "deux"] + + +def test_a_lang_less_form_is_not_a_key() -> None: + multitext = _multitext((None, "orphan"), ("fr", "deux")) + assert list(multitext.keys()) == ["fr"] + assert len(multitext) == 1 + assert None not in multitext + # The view is a Set, so it must not claim to hold what it will not yield. + keys = multitext.keys() + assert None not in keys + assert keys & {None} == set() + assert multitext.get(None) is None # type: ignore[call-overload] + with pytest.raises(KeyError): + multitext[None] # type: ignore[index] + assert [str(form.text) for form in multitext.forms if form.lang is None] == ["orphan"] + + +def test_assignment_updates_the_named_form_and_leaves_later_duplicates() -> None: + multitext = _multitext(("en", "first"), ("en", "second"), ("fr", "deux")) + multitext["en"] = "edited" + # A Form carries annotations and residue the mapping cannot show a caller, + # so assignment never discards one it was not asked about. + assert [(form.lang, str(form.text)) for form in multitext.forms] == [ + ("en", "edited"), + ("en", "second"), + ("fr", "deux"), + ] + + +def test_assignment_coerces_a_plain_string_and_appends_a_new_language() -> None: + multitext = Multitext() + multitext["en"] = "dog" + assert isinstance(multitext["en"], Text) + assert str(multitext["en"]) == "dog" + text = Text(["chien"]) + multitext["fr"] = text + assert multitext["fr"] is text + + +def test_deletion_removes_every_form_for_the_language() -> None: + multitext = _multitext(("en", "first"), ("en", "second"), (None, "orphan"), ("fr", "deux")) + del multitext["en"] + assert "en" not in multitext + assert [(form.lang, str(form.text)) for form in multitext.forms] == [ + (None, "orphan"), + ("fr", "deux"), + ] + with pytest.raises(KeyError): + del multitext["en"] + + +def test_deleting_through_the_mapping_while_iterating_it_reaches_every_language() -> None: + multitext = _multitext(("en", "a"), ("fr", "b"), ("de", "c"), ("es", "d")) + keys = multitext.keys() # a live view, walked while its mapping shrinks + for lang in keys: + del multitext[lang] + assert multitext.forms == [] + + multitext = _multitext(("en", "a"), ("fr", "b"), ("de", "c"), ("es", "d")) + for lang in multitext: + if lang != "en": + del multitext[lang] + assert list(multitext.keys()) == ["en"] + + +def test_truthiness_asks_whether_there_is_anything_to_serialize() -> None: + assert not Multitext() + # Empty as a mapping, but there is a form the writer must emit. The other + # case truthiness exists for — residue and no forms at all — needs a parsed + # document to build, so test_writer owns it. + lang_less = _multitext((None, "orphan")) + assert lang_less + assert len(lang_less) == 0 + + +def test_equality_compares_form_lists_not_mapping_contents() -> None: + assert _multitext(("en", "dog")) == _multitext(("en", "dog")) + # Stricter than Mapping equality: a form no key reaches still counts. + assert _multitext(("en", "dog")) != _multitext(("en", "dog"), (None, "orphan")) + assert _multitext(("en", "dog")) != {"en": Text(["dog"])} + + +def test_repr_is_dict_shaped_until_the_forms_are_not_one_per_language() -> None: + assert repr(Multitext()) == "Multitext({})" + assert repr(_multitext(("en", "dog"), ("fr", "chien"))) == ( + "Multitext({'en': 'dog', 'fr': 'chien'})" + ) + # A dict literal cannot hold either of these, and keys() excludes both, so + # the shape changes rather than printing keys the mapping does not have. + assert repr(_multitext(("en", "first"), ("en", "second"))) == ( + "Multitext([('en', 'first'), ('en', 'second')])" + ) + assert repr(_multitext((None, "orphan"))) == "Multitext([(None, 'orphan')])" + # The views repr through the mapping, so the shape reaches them too. + assert repr(_multitext(("en", "dog")).keys()) == "KeysView(Multitext({'en': 'dog'}))" diff --git a/tests/test_writer.py b/tests/test_writer.py index 57ae9d6..b5da420 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -209,6 +209,40 @@ def test_out_of_schema_content_survives_touched_reserialization(tmp_path: Path) assert _semantic_bytes(result) != b"" # well-formed enough to canonicalize +RESIDUE_ONLY_MULTITEXT = b""" + + + +ONE + + +""" + + +def test_multitext_carrying_only_residue_survives_touched_reserialization( + tmp_path: Path, +) -> None: + # A multitext with no forms is empty as a mapping, so the writer's decision + # to emit it cannot come from len(). Nothing else in the corpus has this + # shape, and losing it would lose the attribute silently. + source = tmp_path / "residue-only.lift" + source.write_bytes(RESIDUE_ONLY_MULTITEXT) + lexicon = sil_lift.load(source) + entry = lexicon.find(id="one") + assert entry is not None + assert len(entry.lexical_unit) == 0 + assert entry.lexical_unit.forms == [] + assert entry.lexical_unit # the residue alone is worth serializing + + entry.senses[0].glosses[0].text = sil_lift.Text(["ONE (edited)"]) + out = tmp_path / "roundtrip.lift" + lexicon.save(out) + assert b'note="keepme"' in out.read_bytes() + reloaded = sil_lift.load(out).find(id="one") + assert reloaded is not None + assert reloaded.lexical_unit.extra + + # Residue that is neither an element nor an attribute: a processing instruction, # stray character data in element-only contexts (both as an element's leading # text and as a later child's tail), a comment and a processing instruction