From ba0846b58989b3ebadf309fdaf1c13fb684393f8 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 2 Sep 2026 15:30:27 -0400 Subject: [PATCH 1/4] Stamp dateModified on the entries save() rewrites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lexicon.save() and save_zip() take stamp (default True) and when. Before serializing, stamp_entries() sets dateModified — filling a blank dateCreated with the same moment — on every entry whose canonical digest moved while its date stayed where its baseline had it. Until now nothing in the library generated a timestamp, so an edited entry went out under the date it was read with and every tool that reconciles LIFT on dateModified saw an unmodified lexicon. stamp=False writes the model exactly as it stands; when= supplies the moment in place of the wall clock (UTC at seconds precision, the 20-character form real FieldWorks exports use without exception), which is what makes stamped output byte-reproducible. Entries only. All 35,318 entries in the seven FieldWorks 8.3-9.0 exports in The Combine's Backend.Tests/Assets carry both stamps and not one of their 69,754 sub-entry nodes carries either, and an entry's digest already spans its whole subtree, so an edit to a nested subsense stamps the entry containing it. _ExtensibleNoFields._stamp holds the policy, so the other eight date-bearing types inherit it if they ever need it. _EntryRecord gains the dateModified it held when the record was taken, which is what separates "the content changed and the date did not" from "the caller set the date deliberately". The parse-time records keep driving byte reuse and change detection; each save records what it wrote in Lexicon._stamps and the next save measures against that, without which a second round of edits on one loaded lexicon would read as caller-set and ship unstamped. Keeping the two baselines apart is what leaves changed_entries() answering "since the load" rather than "since the last save". An entry still matching its parse-time record is recorded nowhere, so the bookkeeping is the size of the edit. An entry with no baseline at all — appended after the load, or in a lexicon built from scratch — is stamped only where its dateModified is blank, so an exporter carrying real dates in from another data model keeps them. Its first save records a baseline either way, so a later edit to it does bump. iter_problems() does not stamp. It stays read-only, and its docstring says the bytes it validates precede the stamping a save does rather than claiming to be what save() would write; nothing generated is ever a finding, so validating first and saving after is sound. canonicalize() generates nothing either: sorting and reformatting change no entry's content. An unparseable date is residue rather than a date, so a stamp replaces it and the original string is dropped — a consequence pinned by a test and documented alongside the rest in docs/en/fidelity.md, with the guides that teach editing and building an export, and the streaming writer's note that it stamps nothing. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 11 + README.md | 7 +- docs/en/fidelity.md | 4 +- docs/en/guides/build-export.md | 7 +- docs/en/guides/bulk-edit-glosses.md | 1 + docs/en/guides/large-files.md | 1 + docs/en/guides/lift-export-interop.md | 1 + docs/en/guides/read-edit-write.md | 2 + docs/en/guides/validate.md | 2 +- docs/en/index.md | 2 +- src/sil_lift/_canonical.py | 4 + src/sil_lift/_model.py | 84 ++++++- src/sil_lift/_reader.py | 5 +- src/sil_lift/_writer.py | 89 ++++++- tests/test_stamp.py | 345 ++++++++++++++++++++++++++ 15 files changed, 545 insertions(+), 20 deletions(-) create mode 100644 tests/test_stamp.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ec04d9b..5fdb6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,17 @@ releases may contain breaking changes. represent — a lone surrogate, which only an API assignment can introduce — is refused with `LiftWriteError` naming the node, and reported by validation as `lone-surrogate`. +- Generated timestamps on save: `Lexicon.save()` and `Lexicon.save_zip()` stamp + `dateModified` on every entry whose content changed since it was loaded, and + fill a blank `dateCreated` with the same moment — an edit shipped under its + loaded date looks unmodified to everything that reconciles on that attribute. + `` only, however deep the edit (the parse-time digests are what drive + it, so an edit to a nested subsense stamps the entry containing it); an entry + whose date the caller set deliberately is left alone, as is an entry created + since the load that already carries one, and reordering stamps nothing. + `stamp=False` writes the model exactly as it stands, and `when=` supplies the + moment in place of the wall clock (UTC at seconds precision), which is what + keeps stamped output byte-reproducible. - Change detection against the loaded document, reading the same parse-time digests. `Lexicon.changed_entries()` reports entries whose content differs (an entry's digest covers its whole subtree, so an edit at any depth reports diff --git a/README.md b/README.md index f9796ef..585b494 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ for entry in lex.entries: ... entry = lex.find(id="hoofd_a1b2") entry.senses[0].definition["en"] = "head (anatomy)" -lex.save() # untouched entries byte-identical +lex.save() # edits stamped; the rest verbatim ``` **Status: pre-release, under active development.** The API is not yet stable. @@ -35,7 +35,10 @@ not understand. - Entries you did not modify are written back **byte-identical**, even when other entries changed. - Entries you did modify are re-serialized in a documented canonical form, with - all out-of-schema content (unknown elements, attributes, comments) preserved. + all out-of-schema content (unknown elements, attributes, comments) preserved, + and are stamped with a fresh `dateModified` so the edit does not go out under + the date it was loaded with (`save(stamp=False)` writes the dates the model + holds; `save(when=...)` pins the moment). - Whitespace inside `` is never altered — not even for indentation. The precise rules and their few edge cases are documented in diff --git a/docs/en/fidelity.md b/docs/en/fidelity.md index 6b9c2df..de748b6 100644 --- a/docs/en/fidelity.md +++ b/docs/en/fidelity.md @@ -8,7 +8,7 @@ Any well-formed LIFT 0.13 document loads — schema-invalid content included. Wh ## Saving an unchanged document -`load()` → `save()` with no edits writes **byte-identical output** — no reformatting, no re-escaping, no reordering, byte-order marks and XML declarations included. There is currently no normalization list: identity is exact. +`load()` → `save()` with no edits writes **byte-identical output** — no reformatting, no re-escaping, no reordering, byte-order marks and XML declarations included. There is currently no normalization list: identity is exact. Timestamps are generated from content, so an unchanged document has none generated either. Exceptions (the writer falls back to full canonical serialization, which is semantically complete but not byte-preserving): @@ -22,6 +22,8 @@ Exceptions (the writer falls back to full canonical serialization, which is sema - **Untouched entries are emitted verbatim from their original bytes.** An entry counts as touched if any part of its model object changed since parse (detected by canonical-serialization snapshot, not a dirty flag). - **Touched entries are re-serialized canonically and completely**: UTF-8, 2-space indentation _outside_ mixed content (whitespace inside `` and `` is never altered), a documented child grouping per element (e.g. entry: lexical-unit, citation, pronunciations, variants, senses, notes, relations, etymologies, annotations, traits, fields), fixed attribute order, dates in ISO-8601 (`Z` for UTC). All residue is re-emitted; its position is restored to the original child index, clamped to the new child list (an approximation — exact byte positions are only guaranteed for untouched entries). - Adding, removing, or reordering entries re-serializes the document structure but still emits every unchanged entry's bytes verbatim. +- **A touched entry is stamped.** `save()` writes a fresh `dateModified` on every entry whose content changed since it was read, and fills a blank `dateCreated` with the same moment — an edit shipped under its loaded date looks unmodified to everything that reconciles on that attribute, FieldWorks and The Combine's LIFT import included. Only `` is stamped: no node below one, and nothing in the header. An entry whose date the caller set deliberately keeps it, and so does an entry created since the load that already carries one. A date the model could not parse is [residue](#reading) rather than a date, so a stamp replaces it and the original string is dropped — an edited entry is better off carrying a real date than `dateModified="whenever"`. `save(stamp=False)` writes the model exactly as it stands, residue included. +- **A generated stamp is the one thing in the output that is not a function of the input.** Stamps are UTC at seconds precision (`YYYY-MM-DDTHH:MM:SSZ` — the shape every surveyed FieldWorks export uses), read from the wall clock. `save(when=...)` supplies the moment instead, which is what keeps stamped output reproducible for a diff-based CI gate. !!! note ""Canonical" here is not related to any other Canonical XML" Canonical form on this page means `sil-lift`'s own documented shape, described in a bullet above. It is unrelated to W3C's Canonical XML (C14N) process. It is unrelated to `SIL.Core`'s `CanonicalXmlSettings` class. diff --git a/docs/en/guides/build-export.md b/docs/en/guides/build-export.md index ce13b26..b073def 100644 --- a/docs/en/guides/build-export.md +++ b/docs/en/guides/build-export.md @@ -52,7 +52,7 @@ ranges.add_range("grammatical-info").add_element("Noun").label["en"] = "noun" ranges.add_range("semantic-domain-ddp4").add_element("1.6.1.2").label["en"] = "Bird" lex.add_ranges_file(ranges, href="birds.lift-ranges") -# Validate what save() would write, before touching the disk. +# Validate the document as it stands, before touching the disk. problems = list(lex.iter_problems()) print(f"validation: {len(problems)} problem(s)") @@ -79,7 +79,7 @@ print((out / "birds.lift-ranges").read_text(encoding="utf-8"), end="") - +
nkhuku @@ -161,7 +161,8 @@ print((out / "birds.lift-ranges").read_text(encoding="utf-8"), end="") - A `URLRef` is an href plus an optional caption/label multitext — used for both `` (audio) and `` (photos). The pronunciation here follows The Combine's convention of an `en` form reading `Speaker: `. - App-specific data with no native LIFT home rides as a `` (or ``): FieldWorks reads these as custom fields and The Combine preserves them. - Give every entry a real, stable `guid` (e.g. from `uuid.uuid4()`, reused across exports) — a later re-import updates the entry in place rather than duplicating it. `sil-lift validate --require-ids` enforces this. -- `lex.iter_problems()` validates the in-memory document (what `save()` would write) before anything hits disk; here it is clean. Because the lexicon has no folder yet, the media-presence and companion-href checks are skipped — run [`sil-lift validate`](cli.md) on the saved output (or with `--no-check-media`) once the audio and photo files are in place. +- The `dateCreated`/`dateModified` in the output above are not in the script: `save()` stamped them with the moment it ran, because an entry it is writing for the first time carries no date of its own and the tools that import LIFT decide what to update from `dateModified`. Two knobs, both on `save()`: `when=` supplies the moment instead of reading the clock — that is what makes a generated export byte-reproducible, so a CI job can diff it — and `stamp=False` writes no dates at all. A date you set yourself is left alone either way, so an exporter carrying real timestamps over from its own data model keeps them. Nothing below `` is ever stamped. +- `lex.iter_problems()` validates the in-memory document before anything hits disk; here it is clean. Because the lexicon has no folder yet, the media-presence and companion-href checks are skipped — run [`sil-lift validate`](cli.md) on the saved output (or with `--no-check-media`) once the audio and photo files are in place. ## Packaging diff --git a/docs/en/guides/bulk-edit-glosses.md b/docs/en/guides/bulk-edit-glosses.md index e5976cb..63ab988 100644 --- a/docs/en/guides/bulk-edit-glosses.md +++ b/docs/en/guides/bulk-edit-glosses.md @@ -59,6 +59,7 @@ A few things worth noting: - It compares content, not destination, so guard only an in-place save with it: `lex.save(some_other_dir / "dictionary.lift")` writes the document and its companions to a location that has nothing in it yet, whether or not anything changed. - It is a guard, not a speed-up — answering it digests every entry, which is the same work `save()` does to decide which source bytes it can reuse, so what you skip is the write itself (an unchanged file-modification time, no spurious diff), not the effort of deciding. - Validating in memory (`lex.iter_problems()`) serializes the edited state first, so it correctly reflects the edit before anything is written to disk. Aborting on any `"error"`-level `Problem` — warnings are left for the caller to decide about — means a bad edit never reaches `save()`. +- `lex.save()` stamps a fresh `dateModified` on exactly those changed entries as it writes them, filling `dateCreated` where it was blank. Without that the edited entries would go out under the dates they were loaded with, and a lexicon this script has rewritten would look untouched to FieldWorks or to The Combine's LIFT import, both of which decide what to update from `dateModified`. It stamps `` only, however deep the edit was — the entry containing the edited subsense gloss, not the sense. `save(stamp=False)` writes the dates the model holds; `save(when=...)` supplies the moment rather than reading the clock, so a pipeline that diffs its own output stays reproducible. Glosses aren't the only thing worth touching this way. The same `Multitext` mapping surface applies to definitions and every other multilingual field on an entry or sense: diff --git a/docs/en/guides/large-files.md b/docs/en/guides/large-files.md index f2bf498..fe243c1 100644 --- a/docs/en/guides/large-files.md +++ b/docs/en/guides/large-files.md @@ -24,4 +24,5 @@ Notes: - The writer's output is exactly what the full-document canonical serializer would produce for the same content — the two modes never drift apart. - Streaming mode reuses no source bytes: output is always canonical. Root-level LIFT residue — comments between entries and out-of-schema attributes on `` — is not carried; entries and the header are complete, residue included. +- Nor does it generate timestamps. An entry is written with the dates it carries, since a streaming writer has no loaded document to compare it against — the stamping [`Lexicon.save()`](../fidelity.md#saving-an-edited-document) does needs that baseline. Set `entry.date_modified` yourself on the entries this pass rewrites. - If the body of an `open_writer` block raises, the file is left visibly unterminated (no closing ``) — a half-written lexicon must not look complete. diff --git a/docs/en/guides/lift-export-interop.md b/docs/en/guides/lift-export-interop.md index 046a8e8..4e986e2 100644 --- a/docs/en/guides/lift-export-interop.md +++ b/docs/en/guides/lift-export-interop.md @@ -14,6 +14,7 @@ LIFT is usually moved around as a single `.zip` — FieldWorks and The Combine b - Extraction is capped at 10 GiB and 100,000 members; a package over either limit is refused with a `LiftParseError`, as is one whose member paths escape the extraction directory. - **Write:** `Lexicon.save_zip("out.zip", wrap_folder="MyDict")` packages the `.lift`, its `.lift-ranges`, and every other file in the source folder (media, `WritingSystems/`, `consent/`, ...) into a zip. - `wrap_folder` defaults to a top-level folder named after the zip (the FieldWorks/Combine import convention); pass `False` for a flat archive. + - Entries whose content changed since the load are stamped with a fresh `dateModified` on the way out, exactly as on `save()` — a package is what an importing tool reconciles from, so a stale date there is what makes an updated lexicon look untouched. `stamp=False` writes the dates the model holds; `when=` pins the moment. The `.lift` and `.lift-ranges` keep their byte-fidelity inside the package; the zip container itself is not byte-reproducible. diff --git a/docs/en/guides/read-edit-write.md b/docs/en/guides/read-edit-write.md index c7fe813..cd8e9ec 100644 --- a/docs/en/guides/read-edit-write.md +++ b/docs/en/guides/read-edit-write.md @@ -41,6 +41,8 @@ lex.save("elsewhere.lift") Entries you didn't modify are written back **byte-identical**; a document you didn't modify at all is byte-identical from the first byte to the last. See [Fidelity guarantees](../fidelity.md) for the precise contract. +The entries you did modify go out with a fresh `dateModified` (and a `dateCreated` if they had none), so an edit doesn't ship under the date it was loaded with — the tools that merge LIFT decide what changed from that attribute. `lex.save(stamp=False)` writes the dates the model holds and nothing more; `lex.save(when=...)` pins the moment instead of reading the clock. + ## Building from scratch ```python diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index b6fb006..0430526 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -44,7 +44,7 @@ Every finding carries one of these, whichever layer produced it — `schema` and | `undefined-range-value` | warning | a grammatical-info or range-keyed trait value the range does not list | | `uri-not-rfc` | warning | an href that is not a valid URI — FLEx's `file://C:/...` | -All three layers work from what `save()` would write, so a document that cannot be serialized at all is reported as a single `lone-surrogate` error instead — see [Fidelity guarantees](../fidelity.md#content-xml-cannot-represent). +All three layers work from the document serialized as it stands, so one that cannot be serialized at all is reported as a single `lone-surrogate` error instead — see [Fidelity guarantees](../fidelity.md#content-xml-cannot-represent). Validating is read-only, which is the one way those bytes differ from the bytes `save()` writes: it reports the document before the `dateModified` stamping a save does. Nothing generated is ever a finding, so validate-then-save is sound. ## Real-world FieldWorks (FLEx) output diff --git a/docs/en/index.md b/docs/en/index.md index 889b6e4..4f0ca96 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -28,5 +28,5 @@ for entry in lex.entries: entry = lex.find(guid="0f5a9c3e-...") # or lex.find(id="hoofd_a1b2") entry.senses[0].definition["en"] = "head (anatomy)" -lex.save() # untouched entries byte-identical; edited entry re-serialized +lex.save() # edited entry re-serialized and re-stamped; the rest byte-identical ``` diff --git a/src/sil_lift/_canonical.py b/src/sil_lift/_canonical.py index 10c08e3..54e17e3 100644 --- a/src/sil_lift/_canonical.py +++ b/src/sil_lift/_canonical.py @@ -72,6 +72,10 @@ def canonicalize(src: str | os.PathLike[str], dst: str | os.PathLike[str]) -> No diff cleanly. Text content is never whitespace-normalized. The whole document is held in memory (sorting requires it; the C# oracle buffers too). + No timestamp is generated either: sorting and reformatting change no entry's + content, so nothing here is a modification to stamp. The output is a pure + function of the input. + Only the ``.lift`` file is written: companion ``.lift-ranges`` files are neither read nor rewritten (the source is loaded with ``resolve_ranges=False``). Sort a ranges file separately via diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 8597750..aed2ab6 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -27,7 +27,7 @@ from typing import Literal from ._validate import Problem - from ._writer import _RangesSourceInfo, _SourceInfo + from ._writer import _EntryRecord, _RangesSourceInfo, _SourceInfo __all__ = [ "Changes", @@ -62,6 +62,22 @@ class _ExtensibleNoFields: traits: list[Trait] = field(default_factory=list) extra: Extras = field(default_factory=Extras) + def _stamp(self, when: datetime) -> None: + """Record ``when`` as this node's modification moment. + + A blank ``dateCreated`` is filled with the same moment: a node whose + creation went unrecorded was created no later than the change being + stamped, and leaving it blank while ``dateModified`` fills in reads as + a node that was modified before it existed. + + The one place either date is generated, so that the nine date-bearing + types share one policy — though only :class:`Entry` is stamped today + (see :meth:`Lexicon.save`). + """ + self.date_modified = when + if self.date_created is None: + self.date_created = when + @dataclass(slots=True, kw_only=True) class _Extensible(_ExtensibleNoFields): @@ -472,6 +488,7 @@ class Lexicon: __slots__ = ( "_source", + "_stamps", "_tempdir", "entries", "extra", @@ -497,6 +514,7 @@ def __init__( self.extra = extra if extra is not None else Extras() self.ranges_files: dict[Path, RangesFile] = {} self._source: _SourceInfo | None = None # set by the reader + self._stamps: dict[int, _EntryRecord] = {} # stamping baselines (see stamp_entries) self._tempdir: tempfile.TemporaryDirectory[str] | None = None # zip extraction, if any @classmethod @@ -553,7 +571,21 @@ def _resolve_ranges(self) -> None: if exists and resolved not in self.ranges_files: self.ranges_files[resolved] = RangesFile.load(candidate) - def save(self, path: str | os.PathLike[str] | None = None) -> None: + def _apply_stamps(self, stamp: bool, when: datetime | None) -> None: + """The stamping step shared by :meth:`save` and :meth:`save_zip`.""" + if not stamp: + return + from ._writer import default_now, stamp_entries + + stamp_entries(self, when if when is not None else default_now()) + + def save( + self, + path: str | os.PathLike[str] | None = None, + *, + stamp: bool = True, + when: datetime | None = None, + ) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. Untouched entries are emitted byte-identical to the source; modified @@ -564,10 +596,26 @@ def save(self, path: str | os.PathLike[str] | None = None) -> None: name in the *same* directory leaves companions at their original paths (they are shared with the original document, not copied). + Every entry whose content changed since the load goes out with a fresh + ``dateModified``, and with a ``dateCreated`` if it had none — an edit + shipped under its loaded date looks unmodified to everything downstream + that reconciles on that attribute. Three cases are left as they stand: + an entry whose date the caller set deliberately, an entry created since + the load that already carries one, and an untouched entry, reordering + included (see :meth:`sort`). Depth does not matter: an edit to a gloss + on a nested subsense stamps the entry containing it. This mutates the + model — here, and in a ``save(path)`` used to export a copy — and costs + one canonical serialization pass over the entries. + + ``stamp=False`` writes the model exactly as it stands. ``when`` + supplies the moment in place of the clock (UTC at seconds precision), + which is what makes stamped output byte-reproducible. + Raises :class:`ValueError` if no target path is available (none was passed and the lexicon was not loaded from a file), and :class:`~sil_lift.LiftWriteError` if the model holds content XML cannot - represent (a lone surrogate) — nothing is written in that case. + represent (a lone surrogate) — nothing is written in that case, though + stamps already applied stay on the model. """ from ._writer import render_document @@ -575,6 +623,7 @@ def save(self, path: str | os.PathLike[str] | None = None) -> None: if target is None: raise ValueError("no target path: pass save(path) or load the lexicon from a file") original_dir = self.path.parent if self.path is not None else None + self._apply_stamps(stamp, when) target.write_bytes(render_document(self)) self.path = target relocating = not _same_dir(target.parent, original_dir) @@ -594,7 +643,14 @@ def save(self, path: str | os.PathLike[str] | None = None) -> None: if ranges_file.path is not None } - def save_zip(self, path: str | os.PathLike[str], *, wrap_folder: str | bool = True) -> None: + def save_zip( + self, + path: str | os.PathLike[str], + *, + wrap_folder: str | bool = True, + stamp: bool = True, + when: datetime | None = None, + ) -> None: """Write the lexicon and its folder companions as a zip package. The ``.lift`` and ``.lift-ranges`` are (re-)serialized with the usual @@ -605,9 +661,13 @@ def save_zip(self, path: str | os.PathLike[str], *, wrap_folder: str | bool = Tr convention FieldWorks and The Combine expect on import — ``False`` writes the files at the archive root, and a string uses that folder name. The archive container itself is not byte-reproducible. + + ``stamp`` and ``when`` work exactly as on :meth:`save`, and matter more + here: a package is the hand-off to the tools that read ``dateModified``. """ from ._zip import save_zip + self._apply_stamps(stamp, when) save_zip(self, Path(path), wrap_folder=wrap_folder) def sort(self) -> None: @@ -779,11 +839,17 @@ def changes(self) -> Changes: def iter_problems(self, *, require_ids: bool = False) -> Iterator[Problem]: """Validate the in-memory state (schema layers + semantic checks). - The schema layers need serialized bytes: what :meth:`save` would - write is validated, so in-memory edits are always visible. For an - untouched loaded document those are the source bytes (line numbers - match the file on disk); otherwise serialization is a documented - cost on large lexicons. + The schema layers need serialized bytes, so the document is serialized + as it stands and those bytes are validated: in-memory edits are always + visible. For an untouched loaded document they are the source bytes + (line numbers match the file on disk); otherwise serialization is a + documented cost on large lexicons. + + Read-only, which is the one way the bytes validated here can differ + from the bytes written: :meth:`save` stamps ``dateModified`` on edited + entries, and this reports the document as it stands, before any of + that. Nothing generated is ever a finding — a stamp is a well-formed + date in a valid place — so validating first and saving after is sound. With ``require_ids``, entries missing a ``guid`` and senses missing an ``id`` are reported as ``missing-id`` errors — stricter than LIFT (both diff --git a/src/sil_lift/_reader.py b/src/sil_lift/_reader.py index ef98f1d..64d025d 100644 --- a/src/sil_lift/_reader.py +++ b/src/sil_lift/_reader.py @@ -135,7 +135,10 @@ def _attach_source(lexicon: Lexicon, data: bytes, root: etree._Element) -> None: root_open_end=result.root_open_end, root_self_closing=result.root_self_closing, children=result.children, - entry_records=[_EntryRecord(entry, entry_digest(entry)) for entry in lexicon.entries], + entry_records=[ + _EntryRecord(entry, entry_digest(entry), entry.date_modified) + for entry in lexicon.entries + ], header_digest=header_digest(lexicon.header) if header_regions else None, producer=lexicon.producer, root_extra_attrs=dict(lexicon.extra._attrs), diff --git a/src/sil_lift/_writer.py b/src/sil_lift/_writer.py index fcb38d9..08704fc 100644 --- a/src/sil_lift/_writer.py +++ b/src/sil_lift/_writer.py @@ -14,7 +14,9 @@ are re-serialized canonically. A fully-unchanged document therefore reassembles byte-identically. -Snapshots are sha256 digests of canonical bytes, taken at parse time. +Snapshots are sha256 digests of canonical bytes, taken at parse time. The same +digests decide, in :func:`stamp_entries`, which entries an edit has left with a +stale ``dateModified``. Both paths refuse content XML cannot represent: see :func:`_guarded`. """ @@ -24,6 +26,7 @@ import hashlib from collections import Counter from dataclasses import dataclass +from datetime import UTC, date, datetime from typing import TYPE_CHECKING from lxml import etree @@ -53,7 +56,6 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable - from datetime import date, datetime from ._extras import _ExtraNode from ._scan import ChildRegion @@ -61,12 +63,14 @@ __all__ = [ "canonical_document", "canonical_ranges_document", + "default_now", "entry_digest", "header_digest", "node_diff", "range_digest", "render_document", "render_ranges_document", + "stamp_entries", ] _FRAGMENT_PARSER = etree.XMLParser(resolve_entities=False, no_network=True) @@ -105,8 +109,17 @@ def _entry_label(entry: Entry) -> str: @dataclass(slots=True) class _EntryRecord: + """An entry's canonical digest and ``dateModified`` as of one baseline moment. + + The reader builds one per entry at parse time, and those drive byte reuse + and change detection. :func:`stamp_entries` builds a second set as it + saves, so the baseline the stamping policy measures against moves forward + with each save while the parse-time one stays where it is. + """ + entry: Entry # strong ref: keeps id() stable for the identity check digest: bytes + date_modified: datetime | date | None @dataclass(slots=True) @@ -153,6 +166,78 @@ def range_digest(range_: Range) -> bytes: return hashlib.sha256(canonical_range_bytes(range_)).digest() +# --- generated timestamps ------------------------------------------------------- + + +def default_now() -> datetime: + """The clock for generated timestamps: UTC at seconds precision. + + Seconds precision is the shape real exports use. Across the seven + FieldWorks 8.3-9.0 exports in The Combine's ``Backend.Tests/Assets``, all + 70,636 ``dateCreated``/``dateModified`` literals are exactly the + 20-character ``YYYY-MM-DDTHH:MM:SSZ`` form — no bare dates, numeric + offsets, or fractional seconds. :func:`_fmt_date` renders an aware UTC + value that way. + """ + return datetime.now(UTC).replace(microsecond=0) + + +def _needs_stamp(entry: Entry, baseline: _EntryRecord | None, digest: bytes) -> bool: + """Whether the content moved since ``baseline`` while the date stayed put. + + Both halves matter. The digest covers the entry's whole subtree, so it + catches an edit at any depth; the date comparison is what leaves an entry + the caller dated by hand alone, since a deliberate value is not one to + overwrite. + + Without a baseline there is nothing to compare — an entry added since the + load, or a lexicon built from scratch. A date already on such an entry is + taken as deliberate too (a migration carrying dates in from another format + is the case that matters), so only a blank one is filled. + """ + if baseline is None: + return entry.date_modified is None + return digest != baseline.digest and entry.date_modified == baseline.date_modified + + +def stamp_entries(lexicon: Lexicon, when: datetime) -> None: + """Stamp every entry whose content changed without its ``dateModified`` changing. + + Costs one canonical serialization pass over the entries. + + Each save leaves behind the state it wrote, in ``lexicon._stamps``, and the + next save measures against that rather than against the load. Without it a + second round of edits on the same in-memory lexicon would ship unstamped: + its content differs from the loaded content all right, but so does its date + — this library's own stamp from the first save — which reads exactly like + a date the caller set deliberately. An entry still matching its parse-time + record needs no such override and keeps none, so the common save of a + handful of edited entries remembers only those. + """ + source = lexicon._source + at_parse = ( + {id(record.entry): record for record in source.entry_records} if source is not None else {} + ) + for entry in lexicon.entries: + key = id(entry) + record = at_parse.get(key) + baseline = lexicon._stamps.get(key) + if baseline is None: + baseline = record + digest = entry_digest(entry) + if _needs_stamp(entry, baseline, digest): + entry._stamp(when) + digest = entry_digest(entry) # the dates are part of an entry's bytes + if ( + record is not None + and record.digest == digest + and record.date_modified == entry.date_modified + ): + lexicon._stamps.pop(key, None) + else: + lexicon._stamps[key] = _EntryRecord(entry, digest, entry.date_modified) + + # --- canonical building blocks --------------------------------------------------- diff --git a/tests/test_stamp.py b/tests/test_stamp.py new file mode 100644 index 0000000..4b2cd5d --- /dev/null +++ b/tests/test_stamp.py @@ -0,0 +1,345 @@ +"""Generated ``dateModified``/``dateCreated`` on save. + +The unit is the entry: across the seven FieldWorks 8.3-9.0 exports in The +Combine's ``Backend.Tests/Assets``, all 35,318 entries carry both stamps and not +one of 69,754 sub-entry nodes carries either. An entry's digest already covers +its whole subtree, so an edit at any depth belongs to the entry containing it. +""" + +import zipfile +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import sil_lift + +CORPUS_DIR = Path(__file__).parent / "corpus" +DATED = CORPUS_DIR / "misc" / "sample.0.13.lift" # every entry carries both stamps +UNDATED = CORPUS_DIR / "spec-examples" / "0.13" / "subsenses.lift" # no entry carries either + +# Dates no typed field can hold: the reader keeps them as residue instead. +UNPARSEABLE_DATES = b""" + + +one + + +""" + +WHEN = datetime(2026, 3, 4, 5, 6, 7, tzinfo=UTC) +LATER = datetime(2026, 3, 4, 5, 6, 8, tzinfo=UTC) +BY_HAND = datetime(1999, 12, 31, 23, 59, 59, tzinfo=UTC) + + +def _dates(lexicon: sil_lift.Lexicon) -> dict[str | None, tuple[object, object]]: + """Every entry's stamps, keyed by id so a sort() does not disturb the comparison.""" + return {entry.id: (entry.date_created, entry.date_modified) for entry in lexicon.entries} + + +def test_an_untouched_save_stamps_nothing_and_stays_byte_identical(tmp_path: Path) -> None: + """Stamping is driven by content, so a load-and-save writes the source bytes back.""" + lexicon = sil_lift.load(DATED) + before = _dates(lexicon) + out = tmp_path / "out.lift" + lexicon.save(out, when=WHEN) + + assert _dates(lexicon) == before + assert out.read_bytes() == DATED.read_bytes() + # Every entry still matches its parse-time record, so nothing had to be + # remembered for the next save — the bookkeeping tracks edits, not entries. + assert lexicon._stamps == {} + + +def test_an_edit_at_any_depth_stamps_the_containing_entry(tmp_path: Path) -> None: + lexicon = sil_lift.load(UNDATED) + entry = lexicon.entries[0] + entry.senses[0].subsenses[0].glosses[0].text = sil_lift.Text(["edited"]) + out = tmp_path / "out.lift" + lexicon.save(out, when=WHEN) + + assert entry.date_modified == WHEN + assert entry.date_created == WHEN # blank before, so filled with the same moment + assert b'dateCreated="2026-03-04T05:06:07Z" dateModified="2026-03-04T05:06:07Z"' in ( + out.read_bytes() + ) + + +def test_only_the_edited_entry_is_stamped(tmp_path: Path) -> None: + lexicon = sil_lift.load(DATED) + assert len(lexicon.entries) > 1 + target = lexicon.entries[3] + untouched = [entry for entry in lexicon.entries if entry is not target] + before = [(entry.date_created, entry.date_modified) for entry in untouched] + target.lexical_unit["en"] = "edited" + lexicon.save(tmp_path / "out.lift", when=WHEN) + + assert target.date_modified == WHEN + assert [(entry.date_created, entry.date_modified) for entry in untouched] == before + + +def test_an_existing_date_created_survives_the_stamp(tmp_path: Path) -> None: + """dateCreated is filled only when blank: an edit does not re-create an entry.""" + lexicon = sil_lift.load(DATED) + target = lexicon.entries[0] + created = target.date_created + assert created is not None + target.lexical_unit["en"] = "edited" + lexicon.save(tmp_path / "out.lift", when=WHEN) + + assert target.date_created == created + assert target.date_modified == WHEN + + +def test_stamp_false_writes_the_model_exactly_as_it_stands(tmp_path: Path) -> None: + lexicon = sil_lift.load(DATED) + target = lexicon.entries[0] + before = target.date_modified + target.lexical_unit["en"] = "edited" + lexicon.save(tmp_path / "out.lift", stamp=False) + + assert target.date_modified == before + assert lexicon._stamps == {} # a save that stamps nothing remembers nothing + + +def test_a_date_the_caller_set_is_left_alone(tmp_path: Path) -> None: + """Content and date both moved, so the date is the caller's, not a stale one.""" + lexicon = sil_lift.load(DATED) + target = lexicon.entries[0] + target.lexical_unit["en"] = "edited" + target.date_modified = BY_HAND + lexicon.save(tmp_path / "out.lift", when=WHEN) + + assert target.date_modified == BY_HAND + + +def test_sorting_alone_stamps_nothing(tmp_path: Path) -> None: + """Matches the guarantee on Lexicon.sort: reordering leaves entry bytes alone.""" + lexicon = sil_lift.load(DATED) + before = _dates(lexicon) + lexicon.sort() + lexicon.save(tmp_path / "out.lift", when=WHEN) + + assert _dates(lexicon) == before + + +def test_a_second_round_of_edits_on_the_same_lexicon_is_stamped_too(tmp_path: Path) -> None: + """The baseline moves with each save; without that, only the first edit would bump. + + After the first save the entry's date differs from the loaded one — this + library's own stamp — which is indistinguishable from a caller-set date + unless the save records what it wrote. + """ + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + out = tmp_path / "out.lift" + + entry.lexical_unit["en"] = "first" + lexicon.save(out, when=WHEN) + assert entry.date_modified == WHEN + + entry.lexical_unit["en"] = "second" + lexicon.save(out, when=LATER) + assert entry.date_modified == LATER + + +def test_a_save_with_no_intervening_edits_leaves_the_stamp_alone(tmp_path: Path) -> None: + """Stamping twice for one edit would make every save a change downstream.""" + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + entry.lexical_unit["en"] = "edited" + first = tmp_path / "first.lift" + lexicon.save(first, when=WHEN) + + second = tmp_path / "second.lift" + lexicon.save(second, when=LATER) + + assert entry.date_modified == WHEN + assert second.read_bytes() == first.read_bytes() + + +def test_a_hand_set_date_becomes_the_baseline_for_the_next_edit(tmp_path: Path) -> None: + """Deliberate for the save it was set for, stale once the content moves again.""" + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + out = tmp_path / "out.lift" + + entry.lexical_unit["en"] = "edited" + entry.date_modified = BY_HAND + lexicon.save(out, when=WHEN) + assert entry.date_modified == BY_HAND + + entry.lexical_unit["en"] = "edited again" + lexicon.save(out, when=LATER) + assert entry.date_modified == LATER + + +def test_an_entry_added_after_load_is_stamped_only_when_its_date_is_blank( + tmp_path: Path, +) -> None: + """A new entry has nothing to compare against, so a date on it is taken as meant.""" + lexicon = sil_lift.load(UNDATED) + blank = sil_lift.Entry(id="blank") + carried = sil_lift.Entry(id="carried", date_created=BY_HAND, date_modified=BY_HAND) + lexicon.entries += [blank, carried] + out = tmp_path / "out.lift" + lexicon.save(out, when=WHEN) + + assert (blank.date_created, blank.date_modified) == (WHEN, WHEN) + assert (carried.date_created, carried.date_modified) == (BY_HAND, BY_HAND) + + # Its first save recorded a baseline, so the carried date is not frozen for good. + carried.lexical_unit["en"] = "edited" + lexicon.save(out, when=LATER) + assert carried.date_modified == LATER + assert carried.date_created == BY_HAND + + +def test_a_from_scratch_lexicon_is_stamped_on_its_first_save(tmp_path: Path) -> None: + """The build-an-export case: no baseline anywhere, and every entry is new.""" + entry = sil_lift.Entry(id="kanga") + entry.lexical_unit["seh"] = "nkhuku" + lexicon = sil_lift.Lexicon(entries=[entry]) + out = tmp_path / "out.lift" + lexicon.save(out, when=WHEN) + + assert (entry.date_created, entry.date_modified) == (WHEN, WHEN) + + # Re-saving an unedited from-scratch lexicon is not a modification either. + lexicon.save(out, when=LATER) + assert entry.date_modified == WHEN + + entry.lexical_unit["seh"] = "nkhukhu" + lexicon.save(out, when=LATER) + assert entry.date_modified == LATER + assert entry.date_created == WHEN + + +def test_stamping_does_not_reach_below_the_entry(tmp_path: Path) -> None: + """Entry-level only: no sub-entry node is stamped, whatever moved inside it.""" + lexicon = sil_lift.load(UNDATED) + entry = lexicon.entries[0] + sense = entry.senses[0] + sense.date_modified = BY_HAND + sense.subsenses[0].glosses[0].text = sil_lift.Text(["edited"]) + lexicon.save(tmp_path / "out.lift", when=WHEN) + + assert entry.date_modified == WHEN + assert sense.date_modified == BY_HAND + assert sense.date_created is None + assert sense.subsenses[0].date_modified is None + + +def test_the_default_clock_is_utc_at_seconds_precision(tmp_path: Path) -> None: + """No `when`: the wall clock, in the 20-character form real exports use.""" + lexicon = sil_lift.load(UNDATED) + entry = lexicon.entries[0] + entry.lexical_unit["en"] = "edited" + before = datetime.now(UTC).replace(microsecond=0) + out = tmp_path / "out.lift" + lexicon.save(out) + after = datetime.now(UTC) + + stamped = entry.date_modified + assert isinstance(stamped, datetime) + assert stamped.utcoffset() == timedelta(0) + assert stamped.microsecond == 0 + assert before <= stamped <= after + rendered = f'dateModified="{stamped.strftime("%Y-%m-%dT%H:%M:%SZ")}"'.encode() + assert rendered in out.read_bytes() + + +def test_a_stamped_save_is_byte_reproducible_given_when(tmp_path: Path) -> None: + """`when` is what keeps a stamping pipeline diffable: same input, same bytes.""" + outputs = [] + for name in ("first.lift", "second.lift"): + lexicon = sil_lift.load(UNDATED) + lexicon.entries[0].senses[0].subsenses[0].glosses[0].text = sil_lift.Text(["edited"]) + out = tmp_path / name + lexicon.save(out, when=WHEN) + outputs.append(out.read_bytes()) + + assert outputs[0] == outputs[1] + + +def test_validation_does_not_stamp(tmp_path: Path) -> None: + """iter_problems is read-only, so the bytes it reports on precede any stamp.""" + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + before = entry.date_modified + entry.lexical_unit["en"] = "edited" + + assert [problem for problem in lexicon.iter_problems() if problem.level == "error"] == [] + assert entry.date_modified == before + + lexicon.save(tmp_path / "out.lift", when=WHEN) + assert entry.date_modified == WHEN + + +def test_changed_entries_still_reports_a_stamped_entry(tmp_path: Path) -> None: + """The stamping baseline is its own: change detection still answers "since load".""" + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + entry.lexical_unit["en"] = "edited" + lexicon.save(tmp_path / "out.lift", when=WHEN) + + assert [id(reported) for reported in lexicon.changed_entries()] == [id(entry)] + assert lexicon.changes() + + +def _in_its_own_folder(fixture: Path, tmp_path: Path) -> Path: + """A copy of a fixture on its own: save_zip packages its whole folder.""" + folder = tmp_path / "src" + folder.mkdir() + dest = folder / fixture.name + dest.write_bytes(fixture.read_bytes()) + return dest + + +def test_stamping_replaces_a_date_the_model_could_not_hold(tmp_path: Path) -> None: + """A generated stamp wins over an unparseable date, which is only residue. + + A model field always beats stale residue in the writer, and that is the + right way round here: the entry is being rewritten, and "whenever" is no + date at all, so an edit leaves it with one that is. `stamp=False` is what + preserves the original strings. + """ + source = tmp_path / "junk.lift" + source.write_bytes(UNPARSEABLE_DATES) + lexicon = sil_lift.load(source) + entry = lexicon.entries[0] + assert (entry.date_created, entry.date_modified) == (None, None) + + entry.lexical_unit["en"] = "edited" + out = tmp_path / "out.lift" + lexicon.save(out, when=WHEN) + written = out.read_bytes() + assert b'dateCreated="2026-03-04T05:06:07Z" dateModified="2026-03-04T05:06:07Z"' in written + assert b"whenever" not in written + + kept = sil_lift.load(source) + kept.entries[0].lexical_unit["en"] = "edited" + unstamped = tmp_path / "unstamped.lift" + kept.save(unstamped, stamp=False) + assert b'dateCreated="nope" dateModified="whenever"' in unstamped.read_bytes() + + +def test_save_zip_stamps_by_default(tmp_path: Path) -> None: + """A package is the hand-off to the tools that reconcile on dateModified.""" + lexicon = sil_lift.load(_in_its_own_folder(UNDATED, tmp_path)) + entry = lexicon.entries[0] + entry.senses[0].subsenses[0].glosses[0].text = sil_lift.Text(["edited"]) + dest = tmp_path / "pkg.zip" + lexicon.save_zip(dest, when=WHEN) + + assert entry.date_modified == WHEN + with zipfile.ZipFile(dest) as archive: + member = next(name for name in archive.namelist() if name.endswith(".lift")) + assert b'dateModified="2026-03-04T05:06:07Z"' in archive.read(member) + + +def test_save_zip_stamp_false_leaves_the_model_alone(tmp_path: Path) -> None: + lexicon = sil_lift.load(_in_its_own_folder(UNDATED, tmp_path)) + entry = lexicon.entries[0] + entry.senses[0].subsenses[0].glosses[0].text = sil_lift.Text(["edited"]) + lexicon.save_zip(tmp_path / "pkg.zip", stamp=False) + + assert entry.date_modified is None From cd57c06a0326f76503e92d75bd6bea25a5761da1 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 2 Sep 2026 16:00:37 -0400 Subject: [PATCH 2/4] Date only what changed, and commit stamps with the write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five corrections to entry stamping. A document the byte scanner declined had no stamping baseline at all, so every undated entry read as new and a save that changed nothing dated all of them. Byte reuse needs the source bytes but stamping needs only the digests, so the reader now records digests and dates for such a document too, and a no-op save of one leaves it alone. when= is normalized to UTC at whole seconds, so an explicit moment reaches the output in the one form the rest of it uses instead of carrying an offset or fractional seconds through _fmt_date. A naive value is refused: read as UTC and read as local time it names moments hours apart, and picking one silently writes a date the caller did not mean. Stamping now commits with the write. stamp_entries returns what undoes it, save() and save_zip() put the dates and the baseline back when the write does not go through, and the pass decides before it mutates so that the one step that can fail — digesting content XML cannot represent — refuses with nothing stamped rather than half-stamped at whatever entry the refusal came from. The baseline dict is rebuilt each pass rather than updated in place, so an entry appended and later removed is no longer held alive, with its whole subtree, by a record nothing will consult again. Entries the document was loaded with are retained by their parse-time records as before. Entries are iterated by identity, so an entry aliased into the list twice is decided and stamped once — it is one object with one pair of dates, whatever its output happens to say twice. default_now() documents what seconds precision costs: one second holds one date, so a second edit saved inside the same second as the first carries the same stamp. Sub-second precision would buy the distinction at the cost of the form every consumer expects, and when= forces a distinct moment. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 5 +- docs/en/fidelity.md | 3 +- docs/en/guides/build-export.md | 2 +- docs/en/guides/bulk-edit-glosses.md | 2 +- src/sil_lift/_model.py | 57 ++++++--- src/sil_lift/_reader.py | 23 ++++ src/sil_lift/_writer.py | 98 +++++++++++++--- tests/test_stamp.py | 174 ++++++++++++++++++++++++++-- 8 files changed, 315 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fdb6c6..cae0ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,8 +44,9 @@ releases may contain breaking changes. whose date the caller set deliberately is left alone, as is an entry created since the load that already carries one, and reordering stamps nothing. `stamp=False` writes the model exactly as it stands, and `when=` supplies the - moment in place of the wall clock (UTC at seconds precision), which is what - keeps stamped output byte-reproducible. + moment in place of the wall clock — a timezone-aware value, normalized to UTC + whole seconds — which is what keeps stamped output byte-reproducible. The + stamps commit with the write: a refused or failed one puts the dates back. - Change detection against the loaded document, reading the same parse-time digests. `Lexicon.changed_entries()` reports entries whose content differs (an entry's digest covers its whole subtree, so an edit at any depth reports diff --git a/docs/en/fidelity.md b/docs/en/fidelity.md index de748b6..11a5001 100644 --- a/docs/en/fidelity.md +++ b/docs/en/fidelity.md @@ -23,7 +23,8 @@ Exceptions (the writer falls back to full canonical serialization, which is sema - **Touched entries are re-serialized canonically and completely**: UTF-8, 2-space indentation _outside_ mixed content (whitespace inside `` and `` is never altered), a documented child grouping per element (e.g. entry: lexical-unit, citation, pronunciations, variants, senses, notes, relations, etymologies, annotations, traits, fields), fixed attribute order, dates in ISO-8601 (`Z` for UTC). All residue is re-emitted; its position is restored to the original child index, clamped to the new child list (an approximation — exact byte positions are only guaranteed for untouched entries). - Adding, removing, or reordering entries re-serializes the document structure but still emits every unchanged entry's bytes verbatim. - **A touched entry is stamped.** `save()` writes a fresh `dateModified` on every entry whose content changed since it was read, and fills a blank `dateCreated` with the same moment — an edit shipped under its loaded date looks unmodified to everything that reconciles on that attribute, FieldWorks and The Combine's LIFT import included. Only `` is stamped: no node below one, and nothing in the header. An entry whose date the caller set deliberately keeps it, and so does an entry created since the load that already carries one. A date the model could not parse is [residue](#reading) rather than a date, so a stamp replaces it and the original string is dropped — an edited entry is better off carrying a real date than `dateModified="whenever"`. `save(stamp=False)` writes the model exactly as it stands, residue included. -- **A generated stamp is the one thing in the output that is not a function of the input.** Stamps are UTC at seconds precision (`YYYY-MM-DDTHH:MM:SSZ` — the shape every surveyed FieldWorks export uses), read from the wall clock. `save(when=...)` supplies the moment instead, which is what keeps stamped output reproducible for a diff-based CI gate. +- **A generated stamp is the one thing in the output that is not a function of the input.** Stamps are UTC at seconds precision (`YYYY-MM-DDTHH:MM:SSZ` — the shape every surveyed FieldWorks export uses), read from the wall clock. `save(when=...)` supplies the moment instead, which is what keeps stamped output reproducible for a diff-based CI gate; it must be timezone-aware, and is normalized to UTC whole seconds so an explicit moment lands in that same form. One second holds one date, so a second edit saved inside the same second as the first carries the same stamp — the baseline still tracks it, but nothing comparing dates can see it. +- **Stamping commits with the write.** A refused or failed write puts the dates back, so the model never carries a modification date for output that does not exist. `iter_problems()` never stamps at all. !!! note ""Canonical" here is not related to any other Canonical XML" Canonical form on this page means `sil-lift`'s own documented shape, described in a bullet above. It is unrelated to W3C's Canonical XML (C14N) process. It is unrelated to `SIL.Core`'s `CanonicalXmlSettings` class. diff --git a/docs/en/guides/build-export.md b/docs/en/guides/build-export.md index b073def..cbc8e9a 100644 --- a/docs/en/guides/build-export.md +++ b/docs/en/guides/build-export.md @@ -161,7 +161,7 @@ print((out / "birds.lift-ranges").read_text(encoding="utf-8"), end="") - A `URLRef` is an href plus an optional caption/label multitext — used for both `` (audio) and `` (photos). The pronunciation here follows The Combine's convention of an `en` form reading `Speaker: `. - App-specific data with no native LIFT home rides as a `` (or ``): FieldWorks reads these as custom fields and The Combine preserves them. - Give every entry a real, stable `guid` (e.g. from `uuid.uuid4()`, reused across exports) — a later re-import updates the entry in place rather than duplicating it. `sil-lift validate --require-ids` enforces this. -- The `dateCreated`/`dateModified` in the output above are not in the script: `save()` stamped them with the moment it ran, because an entry it is writing for the first time carries no date of its own and the tools that import LIFT decide what to update from `dateModified`. Two knobs, both on `save()`: `when=` supplies the moment instead of reading the clock — that is what makes a generated export byte-reproducible, so a CI job can diff it — and `stamp=False` writes no dates at all. A date you set yourself is left alone either way, so an exporter carrying real timestamps over from its own data model keeps them. Nothing below `` is ever stamped. +- The `dateCreated`/`dateModified` in the output above are not in the script: `save()` stamped them with the moment it ran, because an entry it is writing for the first time carries no date of its own and the tools that import LIFT decide what to update from `dateModified`. Two knobs, both on `save()`: `when=` supplies the moment instead of reading the clock — a timezone-aware `datetime`, normalized to UTC whole seconds, which is what makes a generated export byte-reproducible so a CI job can diff it — and `stamp=False` writes no dates at all. A date you set yourself is left alone either way, so an exporter carrying real timestamps over from its own data model keeps them. Nothing below `` is ever stamped. - `lex.iter_problems()` validates the in-memory document before anything hits disk; here it is clean. Because the lexicon has no folder yet, the media-presence and companion-href checks are skipped — run [`sil-lift validate`](cli.md) on the saved output (or with `--no-check-media`) once the audio and photo files are in place. ## Packaging diff --git a/docs/en/guides/bulk-edit-glosses.md b/docs/en/guides/bulk-edit-glosses.md index 63ab988..ade36d5 100644 --- a/docs/en/guides/bulk-edit-glosses.md +++ b/docs/en/guides/bulk-edit-glosses.md @@ -59,7 +59,7 @@ A few things worth noting: - It compares content, not destination, so guard only an in-place save with it: `lex.save(some_other_dir / "dictionary.lift")` writes the document and its companions to a location that has nothing in it yet, whether or not anything changed. - It is a guard, not a speed-up — answering it digests every entry, which is the same work `save()` does to decide which source bytes it can reuse, so what you skip is the write itself (an unchanged file-modification time, no spurious diff), not the effort of deciding. - Validating in memory (`lex.iter_problems()`) serializes the edited state first, so it correctly reflects the edit before anything is written to disk. Aborting on any `"error"`-level `Problem` — warnings are left for the caller to decide about — means a bad edit never reaches `save()`. -- `lex.save()` stamps a fresh `dateModified` on exactly those changed entries as it writes them, filling `dateCreated` where it was blank. Without that the edited entries would go out under the dates they were loaded with, and a lexicon this script has rewritten would look untouched to FieldWorks or to The Combine's LIFT import, both of which decide what to update from `dateModified`. It stamps `` only, however deep the edit was — the entry containing the edited subsense gloss, not the sense. `save(stamp=False)` writes the dates the model holds; `save(when=...)` supplies the moment rather than reading the clock, so a pipeline that diffs its own output stays reproducible. +- `lex.save()` stamps a fresh `dateModified` on exactly those changed entries as it writes them, filling `dateCreated` where it was blank. Without that the edited entries would go out under the dates they were loaded with, and a lexicon this script has rewritten would look untouched to FieldWorks or to The Combine's LIFT import, both of which decide what to update from `dateModified`. It stamps `` only, however deep the edit was — the entry containing the edited subsense gloss, not the sense. `save(stamp=False)` writes the dates the model holds; `save(when=...)` supplies the moment rather than reading the clock — pass a timezone-aware one — so a pipeline that diffs its own output stays reproducible. The stamps commit with the write: if the save is refused, the dates go back to what they were. Glosses aren't the only thing worth touching this way. The same `Multitext` mapping surface applies to definitions and every other multilingual field on an entry or sense: diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index aed2ab6..c4700ea 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -27,7 +27,7 @@ from typing import Literal from ._validate import Problem - from ._writer import _EntryRecord, _RangesSourceInfo, _SourceInfo + from ._writer import _EntryRecord, _RangesSourceInfo, _SourceInfo, _StampUndo __all__ = [ "Changes", @@ -571,13 +571,17 @@ def _resolve_ranges(self) -> None: if exists and resolved not in self.ranges_files: self.ranges_files[resolved] = RangesFile.load(candidate) - def _apply_stamps(self, stamp: bool, when: datetime | None) -> None: - """The stamping step shared by :meth:`save` and :meth:`save_zip`.""" + def _apply_stamps(self, stamp: bool, when: datetime | None) -> _StampUndo | None: + """The stamping step shared by :meth:`save` and :meth:`save_zip`. + + Returns what undoes the pass, so a write that does not go through can + leave the model as it found it. + """ if not stamp: - return - from ._writer import default_now, stamp_entries + return None + from ._writer import resolve_when, stamp_entries - stamp_entries(self, when if when is not None else default_now()) + return stamp_entries(self, resolve_when(when)) def save( self, @@ -608,14 +612,21 @@ def save( one canonical serialization pass over the entries. ``stamp=False`` writes the model exactly as it stands. ``when`` - supplies the moment in place of the clock (UTC at seconds precision), - which is what makes stamped output byte-reproducible. + supplies the moment in place of the clock, normalized to UTC at seconds + precision, which is what makes stamped output byte-reproducible; it must + be timezone-aware, since a naive moment could as easily mean UTC as + local time. Two stamps of one moment are one moment: a second edit saved + inside the same second as the first carries the same date. + + Stamping commits with the write. A refused or failed write puts the + dates back, so the model never claims a modification that never + reached disk. Raises :class:`ValueError` if no target path is available (none was - passed and the lexicon was not loaded from a file), and - :class:`~sil_lift.LiftWriteError` if the model holds content XML cannot - represent (a lone surrogate) — nothing is written in that case, though - stamps already applied stay on the model. + passed and the lexicon was not loaded from a file) or if ``when`` is + naive, and :class:`~sil_lift.LiftWriteError` if the model holds content + XML cannot represent (a lone surrogate) — nothing is written in that + case, and nothing is stamped. """ from ._writer import render_document @@ -623,8 +634,16 @@ def save( if target is None: raise ValueError("no target path: pass save(path) or load the lexicon from a file") original_dir = self.path.parent if self.path is not None else None - self._apply_stamps(stamp, when) - target.write_bytes(render_document(self)) + undo = self._apply_stamps(stamp, when) + written = False + try: + target.write_bytes(render_document(self)) + written = True + finally: + # The companions below are written after this point; a failure there + # leaves the .lift on disk carrying these stamps, so they stand. + if not written and undo is not None: + undo.restore() self.path = target relocating = not _same_dir(target.parent, original_dir) for key, ranges_file in self.ranges_files.items(): @@ -667,8 +686,14 @@ def save_zip( """ from ._zip import save_zip - self._apply_stamps(stamp, when) - save_zip(self, Path(path), wrap_folder=wrap_folder) + undo = self._apply_stamps(stamp, when) + written = False + try: + save_zip(self, Path(path), wrap_folder=wrap_folder) + written = True + finally: + if not written and undo is not None: + undo.restore() def sort(self) -> None: """Sort into canonical order, in place: entries by (guid, id), header diff --git a/src/sil_lift/_reader.py b/src/sil_lift/_reader.py index 64d025d..441b291 100644 --- a/src/sil_lift/_reader.py +++ b/src/sil_lift/_reader.py @@ -59,6 +59,8 @@ def parse_document(path: Path) -> Lexicon: raise LiftParseError(f"{path}: not well-formed XML: {exc}") from exc lexicon = parse_root(root, path=path) _attach_source(lexicon, data, root) + if lexicon._source is None: + _attach_stamp_baseline(lexicon) return lexicon @@ -108,6 +110,27 @@ def _attach_ranges_source(ranges_file: RangesFile, data: bytes, root: etree._Ele ) +def _attach_stamp_baseline(lexicon: Lexicon) -> None: + """Record what a stamping save measures against, for a document with no snapshot. + + Byte reuse needs the source bytes; stamping needs only the digests, which + are available whether or not the scan was declined. Without this the + save-time pass would find no baseline at all for a document that was read + rather than built, and read every undated entry as new — stamping entries + nobody touched, on a save that changed nothing. + + A lone surrogate is the one thing digesting refuses, and it cannot arrive + from a file (the parser rejects both spellings), so this cannot raise for a + document that just parsed. + """ + from ._writer import _EntryRecord, entry_digest + + lexicon._stamps = { + id(entry): _EntryRecord(entry, entry_digest(entry), entry.date_modified) + for entry in lexicon.entries + } + + def _attach_source(lexicon: Lexicon, data: bytes, root: etree._Element) -> None: """Capture the original bytes needed for byte reuse; on any doubt, capture nothing. diff --git a/src/sil_lift/_writer.py b/src/sil_lift/_writer.py index 08704fc..da82a4d 100644 --- a/src/sil_lift/_writer.py +++ b/src/sil_lift/_writer.py @@ -70,6 +70,7 @@ "range_digest", "render_document", "render_ranges_document", + "resolve_when", "stamp_entries", ] @@ -178,10 +179,38 @@ def default_now() -> datetime: 20-character ``YYYY-MM-DDTHH:MM:SSZ`` form — no bare dates, numeric offsets, or fractional seconds. :func:`_fmt_date` renders an aware UTC value that way. + + Two saves inside the same second therefore write the same value: an edit + saved within a second of the previous one leaves ``dateModified`` where it + already stood, which a consumer comparing dates reads as no change. The + stamping baseline still tracks it, so nothing is lost on this side, and + sub-second precision would buy the distinction at the cost of the one form + every consumer expects. Pass ``when`` to force a distinct moment. """ return datetime.now(UTC).replace(microsecond=0) +def resolve_when(when: datetime | None) -> datetime: + """The moment a stamping save writes: ``when`` normalized, or the clock. + + An explicit value is converted to UTC and truncated to the second, so it + lands in the same form :func:`default_now` produces instead of carrying an + offset or fractional seconds into the output. + + A naive value is refused rather than guessed at: reading it as UTC and + reading it as local time give moments hours apart, and picking one silently + writes a date the caller did not mean. + """ + if when is None: + return default_now() + if when.tzinfo is None or when.tzinfo.utcoffset(when) is None: + raise ValueError( + f"when must be timezone-aware, got {when!r}: pass a UTC moment, " + "e.g. datetime.now(timezone.utc)" + ) + return when.astimezone(UTC).replace(microsecond=0) + + def _needs_stamp(entry: Entry, baseline: _EntryRecord | None, digest: bytes) -> bool: """Whether the content moved since ``baseline`` while the date stayed put. @@ -200,10 +229,30 @@ def _needs_stamp(entry: Entry, baseline: _EntryRecord | None, digest: bytes) -> return digest != baseline.digest and entry.date_modified == baseline.date_modified -def stamp_entries(lexicon: Lexicon, when: datetime) -> None: +@dataclass(slots=True) +class _StampUndo: + """How to put a stamping pass back, for a write that then never happened. + + Stamping runs before serialization, so a refused or failed write would + otherwise leave the model dated for output that does not exist. + """ + + lexicon: Lexicon + stamps: dict[int, _EntryRecord] # the baseline dict the pass replaced + dates: list[tuple[Entry, datetime | date | None, datetime | date | None]] + + def restore(self) -> None: + for entry, created, modified in self.dates: + entry.date_created = created + entry.date_modified = modified + self.lexicon._stamps = self.stamps + + +def stamp_entries(lexicon: Lexicon, when: datetime) -> _StampUndo: """Stamp every entry whose content changed without its ``dateModified`` changing. - Costs one canonical serialization pass over the entries. + Costs one canonical serialization pass over the entries. Returns what undoes + it, for a caller whose write does not go through. Each save leaves behind the state it wrote, in ``lexicon._stamps``, and the next save measures against that rather than against the load. Without it a @@ -211,31 +260,44 @@ def stamp_entries(lexicon: Lexicon, when: datetime) -> None: its content differs from the loaded content all right, but so does its date — this library's own stamp from the first save — which reads exactly like a date the caller set deliberately. An entry still matching its parse-time - record needs no such override and keeps none, so the common save of a - handful of edited entries remembers only those. + record needs no such override and is recorded nowhere, so a save of a + handful of edited entries remembers only those, and an entry that has since + left the lexicon is not carried along by a dict that is rebuilt each pass. + + Deciding comes first and mutating second, because digesting is the only + step that can fail (see :func:`_guarded`): a document holding content XML + cannot represent is refused with nothing stamped, rather than half-stamped + at whatever entry the refusal came from. """ source = lexicon._source at_parse = ( {id(record.entry): record for record in source.entry_records} if source is not None else {} ) - for entry in lexicon.entries: - key = id(entry) - record = at_parse.get(key) - baseline = lexicon._stamps.get(key) + previous = lexicon._stamps + # Keyed by identity: an entry aliased into the list twice is one entry, with + # one pair of dates, so it is decided and stamped once. + entries = list({id(entry): entry for entry in lexicon.entries}.values()) + + planned: list[tuple[Entry, bytes, bool]] = [] + for entry in entries: + baseline = previous.get(id(entry)) if baseline is None: - baseline = record + baseline = at_parse.get(id(entry)) digest = entry_digest(entry) - if _needs_stamp(entry, baseline, digest): + planned.append((entry, digest, _needs_stamp(entry, baseline, digest))) + + dates: list[tuple[Entry, datetime | date | None, datetime | date | None]] = [] + stamps: dict[int, _EntryRecord] = {} + for entry, digest, needs_stamp in planned: + if needs_stamp: + dates.append((entry, entry.date_created, entry.date_modified)) entry._stamp(when) digest = entry_digest(entry) # the dates are part of an entry's bytes - if ( - record is not None - and record.digest == digest - and record.date_modified == entry.date_modified - ): - lexicon._stamps.pop(key, None) - else: - lexicon._stamps[key] = _EntryRecord(entry, digest, entry.date_modified) + record = at_parse.get(id(entry)) + if record is None or record.digest != digest or record.date_modified != entry.date_modified: + stamps[id(entry)] = _EntryRecord(entry, digest, entry.date_modified) + lexicon._stamps = stamps + return _StampUndo(lexicon, previous, dates) # --- canonical building blocks --------------------------------------------------- diff --git a/tests/test_stamp.py b/tests/test_stamp.py index 4b2cd5d..ee2ae9e 100644 --- a/tests/test_stamp.py +++ b/tests/test_stamp.py @@ -7,9 +7,11 @@ """ import zipfile -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime, timedelta, timezone from pathlib import Path +import pytest + import sil_lift CORPUS_DIR = Path(__file__).parent / "corpus" @@ -35,6 +37,15 @@ def _dates(lexicon: sil_lift.Lexicon) -> dict[str | None, tuple[object, object]] return {entry.id: (entry.date_created, entry.date_modified) for entry in lexicon.entries} +def _in_its_own_folder(fixture: Path, tmp_path: Path) -> Path: + """A copy of a fixture on its own: save_zip packages its whole folder.""" + folder = tmp_path / "src" + folder.mkdir() + dest = folder / fixture.name + dest.write_bytes(fixture.read_bytes()) + return dest + + def test_an_untouched_save_stamps_nothing_and_stays_byte_identical(tmp_path: Path) -> None: """Stamping is driven by content, so a load-and-save writes the source bytes back.""" lexicon = sil_lift.load(DATED) @@ -228,6 +239,158 @@ def test_stamping_does_not_reach_below_the_entry(tmp_path: Path) -> None: assert sense.subsenses[0].date_modified is None +def test_an_unscannable_document_stamps_only_what_changed(tmp_path: Path) -> None: + """No byte snapshot, but the digests still date the edit and nothing else. + + The scanner declines a source it cannot read, and stamping needs only the + digests, so the reader records a baseline anyway. Without one every undated + entry would look new and a save that changed nothing would stamp them all. + + Note what `changed_entries()` says here by contrast: every entry, because + `save()` does re-serialize the whole file. Stamping asks the narrower + question — what the caller modified — and answers it exactly. + """ + text = UNDATED.read_text(encoding="utf-8").replace('encoding="UTF-8"', 'encoding="UTF-16"') + source = tmp_path / "utf16.lift" + source.write_bytes(text.encode("utf-16")) + lexicon = sil_lift.load(source) + assert lexicon._source is None # no byte baseline, only a stamping one + entry = lexicon.entries[0] + out = tmp_path / "out.lift" + + lexicon.save(out, when=WHEN) + assert (entry.date_created, entry.date_modified) == (None, None) + assert len(lexicon.changed_entries()) == len(lexicon.entries) + + entry.senses[0].subsenses[0].glosses[0].text = sil_lift.Text(["edited"]) + lexicon.save(out, when=WHEN) + assert (entry.date_created, entry.date_modified) == (WHEN, WHEN) + + +def test_when_is_normalized_to_utc_at_seconds_precision(tmp_path: Path) -> None: + """Whatever shape the caller's moment is in, the output keeps the one form.""" + lexicon = sil_lift.load(UNDATED) + entry = lexicon.entries[0] + entry.lexical_unit["en"] = "edited" + out = tmp_path / "out.lift" + lexicon.save(out, when=datetime(2026, 3, 4, 5, 6, 7, 500000, tzinfo=UTC)) + + assert entry.date_modified == WHEN # the fraction is dropped, not rounded + assert b'dateModified="2026-03-04T05:06:07Z"' in out.read_bytes() + + entry.lexical_unit["en"] = "edited again" + offset = timezone(timedelta(hours=5, minutes=30)) + lexicon.save(out, when=datetime(2026, 3, 4, 10, 36, 8, tzinfo=offset)) + + assert entry.date_modified == LATER # the same moment, said in UTC + assert b'dateModified="2026-03-04T05:06:08Z"' in out.read_bytes() + + +def test_a_naive_when_is_refused_and_nothing_is_written(tmp_path: Path) -> None: + """UTC or local would put the stamp hours apart, so neither is assumed.""" + lexicon = sil_lift.load(UNDATED) + entry = lexicon.entries[0] + entry.lexical_unit["en"] = "edited" + out = tmp_path / "out.lift" + + with pytest.raises(ValueError, match="timezone-aware"): + lexicon.save(out, when=datetime(2026, 3, 4, 5, 6, 7)) + + assert entry.date_modified is None + assert not out.exists() + + +def test_a_refused_write_leaves_nothing_stamped(tmp_path: Path) -> None: + """Stamping commits with the write: no file, no modification date. + + The refusal comes from the last entry, so this also pins that the entries + ahead of it are not left half-stamped — deciding happens before mutating. + """ + lexicon = sil_lift.load(DATED) + edited, offending = lexicon.entries[0], lexicon.entries[-1] + before = _dates(lexicon) + baseline = dict(lexicon._stamps) + edited.lexical_unit["en"] = "edited" + offending.senses[0].glosses.append(sil_lift.Form(lang="en", text=sil_lift.Text(["\ud800"]))) + out = tmp_path / "out.lift" + + with pytest.raises(sil_lift.LiftWriteError): + lexicon.save(out, when=WHEN) + + assert _dates(lexicon) == before + assert lexicon._stamps == baseline + assert not out.exists() + + +def test_a_refused_zip_write_leaves_nothing_stamped(tmp_path: Path) -> None: + lexicon = sil_lift.load(_in_its_own_folder(UNDATED, tmp_path)) + entry = lexicon.entries[0] + entry.lexical_unit["en"] = "edited" + entry.senses[0].glosses.append(sil_lift.Form(lang="en", text=sil_lift.Text(["\ud800"]))) + + with pytest.raises(sil_lift.LiftWriteError): + lexicon.save_zip(tmp_path / "pkg.zip", when=WHEN) + + assert entry.date_modified is None + + +def test_a_removed_entry_drops_out_of_the_stamping_baseline(tmp_path: Path) -> None: + """The baseline dict is rebuilt each save, so it holds no entry the lexicon lost. + + An entry the document was loaded with is retained by its parse-time record + either way (that is what makes `removed_entries()` work); one appended and + then dropped would otherwise be kept alive here, subtree and all, by a + baseline nothing will ever consult again. + """ + lexicon = sil_lift.load(UNDATED) + appended = sil_lift.Entry(id="temporary") + lexicon.entries.append(appended) + out = tmp_path / "out.lift" + lexicon.save(out, when=WHEN) + assert [record.entry.id for record in lexicon._stamps.values()] == ["temporary"] + + lexicon.entries.remove(appended) + lexicon.save(out, when=LATER) + + assert lexicon._stamps == {} + + +def test_an_aliased_entry_is_stamped_once(tmp_path: Path) -> None: + """One entry object, one pair of dates — however many list slots point at it.""" + lexicon = sil_lift.load(UNDATED) + entry = lexicon.entries[0] + lexicon.entries.append(entry) # the same object, written out twice + entry.lexical_unit["en"] = "edited" + out = tmp_path / "out.lift" + lexicon.save(out, when=WHEN) + + assert entry.date_modified == WHEN + assert len(lexicon._stamps) == 1 + assert out.read_bytes().count(b'dateModified="2026-03-04T05:06:07Z"') == 2 + + +def test_two_saves_of_one_moment_share_a_stamp(tmp_path: Path) -> None: + """The documented limit of seconds precision, said with an explicit moment. + + The same second cannot hold two distinct dates, so a second edit saved + within one reads as no change to anything comparing dates. The stamping + baseline still tracks it, so the next distinct moment stamps normally. + """ + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + out = tmp_path / "out.lift" + + entry.lexical_unit["en"] = "one" + lexicon.save(out, when=WHEN) + entry.lexical_unit["en"] = "two" + lexicon.save(out, when=WHEN) + assert entry.date_modified == WHEN + + entry.lexical_unit["en"] = "three" + lexicon.save(out, when=LATER) + assert entry.date_modified == LATER + + def test_the_default_clock_is_utc_at_seconds_precision(tmp_path: Path) -> None: """No `when`: the wall clock, in the 20-character form real exports use.""" lexicon = sil_lift.load(UNDATED) @@ -285,15 +448,6 @@ def test_changed_entries_still_reports_a_stamped_entry(tmp_path: Path) -> None: assert lexicon.changes() -def _in_its_own_folder(fixture: Path, tmp_path: Path) -> Path: - """A copy of a fixture on its own: save_zip packages its whole folder.""" - folder = tmp_path / "src" - folder.mkdir() - dest = folder / fixture.name - dest.write_bytes(fixture.read_bytes()) - return dest - - def test_stamping_replaces_a_date_the_model_could_not_hold(tmp_path: Path) -> None: """A generated stamp wins over an unparseable date, which is only residue. From 49efb2b9e36cd8cf20cfbdb27481a55c5875f787 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 2 Sep 2026 16:03:27 -0400 Subject: [PATCH 3/4] Say less about stamping in more places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal vocabulary is out of the guides: a reader has no use for the word baseline, and the streaming note now says what the limitation is — a pass that never saw the document cannot tell which entries changed — rather than naming the machinery it lacks. Refused-write behavior is stated where it is a contract, in fidelity.md and on save(), and no longer repeated in a guide whose script aborts before it saves. The comments on the clock, the undo, the identity keying and the two-phase pass keep their reasons and drop the reassurance around them. Co-Authored-By: Claude Opus 5 (1M context) --- docs/en/fidelity.md | 4 ++-- docs/en/guides/bulk-edit-glosses.md | 2 +- docs/en/guides/large-files.md | 2 +- src/sil_lift/_model.py | 13 +++++------ src/sil_lift/_reader.py | 15 ++++++------ src/sil_lift/_writer.py | 36 ++++++++++++++--------------- tests/test_stamp.py | 9 ++------ 7 files changed, 36 insertions(+), 45 deletions(-) diff --git a/docs/en/fidelity.md b/docs/en/fidelity.md index 11a5001..fbbe01d 100644 --- a/docs/en/fidelity.md +++ b/docs/en/fidelity.md @@ -23,8 +23,8 @@ Exceptions (the writer falls back to full canonical serialization, which is sema - **Touched entries are re-serialized canonically and completely**: UTF-8, 2-space indentation _outside_ mixed content (whitespace inside `` and `` is never altered), a documented child grouping per element (e.g. entry: lexical-unit, citation, pronunciations, variants, senses, notes, relations, etymologies, annotations, traits, fields), fixed attribute order, dates in ISO-8601 (`Z` for UTC). All residue is re-emitted; its position is restored to the original child index, clamped to the new child list (an approximation — exact byte positions are only guaranteed for untouched entries). - Adding, removing, or reordering entries re-serializes the document structure but still emits every unchanged entry's bytes verbatim. - **A touched entry is stamped.** `save()` writes a fresh `dateModified` on every entry whose content changed since it was read, and fills a blank `dateCreated` with the same moment — an edit shipped under its loaded date looks unmodified to everything that reconciles on that attribute, FieldWorks and The Combine's LIFT import included. Only `` is stamped: no node below one, and nothing in the header. An entry whose date the caller set deliberately keeps it, and so does an entry created since the load that already carries one. A date the model could not parse is [residue](#reading) rather than a date, so a stamp replaces it and the original string is dropped — an edited entry is better off carrying a real date than `dateModified="whenever"`. `save(stamp=False)` writes the model exactly as it stands, residue included. -- **A generated stamp is the one thing in the output that is not a function of the input.** Stamps are UTC at seconds precision (`YYYY-MM-DDTHH:MM:SSZ` — the shape every surveyed FieldWorks export uses), read from the wall clock. `save(when=...)` supplies the moment instead, which is what keeps stamped output reproducible for a diff-based CI gate; it must be timezone-aware, and is normalized to UTC whole seconds so an explicit moment lands in that same form. One second holds one date, so a second edit saved inside the same second as the first carries the same stamp — the baseline still tracks it, but nothing comparing dates can see it. -- **Stamping commits with the write.** A refused or failed write puts the dates back, so the model never carries a modification date for output that does not exist. `iter_problems()` never stamps at all. +- **A generated stamp is the one thing in the output that is not a function of the input.** Stamps are UTC at seconds precision (`YYYY-MM-DDTHH:MM:SSZ` — the shape every surveyed FieldWorks export uses), read from the wall clock. `save(when=...)` supplies the moment instead, which is what keeps stamped output reproducible for a diff-based CI gate; it must be timezone-aware, and is normalized to UTC whole seconds so an explicit moment lands in that same form. One second holds one date, so an edit saved within a second of the previous one carries the same stamp. +- **Stamping commits with the write.** A refused or failed write puts the dates back, so the model never carries a modification date for output that does not exist. !!! note ""Canonical" here is not related to any other Canonical XML" Canonical form on this page means `sil-lift`'s own documented shape, described in a bullet above. It is unrelated to W3C's Canonical XML (C14N) process. It is unrelated to `SIL.Core`'s `CanonicalXmlSettings` class. diff --git a/docs/en/guides/bulk-edit-glosses.md b/docs/en/guides/bulk-edit-glosses.md index ade36d5..5c31ab5 100644 --- a/docs/en/guides/bulk-edit-glosses.md +++ b/docs/en/guides/bulk-edit-glosses.md @@ -59,7 +59,7 @@ A few things worth noting: - It compares content, not destination, so guard only an in-place save with it: `lex.save(some_other_dir / "dictionary.lift")` writes the document and its companions to a location that has nothing in it yet, whether or not anything changed. - It is a guard, not a speed-up — answering it digests every entry, which is the same work `save()` does to decide which source bytes it can reuse, so what you skip is the write itself (an unchanged file-modification time, no spurious diff), not the effort of deciding. - Validating in memory (`lex.iter_problems()`) serializes the edited state first, so it correctly reflects the edit before anything is written to disk. Aborting on any `"error"`-level `Problem` — warnings are left for the caller to decide about — means a bad edit never reaches `save()`. -- `lex.save()` stamps a fresh `dateModified` on exactly those changed entries as it writes them, filling `dateCreated` where it was blank. Without that the edited entries would go out under the dates they were loaded with, and a lexicon this script has rewritten would look untouched to FieldWorks or to The Combine's LIFT import, both of which decide what to update from `dateModified`. It stamps `` only, however deep the edit was — the entry containing the edited subsense gloss, not the sense. `save(stamp=False)` writes the dates the model holds; `save(when=...)` supplies the moment rather than reading the clock — pass a timezone-aware one — so a pipeline that diffs its own output stays reproducible. The stamps commit with the write: if the save is refused, the dates go back to what they were. +- `lex.save()` stamps a fresh `dateModified` on exactly those changed entries as it writes them, filling `dateCreated` where it was blank. Without that the edited entries would go out under the dates they were loaded with, and a lexicon this script has rewritten would look untouched to FieldWorks or to The Combine's LIFT import, both of which decide what to update from `dateModified`. It stamps `` only, however deep the edit was — the entry containing the edited subsense gloss, not the sense. `save(stamp=False)` writes the dates the model holds; `save(when=...)` supplies the moment rather than reading the clock — pass a timezone-aware one — so a pipeline that diffs its own output stays reproducible. Glosses aren't the only thing worth touching this way. The same `Multitext` mapping surface applies to definitions and every other multilingual field on an entry or sense: diff --git a/docs/en/guides/large-files.md b/docs/en/guides/large-files.md index fe243c1..c2dc252 100644 --- a/docs/en/guides/large-files.md +++ b/docs/en/guides/large-files.md @@ -24,5 +24,5 @@ Notes: - The writer's output is exactly what the full-document canonical serializer would produce for the same content — the two modes never drift apart. - Streaming mode reuses no source bytes: output is always canonical. Root-level LIFT residue — comments between entries and out-of-schema attributes on `` — is not carried; entries and the header are complete, residue included. -- Nor does it generate timestamps. An entry is written with the dates it carries, since a streaming writer has no loaded document to compare it against — the stamping [`Lexicon.save()`](../fidelity.md#saving-an-edited-document) does needs that baseline. Set `entry.date_modified` yourself on the entries this pass rewrites. +- Nor does it generate timestamps, as [`Lexicon.save()`](../fidelity.md#saving-an-edited-document) does. An entry is written with the dates it carries: a streaming pass never sees the document as it was, so it cannot tell which entries you changed. Set `entry.date_modified` yourself on the ones this pass rewrites. - If the body of an `open_writer` block raises, the file is left visibly unterminated (no closing ``) — a half-written lexicon must not look complete. diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index c4700ea..3f743aa 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -615,12 +615,11 @@ def save( supplies the moment in place of the clock, normalized to UTC at seconds precision, which is what makes stamped output byte-reproducible; it must be timezone-aware, since a naive moment could as easily mean UTC as - local time. Two stamps of one moment are one moment: a second edit saved - inside the same second as the first carries the same date. + local time. Seconds are the resolution, so an edit saved within a second + of the previous one carries the same date. - Stamping commits with the write. A refused or failed write puts the - dates back, so the model never claims a modification that never - reached disk. + Stamping commits with the write: a refused or failed one puts the dates + back, so the model never carries a date for output that does not exist. Raises :class:`ValueError` if no target path is available (none was passed and the lexicon was not loaded from a file) or if ``when`` is @@ -640,8 +639,8 @@ def save( target.write_bytes(render_document(self)) written = True finally: - # The companions below are written after this point; a failure there - # leaves the .lift on disk carrying these stamps, so they stand. + # A companion failing further down leaves the .lift on disk carrying + # these stamps, so from here on they stand. if not written and undo is not None: undo.restore() self.path = target diff --git a/src/sil_lift/_reader.py b/src/sil_lift/_reader.py index 441b291..96b9aea 100644 --- a/src/sil_lift/_reader.py +++ b/src/sil_lift/_reader.py @@ -114,14 +114,13 @@ def _attach_stamp_baseline(lexicon: Lexicon) -> None: """Record what a stamping save measures against, for a document with no snapshot. Byte reuse needs the source bytes; stamping needs only the digests, which - are available whether or not the scan was declined. Without this the - save-time pass would find no baseline at all for a document that was read - rather than built, and read every undated entry as new — stamping entries - nobody touched, on a save that changed nothing. - - A lone surrogate is the one thing digesting refuses, and it cannot arrive - from a file (the parser rejects both spellings), so this cannot raise for a - document that just parsed. + are available whether or not the scan was declined. Without this a document + that was read rather than built would reach a save with no baseline at all, + and every undated entry in it would look new — stamping entries nobody + touched, on a save that changed nothing. + + Digesting refuses only a lone surrogate, which cannot arrive from a file, so + this cannot raise for a document that has just parsed. """ from ._writer import _EntryRecord, entry_digest diff --git a/src/sil_lift/_writer.py b/src/sil_lift/_writer.py index da82a4d..deb0ff5 100644 --- a/src/sil_lift/_writer.py +++ b/src/sil_lift/_writer.py @@ -180,12 +180,10 @@ def default_now() -> datetime: offsets, or fractional seconds. :func:`_fmt_date` renders an aware UTC value that way. - Two saves inside the same second therefore write the same value: an edit - saved within a second of the previous one leaves ``dateModified`` where it - already stood, which a consumer comparing dates reads as no change. The - stamping baseline still tracks it, so nothing is lost on this side, and - sub-second precision would buy the distinction at the cost of the one form - every consumer expects. Pass ``when`` to force a distinct moment. + One second holds one date, so an edit saved within a second of the previous + one carries the same stamp and reads as no change to anything comparing + dates. Sub-second precision would buy that distinction at the cost of the + one form every consumer expects; ``when`` forces a distinct moment. """ return datetime.now(UTC).replace(microsecond=0) @@ -231,10 +229,10 @@ def _needs_stamp(entry: Entry, baseline: _EntryRecord | None, digest: bytes) -> @dataclass(slots=True) class _StampUndo: - """How to put a stamping pass back, for a write that then never happened. + """How to put a stamping pass back if the write it ran for never happens. - Stamping runs before serialization, so a refused or failed write would - otherwise leave the model dated for output that does not exist. + Stamping precedes serialization, so without this a refused or failed write + would leave the model dated for output that does not exist. """ lexicon: Lexicon @@ -260,22 +258,22 @@ def stamp_entries(lexicon: Lexicon, when: datetime) -> _StampUndo: its content differs from the loaded content all right, but so does its date — this library's own stamp from the first save — which reads exactly like a date the caller set deliberately. An entry still matching its parse-time - record needs no such override and is recorded nowhere, so a save of a - handful of edited entries remembers only those, and an entry that has since - left the lexicon is not carried along by a dict that is rebuilt each pass. - - Deciding comes first and mutating second, because digesting is the only - step that can fail (see :func:`_guarded`): a document holding content XML - cannot represent is refused with nothing stamped, rather than half-stamped - at whatever entry the refusal came from. + record needs no such override and is recorded nowhere, so a save of a few + edited entries remembers only those; rebuilding the dict each pass also + drops entries that have since left the lexicon, which would otherwise stay + alive here for a baseline nothing will consult again. + + Deciding comes before mutating because digesting is the only step that can + fail (see :func:`_guarded`): content XML cannot represent is refused with + nothing stamped rather than half-stamped at the entry it was found on. """ source = lexicon._source at_parse = ( {id(record.entry): record for record in source.entry_records} if source is not None else {} ) previous = lexicon._stamps - # Keyed by identity: an entry aliased into the list twice is one entry, with - # one pair of dates, so it is decided and stamped once. + # One object holds one pair of dates, so an entry aliased into the list + # twice is decided and stamped once. entries = list({id(entry): entry for entry in lexicon.entries}.values()) planned: list[tuple[Entry, bytes, bool]] = [] diff --git a/tests/test_stamp.py b/tests/test_stamp.py index ee2ae9e..2c04f1e 100644 --- a/tests/test_stamp.py +++ b/tests/test_stamp.py @@ -242,10 +242,6 @@ def test_stamping_does_not_reach_below_the_entry(tmp_path: Path) -> None: def test_an_unscannable_document_stamps_only_what_changed(tmp_path: Path) -> None: """No byte snapshot, but the digests still date the edit and nothing else. - The scanner declines a source it cannot read, and stamping needs only the - digests, so the reader records a baseline anyway. Without one every undated - entry would look new and a save that changed nothing would stamp them all. - Note what `changed_entries()` says here by contrast: every entry, because `save()` does re-serialize the whole file. Stamping asks the narrower question — what the caller modified — and answers it exactly. @@ -338,9 +334,8 @@ def test_a_removed_entry_drops_out_of_the_stamping_baseline(tmp_path: Path) -> N """The baseline dict is rebuilt each save, so it holds no entry the lexicon lost. An entry the document was loaded with is retained by its parse-time record - either way (that is what makes `removed_entries()` work); one appended and - then dropped would otherwise be kept alive here, subtree and all, by a - baseline nothing will ever consult again. + either way — that is what makes `removed_entries()` work — but one appended + and then dropped would otherwise be kept alive here, subtree and all. """ lexicon = sil_lift.load(UNDATED) appended = sil_lift.Entry(id="temporary") From fadf3b33c77702d9c041566c814ca7e2d1c07a74 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 2 Sep 2026 17:49:20 -0400 Subject: [PATCH 4/4] Note the dates an unstamped save writes, and compare dates as written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two paths left a date the caller set unstampable for good. A save that stamps nothing still writes what the model holds, so a date the caller put there is the date now on disk. note_caller_dates records it, and the next stamping save measures a further edit against it. Without that the entry kept failing the stale test — its date differs from the load, which is exactly what a deliberate date looks like — and was never stamped again, while the same sequence through a stamping save bumped normally. An entry written unstamped under the date it was loaded with is deliberately not noted: its content is on disk under a date that no longer describes it, and the next stamping save should still say so. Dates are compared as the document will carry them rather than as moments. Two aware values an hour and an offset apart are the same instant and equal to ==, so restating 2008-12-12T09:42:48+10:00 at -05:00 — an edit, and an edit to the date itself — read as content changed without its date, and the stamp overwrote the one thing the caller had touched. The parse-time record lookup and the identity keying move into helpers shared by both passes, and _apply_stamps now always returns an undo. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 19 +++++----- src/sil_lift/_writer.py | 79 +++++++++++++++++++++++++++++++++++------ tests/test_stamp.py | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 18 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 3f743aa..f2ba6e2 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -571,16 +571,17 @@ def _resolve_ranges(self) -> None: if exists and resolved not in self.ranges_files: self.ranges_files[resolved] = RangesFile.load(candidate) - def _apply_stamps(self, stamp: bool, when: datetime | None) -> _StampUndo | None: + def _apply_stamps(self, stamp: bool, when: datetime | None) -> _StampUndo: """The stamping step shared by :meth:`save` and :meth:`save_zip`. Returns what undoes the pass, so a write that does not go through can - leave the model as it found it. + leave the model as it found it. A save that stamps nothing still notes + the dates it writes, so the next stamping save measures against them. """ - if not stamp: - return None - from ._writer import resolve_when, stamp_entries + from ._writer import note_caller_dates, resolve_when, stamp_entries + if not stamp: + return note_caller_dates(self) return stamp_entries(self, resolve_when(when)) def save( @@ -611,7 +612,9 @@ def save( model — here, and in a ``save(path)`` used to export a copy — and costs one canonical serialization pass over the entries. - ``stamp=False`` writes the model exactly as it stands. ``when`` + ``stamp=False`` writes the model exactly as it stands — though a date + you set yourself is noted even then, so that a later edit to that entry + is stamped rather than left on a date it has outgrown. ``when`` supplies the moment in place of the clock, normalized to UTC at seconds precision, which is what makes stamped output byte-reproducible; it must be timezone-aware, since a naive moment could as easily mean UTC as @@ -641,7 +644,7 @@ def save( finally: # A companion failing further down leaves the .lift on disk carrying # these stamps, so from here on they stand. - if not written and undo is not None: + if not written: undo.restore() self.path = target relocating = not _same_dir(target.parent, original_dir) @@ -691,7 +694,7 @@ def save_zip( save_zip(self, Path(path), wrap_folder=wrap_folder) written = True finally: - if not written and undo is not None: + if not written: undo.restore() def sort(self) -> None: diff --git a/src/sil_lift/_writer.py b/src/sil_lift/_writer.py index deb0ff5..e456d89 100644 --- a/src/sil_lift/_writer.py +++ b/src/sil_lift/_writer.py @@ -67,6 +67,7 @@ "entry_digest", "header_digest", "node_diff", + "note_caller_dates", "range_digest", "render_document", "render_ranges_document", @@ -209,6 +210,19 @@ def resolve_when(when: datetime | None) -> datetime: return when.astimezone(UTC).replace(microsecond=0) +def _dates_differ(left: datetime | date | None, right: datetime | date | None) -> bool: + """Whether two dates would reach the document as different attribute values. + + Rendered rather than compared as moments, because the question is always + whether the caller changed the date the file will carry. Two aware values an + hour and an offset apart are the same instant and equal to ``==``, so + re-expressing a date in another offset would otherwise register as leaving + it alone — and being overwritten, though it is the one thing the caller + touched. + """ + return _fmt_opt_date(left) != _fmt_opt_date(right) + + def _needs_stamp(entry: Entry, baseline: _EntryRecord | None, digest: bytes) -> bool: """Whether the content moved since ``baseline`` while the date stayed put. @@ -224,7 +238,9 @@ def _needs_stamp(entry: Entry, baseline: _EntryRecord | None, digest: bytes) -> """ if baseline is None: return entry.date_modified is None - return digest != baseline.digest and entry.date_modified == baseline.date_modified + return digest != baseline.digest and not _dates_differ( + entry.date_modified, baseline.date_modified + ) @dataclass(slots=True) @@ -246,6 +262,51 @@ def restore(self) -> None: self.lexicon._stamps = self.stamps +def note_caller_dates(lexicon: Lexicon) -> _StampUndo: + """Move the stamping baseline onto a date the caller set, for a save that stamps nothing. + + A ``stamp=False`` save writes what the model holds, so a date the caller + put there is the date now on disk, and the next stamping save has to measure + a further edit against it. Without this the entry would keep failing the + stale test for good — its date differs from the load, which is exactly what + a deliberate date looks like — and never be stamped again. A stamping save + makes the same adoption for the stamps it writes. + + Only a date that moved off its baseline is noted, and only that entry is + digested. An entry written unstamped under the date it was loaded with is + left out on purpose: its content is on disk under a date that no longer + describes it, and the next stamping save should still say so. + """ + at_parse = _parse_time_records(lexicon) + previous = lexicon._stamps + stamps = dict(previous) + for entry in _by_identity(lexicon): + key = id(entry) + baseline = previous.get(key) + if baseline is None: + baseline = at_parse.get(key) + if baseline is None: + if entry.date_modified is None: + continue # an undated new entry is still new + elif not _dates_differ(entry.date_modified, baseline.date_modified): + continue + stamps[key] = _EntryRecord(entry, entry_digest(entry), entry.date_modified) + lexicon._stamps = stamps + return _StampUndo(lexicon, previous, []) + + +def _parse_time_records(lexicon: Lexicon) -> dict[int, _EntryRecord]: + source = lexicon._source + if source is None: + return {} + return {id(record.entry): record for record in source.entry_records} + + +def _by_identity(lexicon: Lexicon) -> list[Entry]: + """The entries, each once: one object holds one pair of dates.""" + return list({id(entry): entry for entry in lexicon.entries}.values()) + + def stamp_entries(lexicon: Lexicon, when: datetime) -> _StampUndo: """Stamp every entry whose content changed without its ``dateModified`` changing. @@ -267,17 +328,11 @@ def stamp_entries(lexicon: Lexicon, when: datetime) -> _StampUndo: fail (see :func:`_guarded`): content XML cannot represent is refused with nothing stamped rather than half-stamped at the entry it was found on. """ - source = lexicon._source - at_parse = ( - {id(record.entry): record for record in source.entry_records} if source is not None else {} - ) + at_parse = _parse_time_records(lexicon) previous = lexicon._stamps - # One object holds one pair of dates, so an entry aliased into the list - # twice is decided and stamped once. - entries = list({id(entry): entry for entry in lexicon.entries}.values()) planned: list[tuple[Entry, bytes, bool]] = [] - for entry in entries: + for entry in _by_identity(lexicon): baseline = previous.get(id(entry)) if baseline is None: baseline = at_parse.get(id(entry)) @@ -292,7 +347,11 @@ def stamp_entries(lexicon: Lexicon, when: datetime) -> _StampUndo: entry._stamp(when) digest = entry_digest(entry) # the dates are part of an entry's bytes record = at_parse.get(id(entry)) - if record is None or record.digest != digest or record.date_modified != entry.date_modified: + if ( + record is None + or record.digest != digest + or _dates_differ(record.date_modified, entry.date_modified) + ): stamps[id(entry)] = _EntryRecord(entry, digest, entry.date_modified) lexicon._stamps = stamps return _StampUndo(lexicon, previous, dates) diff --git a/tests/test_stamp.py b/tests/test_stamp.py index 2c04f1e..16f2840 100644 --- a/tests/test_stamp.py +++ b/tests/test_stamp.py @@ -183,6 +183,73 @@ def test_a_hand_set_date_becomes_the_baseline_for_the_next_edit(tmp_path: Path) assert entry.date_modified == LATER +def test_an_unstamped_save_notes_a_date_the_caller_set(tmp_path: Path) -> None: + """Which kind of save ran in between must not decide whether a later edit is stamped. + + An unstamped save writes the caller's date to disk, so it becomes the date + the next stamping save measures against — as it would have if that save had + stamped. Otherwise the entry reads as deliberately dated for good and is + never stamped again. + """ + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + out = tmp_path / "out.lift" + + entry.lexical_unit["en"] = "one" + entry.date_modified = BY_HAND + lexicon.save(out, stamp=False) + assert entry.date_modified == BY_HAND # written as it stands, as asked + + entry.lexical_unit["en"] = "two" + lexicon.save(out, when=WHEN) + assert entry.date_modified == WHEN + + +def test_an_unstamped_save_leaves_an_untouched_date_to_the_next_stamp( + tmp_path: Path, +) -> None: + """The other half: content on disk under a date that no longer describes it. + + Nothing is noted for an entry written unstamped under the date it was + loaded with, so the next stamping save still has the edit to report. + """ + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + loaded = entry.date_modified + out = tmp_path / "out.lift" + + entry.lexical_unit["en"] = "edited" + lexicon.save(out, stamp=False) + assert entry.date_modified == loaded + + lexicon.save(out, when=WHEN) # no further edit needed: the first one still stands + assert entry.date_modified == WHEN + + +def test_the_same_instant_in_another_offset_is_the_caller_touching_the_date( + tmp_path: Path, +) -> None: + """Dates are compared as the document will carry them, not as moments. + + Rewriting `2008-12-12T09:42:48+10:00` as the same instant at `-05:00` is an + edit — the file changes — and it is an edit to the date itself, so it is the + one thing a stamp must not overwrite. Comparing the two as moments would + call them equal and take the entry for content edited without its date. + """ + lexicon = sil_lift.load(DATED) + entry = lexicon.entries[0] + loaded = entry.date_modified + assert isinstance(loaded, datetime) + restated = loaded.astimezone(timezone(timedelta(hours=-5))) + assert restated == loaded # the same moment to ==, a different attribute value + entry.date_modified = restated + + lexicon.save(tmp_path / "out.lift", when=WHEN) + + assert entry.date_modified == restated + assert entry.date_modified.utcoffset() == timedelta(hours=-5) + + def test_an_entry_added_after_load_is_stamped_only_when_its_date_is_blank( tmp_path: Path, ) -> None: