From 864f70ae7343d6de76b2e3b45fe94322602343d8 Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:49:50 -0600 Subject: [PATCH 1/2] missing_docs: stop the drift-watch triage from silently seeing nothing Found by the first manual validation run of the drift-watch skill. All three defects fail the same way: the run exits 0, reports nothing, and that is indistinguishable from "nothing shipped". 1. Triage baseline ignored the release marker Two independent state files track releases. surface_snapshot.json records what was last *observed*; last_release_processed.json records what was last *triaged*. A bookkeeping PR that regenerates the snapshot advances the first without touching the second, and #603 did exactly that -- on main today the snapshot reads 2026.08.19 while the marker reads 2026.08.18. The gate fires (marker behind) but --diff returns an empty changelog_review (snapshot ahead), so the primary triage input is empty and the release gets marked processed having triaged nothing. The baseline is now the earlier of the two markers. Against current main that moves changelog_review from 0 items to 3. I introduced this when I split the marker out of the snapshot. The reasoning was sound -- the snapshot is regenerated wholesale and would lose the marker -- but it only considered the snapshot falling behind, not running ahead. 2. oz_updates were2. oz_updates were2. oz_updates were2. oz_updatrepo2. oz_updates were2. oz_updates were2. oz_up no2. oz_updates were2. oz_updates were2. oz_updates were2. oz_updatrepo2. nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnntfnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnh was not executable. Texecutable. Texecutable. Texecutable. Texecutable. Texecun. The field nThe field nThe field nThe field nThe field nThe fiel in the currcurrcurrcue pcurrcurrcurrcue pcurrcurrcurrcue pcurrcurrcurrcue pcurrcuchangelog heading would fail silently Section matching is a hardcoded allowlist against a user-facing heading, so the rename would zero out that section with no error. Added the Automation Platform variants, plus a guard that treats bullets under an unrecounrecounrecounrecounrecounrecounrecounrecounrecxes are listed as dededededededededededededededededededededededededededededededededededed ttttttttttttttttttttttttttttttttttttttttttttttttttttter launch posts use prose headings and policing all 277 entries would fail every run. Also adds the __future__ annotations import that check_new_release.py already had. Without it audit_docs.py cannot be imported on Python 3.9,already had. Without it audit_docs.py cannot be imported on Python 3.9,already had. Without it audit_docs.py cannot be importedver the baseline, the normalizer, the tracked/untracked/unknown split, and thtracked/untracked/unknown split, and thtracked/untrp.dev> --- .agents/skills/missing_docs/SKILL.md | 8 + .../skills/missing_docs/scripts/audit_docs.py | 166 ++++++++++++++++-- .../missing_docs/scripts/check_new_release.py | 20 ++- .../missing_docs/scripts/test_audit_docs.py | 84 +++++++++ 4 files changed, 265 insertions(+), 13 deletions(-) diff --git a/.agents/skills/missing_docs/SKILL.md b/.agents/skills/missing_docs/SKILL.md index 9a1c0ce24..977d537dd 100644 --- a/.agents/skills/missing_docs/SKILL.md +++ b/.agents/skills/missing_docs/SKILL.md @@ -223,6 +223,11 @@ them through these three layers instead, in order of preference: gate already fetches. Release-gated, low-noise, and currently the most direct signal for platform-side changes. Triage these bullets exactly like changelog bullets: same gates, same ledger, same evidence requirement. + + `check_new_release.py` prints them; the audit never sees them, because it is offline + by design and they are not part of the markdown changelog it parses. Read them from + the gate's output, or `--json` for the full array. `oz_updates` is the API's field + name and stays as-is regardless of product naming. 2. **The public Agent API surface** — already covered by audit category 3 and `sync-openapi-spec`. No new machinery; just confirm the release run actually triages these findings rather than deferring them by habit. A released endpoint reaches docs @@ -410,6 +415,9 @@ with the product. Each run: release; record the no-op outcome in run output and **stop**. Exit `1` is a fetch or parse failure; report it and stop rather than proceeding as if nothing shipped. + The gate also prints any `oz_updates` bullets for the release. Keep them — they are + platform-side changes the audit cannot see, and this is the only place they surface. + Do not update the state file yet. It is written in step 5, after triage, so a run that crashes mid-triage retries the same release instead of skipping it. 2. **Audit**: run both modes and save reports. Pass explicit repo paths; verify diff --git a/.agents/skills/missing_docs/scripts/audit_docs.py b/.agents/skills/missing_docs/scripts/audit_docs.py index ee32cbf14..22720402f 100755 --- a/.agents/skills/missing_docs/scripts/audit_docs.py +++ b/.agents/skills/missing_docs/scripts/audit_docs.py @@ -36,6 +36,13 @@ clean audit. """ +# Postponed annotation evaluation keeps `X | None` annotations working on +# Python 3.9, which is still the system interpreter on some contributor +# machines. Without it this module cannot even be imported there, so +# test_audit_docs.py fails before a single test runs. check_new_release.py +# already does this. +from __future__ import annotations + import argparse import json import os @@ -1189,18 +1196,40 @@ def page_slug(md_file: Path, docs_root: Path) -> str: _CHANGELOG_HEADER_RE = re.compile(r"^### (\d{4}\.\d{2}\.\d{2})", re.MULTILINE) -# "Bug fixes" is deliberately untracked: fix bullets rarely create doc surface -# and would double the weekly triage volume. -_CHANGELOG_TRACKED_SECTIONS = ("new features", "improvements", "oz updates") + +# Bold section labels in the docs changelog whose bullets are triaged. +# +# These match a *user-facing heading*, not an API field, so they are exposed to +# product renames. The Automation Platform variants are accepted ahead of the +# rename: if the changelog heading changes and this tuple has not, the section +# silently yields zero bullets and the run under-reports. The guard in +# parse_changelog_entries() is the backstop for a name nobody anticipated. +_CHANGELOG_TRACKED_SECTIONS = ( + "new features", + "improvements", + "oz updates", + "automation platform updates", + "platform updates", +) + +# Sections deliberately not triaged, listed so the unknown-section guard can +# tell "intentionally skipped" from "renamed out from under us". Bug fixes are +# excluded on purpose to keep weekly triage volume manageable. +_CHANGELOG_UNTRACKED_SECTIONS = ("bug fixes", "fixes") def parse_changelog_entries(repo_root: Path) -> list[dict]: """Parse release entries from src/content/docs/changelog/.mdx. Returns [{"version": "2026.06.03", "file": str, "items": - [{"category": "new features", "text": str}]}] sorted newest first. - "New features", "Improvements", and "Oz updates" bullets are tracked — - the sections that may represent undocumented feature launches. + [{"category": "new features", "text": str}], + "unknown_sections": [str]}] sorted newest first. + + Only _CHANGELOG_TRACKED_SECTIONS bullets are collected — the sections that + may represent undocumented feature launches. Any other bold section that + contains bullets and is not in _CHANGELOG_UNTRACKED_SECTIONS is reported in + `unknown_sections` so a renamed heading surfaces as a loud failure instead + of an empty result. """ changelog_dir = repo_root / "src" / "content" / "docs" / "changelog" if not changelog_dir.exists(): @@ -1216,6 +1245,7 @@ def parse_changelog_entries(repo_root: Path) -> list[dict]: end = headers[i + 1].start() if i + 1 < len(headers) else len(content) body = content[header.end():end] items = [] + unknown_sections = set() current_section = None for line in body.splitlines(): stripped = line.strip() @@ -1223,15 +1253,20 @@ def parse_changelog_entries(repo_root: Path) -> list[dict]: if section_match: current_section = section_match.group(1).strip().lower() continue - if current_section in _CHANGELOG_TRACKED_SECTIONS and stripped.startswith("* "): + if not stripped.startswith("* ") or not current_section: + continue + if current_section in _CHANGELOG_TRACKED_SECTIONS: items.append({ "category": current_section, "text": stripped[2:].strip(), }) + elif current_section not in _CHANGELOG_UNTRACKED_SECTIONS: + unknown_sections.add(current_section) entries.append({ "version": header.group(1), "file": str(mdx.relative_to(repo_root)), "items": items, + "unknown_sections": sorted(unknown_sections), }) entries.sort(key=lambda e: e["version"], reverse=True) return entries @@ -2287,18 +2322,92 @@ def diff_snapshots(old: dict, new: dict) -> list[dict]: return findings +_RELEASE_VERSION_RE = re.compile(r"(\d{4}\.\d{2}\.\d{2})") + + +def normalize_release_version(value: str | None) -> str | None: + """Reduce any release identifier to the YYYY.MM.DD form changelog headers use. + + The snapshot stores `2026.08.19`; the release marker stores + `v0.2026.08.19.08.15.stable_01`. Both must compare against changelog entry + versions, so both are normalized through here. + """ + if not value: + return None + match = _RELEASE_VERSION_RE.search(str(value)) + return match.group(1) if match else None + + +def read_last_processed_release(snapshot_path: Path) -> str | None: + """Read the triage marker that sits alongside the snapshot. + + Returns the normalized version last *triaged*, or None if the marker is + missing or unreadable. A missing marker means "never triaged", which is the + safe direction: nothing gets skipped. + """ + marker = snapshot_path.parent / "last_release_processed.json" + if not marker.exists(): + return None + try: + data = json.loads(marker.read_text(encoding="utf-8")) + except Exception as exc: + print(f"Warning: failed to parse {marker}: {exc}", file=sys.stderr) + return None + return normalize_release_version(data.get("last_processed_version")) + + +def changelog_review_baseline(last_seen_version: str | None, + last_triaged_version: str | None) -> str | None: + """Earliest of the observed and triaged markers, normalized. + + Using the earlier of the two is what stops a bookkeeping-only run — which + advances the snapshot without triaging — from hiding a release. + """ + return min( + (v for v in (normalize_release_version(last_seen_version), + normalize_release_version(last_triaged_version)) if v), + default=None, + ) + + +def unreviewed_unknown_sections(changelog_entries: list[dict], + baseline: str | None) -> list[str]: + """Unrecognized bold sections among the entries actually being triaged. + + Scoped to unreviewed entries on purpose. Older launch posts use prose + headings ("Multithread yourself with agents") that are neither tracked + sections nor renames; policing the whole file would fail every run on + history nobody is going to re-triage. + """ + return sorted({ + section + for entry in changelog_entries + if not (baseline and entry["version"] <= baseline) + for section in entry.get("unknown_sections", []) + }) + + def changelog_review_findings(changelog_entries: list[dict], - last_seen_version: str | None) -> list[dict]: - """Emit verification findings for changelog entries newer than the snapshot. + last_seen_version: str | None, + last_triaged_version: str | None = None) -> list[dict]: + """Emit verification findings for changelog entries not yet triaged. The weekly human-curated changelog is the best signal for launches that no static code parse can see (server-side features, Oz web app, experiment rollouts). Each bullet should be verified for real docs coverage — a changelog mention alone is not documentation. + + Two independent markers exist and they can disagree. The snapshot's + `changelog_last_version` records what was last *observed*; bookkeeping that + regenerates the snapshot advances it without triaging anything. + `last_release_processed.json` records what was last *triaged*. Using the + snapshot alone silently drops every entry a bookkeeping-only run skipped + past, so the baseline is the earlier of the two. """ + baseline = changelog_review_baseline(last_seen_version, last_triaged_version) findings = [] for entry in changelog_entries: - if last_seen_version and entry["version"] <= last_seen_version: + if baseline and entry["version"] <= baseline: continue for item in entry["items"]: findings.append({ @@ -2872,6 +2981,7 @@ def guard(label: str, count: int) -> bool: # Change detection (diff + snapshot update) changelog_entries = parse_changelog_entries(repo_root) snapshot_path = Path(args.snapshot) + if args.diff or args.update_snapshot: if warp_repo and warp_server and extraction_ok: current_snapshot = build_snapshot( @@ -2891,8 +3001,42 @@ def guard(label: str, count: int) -> bool: else: print("Running surface change detection (diff)...", file=sys.stderr) findings["surface_changes"] = diff_snapshots(previous, current_snapshot) + snapshot_version = normalize_release_version( + previous.get("changelog_last_version")) + triaged_version = read_last_processed_release(snapshot_path) findings["changelog_review"] = changelog_review_findings( - changelog_entries, previous.get("changelog_last_version")) + changelog_entries, + previous.get("changelog_last_version"), + triaged_version) + if (snapshot_version and triaged_version + and snapshot_version != triaged_version): + print( + "Note: snapshot last saw " + f"{snapshot_version} but {triaged_version} was the last " + "release triaged. Reviewing from the earlier of the two " + "so bookkeeping-only runs cannot skip a release.", + file=sys.stderr, + ) + + # Bullets under a heading the parser does not recognize are + # invisible to triage. Silence there looks identical to "nothing + # shipped", so treat it as an extraction failure instead. + unknown_sections = unreviewed_unknown_sections( + changelog_entries, + changelog_review_baseline( + previous.get("changelog_last_version"), triaged_version)) + if unknown_sections: + audits_skipped.append({ + "audit": "extraction:changelog_sections", + "reason": ( + "bullets found under unrecognized changelog headings " + f"{unknown_sections} in entries awaiting triage \u2014 a " + "section was renamed or added. Add each heading to " + "_CHANGELOG_TRACKED_SECTIONS or " + "_CHANGELOG_UNTRACKED_SECTIONS in audit_docs.py; until " + "then those bullets are invisible to triage" + ), + }) audits_run.append("diff") if args.update_snapshot: snapshot_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/.agents/skills/missing_docs/scripts/check_new_release.py b/.agents/skills/missing_docs/scripts/check_new_release.py index 200f5ed9e..48af190d8 100755 --- a/.agents/skills/missing_docs/scripts/check_new_release.py +++ b/.agents/skills/missing_docs/scripts/check_new_release.py @@ -183,6 +183,14 @@ def main() -> int: print(f"Recorded {version} as processed in {args.state}") return EXIT_NEW_RELEASE + # `oz_updates` is a separate array in the client_version payload, not part + # of the markdown changelog the audit parses. The audit is offline by + # design and never sees it, so this is the only place those bullets surface + # — print the content, not just a count, or the skill's instruction to + # triage them cannot be carried out. The field name is the API's, and stays + # `oz_updates` regardless of product naming. + oz_updates = changelog.get("oz_updates", []) or [] + if args.as_json: print( json.dumps( @@ -191,7 +199,8 @@ def main() -> int: "current_version": version, "last_processed_version": last_processed, "release_date": release_date, - "oz_updates_count": len(changelog.get("oz_updates", []) or []), + "oz_updates_count": len(oz_updates), + "oz_updates": oz_updates, }, indent=2, ) @@ -201,7 +210,14 @@ def main() -> int: print(f"New stable release: {version}") print(f" Released: {release_date or 'unknown'}") print(f" Last processed: {previous}") - print(f" Oz updates: {len(changelog.get('oz_updates', []) or [])}") + print(f" Oz updates: {len(oz_updates)}") + for bullet in oz_updates: + print(f" - {bullet}") + if oz_updates: + print( + " These are platform-side changes the audit cannot see. " + "Triage them against the worthiness criteria like changelog bullets." + ) print("Proceed with the audit.") else: print(f"No new release. Current stable {version} was already processed.") diff --git a/.agents/skills/missing_docs/scripts/test_audit_docs.py b/.agents/skills/missing_docs/scripts/test_audit_docs.py index 3636069fb..1ff38922e 100755 --- a/.agents/skills/missing_docs/scripts/test_audit_docs.py +++ b/.agents/skills/missing_docs/scripts/test_audit_docs.py @@ -223,5 +223,89 @@ def test_map_hygiene_flags_unknown_gated_flag(self): self.assertEqual(gated_findings[0]["entry"], "oz bad") +class TestChangelogTriage(unittest.TestCase): + """Regression cases for the drift-watch triage baseline and section guard. + + All three failures these cover are silent: the run exits 0 and reports + nothing, which is indistinguishable from "nothing shipped". + """ + + ENTRY = ( + "### 2099.01.02 (v0.2099.01.02.00.00)\n\n" + "**New features**\n\n" + "* A tracked bullet.\n\n" + "**Automation Platform updates**\n\n" + "* A bullet under the post-rename platform heading.\n\n" + "**Bug fixes**\n\n" + "* Deliberately untracked.\n\n" + "**Sparkles**\n\n" + "* A bullet under a heading the parser has never seen.\n" + ) + + def _entries(self, text): + with tempfile.TemporaryDirectory() as d: + changelog = Path(d) / "src" / "content" / "docs" / "changelog" + changelog.mkdir(parents=True) + (changelog / "2099.mdx").write_text(text, encoding="utf-8") + return audit_docs.parse_changelog_entries(Path(d)) + + def test_normalize_release_version(self): + """The marker and the snapshot store versions in different shapes.""" + self.assertEqual( + audit_docs.normalize_release_version("v0.2026.08.18.02.52.stable_00"), + "2026.08.18") + self.assertEqual( + audit_docs.normalize_release_version("2026.08.19"), "2026.08.19") + self.assertIsNone(audit_docs.normalize_release_version(None)) + self.assertIsNone(audit_docs.normalize_release_version("not-a-version")) + + def test_baseline_uses_the_earlier_marker(self): + """A snapshot refresh must not skip a release that was never triaged. + + Bookkeeping PRs regenerate surface_snapshot.json, advancing its + changelog pointer without triaging anything. Taking the snapshot alone + as the baseline drops every entry in between. + """ + self.assertEqual( + audit_docs.changelog_review_baseline( + "2026.08.19", "v0.2026.08.18.02.52.stable_00"), + "2026.08.18") + # Either side missing falls back to the other rather than to "all seen". + self.assertEqual( + audit_docs.changelog_review_baseline("2026.08.19", None), "2026.08.19") + self.assertEqual( + audit_docs.changelog_review_baseline(None, "v0.2026.08.18.02.52.stable_00"), + "2026.08.18") + # No markers at all means nothing has been triaged; review everything. + self.assertIsNone(audit_docs.changelog_review_baseline(None, None)) + + def test_desynced_markers_still_surface_the_release(self): + """End to end: the snapshot ahead of the marker must not hide bullets.""" + entries = self._entries(self.ENTRY) + ahead = audit_docs.changelog_review_findings(entries, "2099.01.02") + self.assertEqual(ahead, [], "snapshot-only baseline hides the entry") + recovered = audit_docs.changelog_review_findings( + entries, "2099.01.02", "2099.01.01") + self.assertEqual(len(recovered), 2, recovered) + + def test_tracked_untracked_and_unknown_sections(self): + entry = self._entries(self.ENTRY)[0] + categories = sorted(i["category"] for i in entry["items"]) + self.assertEqual(categories, ["automation platform updates", "new features"]) + # Bug fixes are skipped on purpose, so they must not read as a rename. + self.assertEqual(entry["unknown_sections"], ["sparkles"]) + + def test_unknown_section_guard_ignores_already_triaged_history(self): + """Old launch posts use prose headings; policing them fails every run.""" + entries = self._entries(self.ENTRY) + self.assertEqual( + audit_docs.unreviewed_unknown_sections(entries, "2099.01.02"), [], + "entries at or below the baseline are already triaged") + self.assertEqual( + audit_docs.unreviewed_unknown_sections(entries, "2099.01.01"), + ["sparkles"], + "an unrecognized heading in a pending entry must fail loud") + + if __name__ == "__main__": unittest.main(verbosity=2) From 97214abf13636f94117cb594ef8bd4c55505d2fb Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:11:35 -0600 Subject: [PATCH 2/2] missing_docs: treat a wrong-shaped release marker as unreadable Review catch from both bots on #617. read_last_processed_release() called .get() on whatever json.loads returned, so a marker containing valid JSON of the wrong top-level type -- [], a bare string, a number -- raised AttributeError rather than taking the documented fallback to None. That aborts diff-mode triage entirely, which is a worse outcome than the desync the reader was added to fix. Guarded with isinstance(data, dict) and a warning, matching how check_new_release.py's read_state() already handles the same file. The inconsistency between two readers of one file was the actual defect. Regression test covers every bad shape: list, bare string, number, null, truncated JSON, empty file, and an object missing the expected key, plus the missing-marker and well-formed cases so the fallback cannot be achieved by breaking the happy path. Co-Authored-By: Warp --- .../skills/missing_docs/scripts/audit_docs.py | 15 ++++++- .../missing_docs/scripts/test_audit_docs.py | 42 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.agents/skills/missing_docs/scripts/audit_docs.py b/.agents/skills/missing_docs/scripts/audit_docs.py index 22720402f..3446c24bf 100755 --- a/.agents/skills/missing_docs/scripts/audit_docs.py +++ b/.agents/skills/missing_docs/scripts/audit_docs.py @@ -2342,8 +2342,8 @@ def read_last_processed_release(snapshot_path: Path) -> str | None: """Read the triage marker that sits alongside the snapshot. Returns the normalized version last *triaged*, or None if the marker is - missing or unreadable. A missing marker means "never triaged", which is the - safe direction: nothing gets skipped. + missing, unreadable, or not a JSON object. A missing marker means "never + triaged", which is the safe direction: nothing gets skipped. """ marker = snapshot_path.parent / "last_release_processed.json" if not marker.exists(): @@ -2353,6 +2353,17 @@ def read_last_processed_release(snapshot_path: Path) -> str | None: except Exception as exc: print(f"Warning: failed to parse {marker}: {exc}", file=sys.stderr) return None + # Valid JSON of the wrong shape (a list, a bare string) would raise on + # .get and abort diff-mode triage. Treat it like a parse failure so the + # baseline falls back to the snapshot instead. check_new_release.py's + # read_state() guards the same file the same way. + if not isinstance(data, dict): + print( + f"Warning: {marker} is not a JSON object (got {type(data).__name__}); " + "treating the release as never triaged", + file=sys.stderr, + ) + return None return normalize_release_version(data.get("last_processed_version")) diff --git a/.agents/skills/missing_docs/scripts/test_audit_docs.py b/.agents/skills/missing_docs/scripts/test_audit_docs.py index 1ff38922e..ce5f108ef 100755 --- a/.agents/skills/missing_docs/scripts/test_audit_docs.py +++ b/.agents/skills/missing_docs/scripts/test_audit_docs.py @@ -306,6 +306,48 @@ def test_unknown_section_guard_ignores_already_triaged_history(self): ["sparkles"], "an unrecognized heading in a pending entry must fail loud") + def test_marker_reader_survives_every_bad_shape(self): + """A malformed marker must degrade to "never triaged", never raise. + + Raising here aborts diff-mode triage entirely, which is a worse failure + than the desync this reader exists to fix. Valid JSON of the wrong + top-level type is the case that bites: `[]` reaches `.get` and throws. + """ + cases = { + "list": "[]", + "bare string": '"v0.2026.08.18.02.52.stable_00"', + "number": "42", + "null": "null", + "truncated json": '{"last_processed_version":', + "empty file": "", + "object, wrong key": '{"version": "v0.2026.08.18.02.52.stable_00"}', + } + for label, payload in cases.items(): + with self.subTest(shape=label), tempfile.TemporaryDirectory() as d: + refs = Path(d) + (refs / "last_release_processed.json").write_text( + payload, encoding="utf-8") + self.assertIsNone( + audit_docs.read_last_processed_release( + refs / "surface_snapshot.json"), + f"{label} marker should fall back to None") + + # A missing marker is the ordinary first-run case, not an error. + with tempfile.TemporaryDirectory() as d: + self.assertIsNone(audit_docs.read_last_processed_release( + Path(d) / "surface_snapshot.json")) + + # The well-formed case still reads through. + with tempfile.TemporaryDirectory() as d: + refs = Path(d) + (refs / "last_release_processed.json").write_text( + '{"last_processed_version": "v0.2026.08.18.02.52.stable_00"}', + encoding="utf-8") + self.assertEqual( + audit_docs.read_last_processed_release( + refs / "surface_snapshot.json"), + "2026.08.18") + if __name__ == "__main__": unittest.main(verbosity=2)