diff --git a/.agents/skills/missing_docs/SKILL.md b/.agents/skills/missing_docs/SKILL.md index 9a1c0ce24..977d537dd 100644 --- a/.agents/skills/missing_docs/SKILL.md +++ b/.agents/skills/missing_docs/SKILL.md @@ -223,6 +223,11 @@ them through these three layers instead, in order of preference: gate already fetches. Release-gated, low-noise, and currently the most direct signal for platform-side changes. Triage these bullets exactly like changelog bullets: same gates, same ledger, same evidence requirement. + + `check_new_release.py` prints them; the audit never sees them, because it is offline + by design and they are not part of the markdown changelog it parses. Read them from + the gate's output, or `--json` for the full array. `oz_updates` is the API's field + name and stays as-is regardless of product naming. 2. **The public Agent API surface** — already covered by audit category 3 and `sync-openapi-spec`. No new machinery; just confirm the release run actually triages these findings rather than deferring them by habit. A released endpoint reaches docs @@ -410,6 +415,9 @@ with the product. Each run: release; record the no-op outcome in run output and **stop**. Exit `1` is a fetch or parse failure; report it and stop rather than proceeding as if nothing shipped. + The gate also prints any `oz_updates` bullets for the release. Keep them — they are + platform-side changes the audit cannot see, and this is the only place they surface. + Do not update the state file yet. It is written in step 5, after triage, so a run that crashes mid-triage retries the same release instead of skipping it. 2. **Audit**: run both modes and save reports. Pass explicit repo paths; verify diff --git a/.agents/skills/missing_docs/scripts/audit_docs.py b/.agents/skills/missing_docs/scripts/audit_docs.py index ee32cbf14..3446c24bf 100755 --- a/.agents/skills/missing_docs/scripts/audit_docs.py +++ b/.agents/skills/missing_docs/scripts/audit_docs.py @@ -36,6 +36,13 @@ clean audit. """ +# Postponed annotation evaluation keeps `X | None` annotations working on +# Python 3.9, which is still the system interpreter on some contributor +# machines. Without it this module cannot even be imported there, so +# test_audit_docs.py fails before a single test runs. check_new_release.py +# already does this. +from __future__ import annotations + import argparse import json import os @@ -1189,18 +1196,40 @@ def page_slug(md_file: Path, docs_root: Path) -> str: _CHANGELOG_HEADER_RE = re.compile(r"^### (\d{4}\.\d{2}\.\d{2})", re.MULTILINE) -# "Bug fixes" is deliberately untracked: fix bullets rarely create doc surface -# and would double the weekly triage volume. -_CHANGELOG_TRACKED_SECTIONS = ("new features", "improvements", "oz updates") + +# Bold section labels in the docs changelog whose bullets are triaged. +# +# These match a *user-facing heading*, not an API field, so they are exposed to +# product renames. The Automation Platform variants are accepted ahead of the +# rename: if the changelog heading changes and this tuple has not, the section +# silently yields zero bullets and the run under-reports. The guard in +# parse_changelog_entries() is the backstop for a name nobody anticipated. +_CHANGELOG_TRACKED_SECTIONS = ( + "new features", + "improvements", + "oz updates", + "automation platform updates", + "platform updates", +) + +# Sections deliberately not triaged, listed so the unknown-section guard can +# tell "intentionally skipped" from "renamed out from under us". Bug fixes are +# excluded on purpose to keep weekly triage volume manageable. +_CHANGELOG_UNTRACKED_SECTIONS = ("bug fixes", "fixes") def parse_changelog_entries(repo_root: Path) -> list[dict]: """Parse release entries from src/content/docs/changelog/.mdx. Returns [{"version": "2026.06.03", "file": str, "items": - [{"category": "new features", "text": str}]}] sorted newest first. - "New features", "Improvements", and "Oz updates" bullets are tracked — - the sections that may represent undocumented feature launches. + [{"category": "new features", "text": str}], + "unknown_sections": [str]}] sorted newest first. + + Only _CHANGELOG_TRACKED_SECTIONS bullets are collected — the sections that + may represent undocumented feature launches. Any other bold section that + contains bullets and is not in _CHANGELOG_UNTRACKED_SECTIONS is reported in + `unknown_sections` so a renamed heading surfaces as a loud failure instead + of an empty result. """ changelog_dir = repo_root / "src" / "content" / "docs" / "changelog" if not changelog_dir.exists(): @@ -1216,6 +1245,7 @@ def parse_changelog_entries(repo_root: Path) -> list[dict]: end = headers[i + 1].start() if i + 1 < len(headers) else len(content) body = content[header.end():end] items = [] + unknown_sections = set() current_section = None for line in body.splitlines(): stripped = line.strip() @@ -1223,15 +1253,20 @@ def parse_changelog_entries(repo_root: Path) -> list[dict]: if section_match: current_section = section_match.group(1).strip().lower() continue - if current_section in _CHANGELOG_TRACKED_SECTIONS and stripped.startswith("* "): + if not stripped.startswith("* ") or not current_section: + continue + if current_section in _CHANGELOG_TRACKED_SECTIONS: items.append({ "category": current_section, "text": stripped[2:].strip(), }) + elif current_section not in _CHANGELOG_UNTRACKED_SECTIONS: + unknown_sections.add(current_section) entries.append({ "version": header.group(1), "file": str(mdx.relative_to(repo_root)), "items": items, + "unknown_sections": sorted(unknown_sections), }) entries.sort(key=lambda e: e["version"], reverse=True) return entries @@ -2287,18 +2322,103 @@ def diff_snapshots(old: dict, new: dict) -> list[dict]: return findings +_RELEASE_VERSION_RE = re.compile(r"(\d{4}\.\d{2}\.\d{2})") + + +def normalize_release_version(value: str | None) -> str | None: + """Reduce any release identifier to the YYYY.MM.DD form changelog headers use. + + The snapshot stores `2026.08.19`; the release marker stores + `v0.2026.08.19.08.15.stable_01`. Both must compare against changelog entry + versions, so both are normalized through here. + """ + if not value: + return None + match = _RELEASE_VERSION_RE.search(str(value)) + return match.group(1) if match else None + + +def read_last_processed_release(snapshot_path: Path) -> str | None: + """Read the triage marker that sits alongside the snapshot. + + Returns the normalized version last *triaged*, or None if the marker is + missing, unreadable, or not a JSON object. A missing marker means "never + triaged", which is the safe direction: nothing gets skipped. + """ + marker = snapshot_path.parent / "last_release_processed.json" + if not marker.exists(): + return None + try: + data = json.loads(marker.read_text(encoding="utf-8")) + except Exception as exc: + print(f"Warning: failed to parse {marker}: {exc}", file=sys.stderr) + return None + # Valid JSON of the wrong shape (a list, a bare string) would raise on + # .get and abort diff-mode triage. Treat it like a parse failure so the + # baseline falls back to the snapshot instead. check_new_release.py's + # read_state() guards the same file the same way. + if not isinstance(data, dict): + print( + f"Warning: {marker} is not a JSON object (got {type(data).__name__}); " + "treating the release as never triaged", + file=sys.stderr, + ) + return None + return normalize_release_version(data.get("last_processed_version")) + + +def changelog_review_baseline(last_seen_version: str | None, + last_triaged_version: str | None) -> str | None: + """Earliest of the observed and triaged markers, normalized. + + Using the earlier of the two is what stops a bookkeeping-only run — which + advances the snapshot without triaging — from hiding a release. + """ + return min( + (v for v in (normalize_release_version(last_seen_version), + normalize_release_version(last_triaged_version)) if v), + default=None, + ) + + +def unreviewed_unknown_sections(changelog_entries: list[dict], + baseline: str | None) -> list[str]: + """Unrecognized bold sections among the entries actually being triaged. + + Scoped to unreviewed entries on purpose. Older launch posts use prose + headings ("Multithread yourself with agents") that are neither tracked + sections nor renames; policing the whole file would fail every run on + history nobody is going to re-triage. + """ + return sorted({ + section + for entry in changelog_entries + if not (baseline and entry["version"] <= baseline) + for section in entry.get("unknown_sections", []) + }) + + def changelog_review_findings(changelog_entries: list[dict], - last_seen_version: str | None) -> list[dict]: - """Emit verification findings for changelog entries newer than the snapshot. + last_seen_version: str | None, + last_triaged_version: str | None = None) -> list[dict]: + """Emit verification findings for changelog entries not yet triaged. The weekly human-curated changelog is the best signal for launches that no static code parse can see (server-side features, Oz web app, experiment rollouts). Each bullet should be verified for real docs coverage — a changelog mention alone is not documentation. + + Two independent markers exist and they can disagree. The snapshot's + `changelog_last_version` records what was last *observed*; bookkeeping that + regenerates the snapshot advances it without triaging anything. + `last_release_processed.json` records what was last *triaged*. Using the + snapshot alone silently drops every entry a bookkeeping-only run skipped + past, so the baseline is the earlier of the two. """ + baseline = changelog_review_baseline(last_seen_version, last_triaged_version) findings = [] for entry in changelog_entries: - if last_seen_version and entry["version"] <= last_seen_version: + if baseline and entry["version"] <= baseline: continue for item in entry["items"]: findings.append({ @@ -2872,6 +2992,7 @@ def guard(label: str, count: int) -> bool: # Change detection (diff + snapshot update) changelog_entries = parse_changelog_entries(repo_root) snapshot_path = Path(args.snapshot) + if args.diff or args.update_snapshot: if warp_repo and warp_server and extraction_ok: current_snapshot = build_snapshot( @@ -2891,8 +3012,42 @@ def guard(label: str, count: int) -> bool: else: print("Running surface change detection (diff)...", file=sys.stderr) findings["surface_changes"] = diff_snapshots(previous, current_snapshot) + snapshot_version = normalize_release_version( + previous.get("changelog_last_version")) + triaged_version = read_last_processed_release(snapshot_path) findings["changelog_review"] = changelog_review_findings( - changelog_entries, previous.get("changelog_last_version")) + changelog_entries, + previous.get("changelog_last_version"), + triaged_version) + if (snapshot_version and triaged_version + and snapshot_version != triaged_version): + print( + "Note: snapshot last saw " + f"{snapshot_version} but {triaged_version} was the last " + "release triaged. Reviewing from the earlier of the two " + "so bookkeeping-only runs cannot skip a release.", + file=sys.stderr, + ) + + # Bullets under a heading the parser does not recognize are + # invisible to triage. Silence there looks identical to "nothing + # shipped", so treat it as an extraction failure instead. + unknown_sections = unreviewed_unknown_sections( + changelog_entries, + changelog_review_baseline( + previous.get("changelog_last_version"), triaged_version)) + if unknown_sections: + audits_skipped.append({ + "audit": "extraction:changelog_sections", + "reason": ( + "bullets found under unrecognized changelog headings " + f"{unknown_sections} in entries awaiting triage \u2014 a " + "section was renamed or added. Add each heading to " + "_CHANGELOG_TRACKED_SECTIONS or " + "_CHANGELOG_UNTRACKED_SECTIONS in audit_docs.py; until " + "then those bullets are invisible to triage" + ), + }) audits_run.append("diff") if args.update_snapshot: snapshot_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/.agents/skills/missing_docs/scripts/check_new_release.py b/.agents/skills/missing_docs/scripts/check_new_release.py index 200f5ed9e..48af190d8 100755 --- a/.agents/skills/missing_docs/scripts/check_new_release.py +++ b/.agents/skills/missing_docs/scripts/check_new_release.py @@ -183,6 +183,14 @@ def main() -> int: print(f"Recorded {version} as processed in {args.state}") return EXIT_NEW_RELEASE + # `oz_updates` is a separate array in the client_version payload, not part + # of the markdown changelog the audit parses. The audit is offline by + # design and never sees it, so this is the only place those bullets surface + # — print the content, not just a count, or the skill's instruction to + # triage them cannot be carried out. The field name is the API's, and stays + # `oz_updates` regardless of product naming. + oz_updates = changelog.get("oz_updates", []) or [] + if args.as_json: print( json.dumps( @@ -191,7 +199,8 @@ def main() -> int: "current_version": version, "last_processed_version": last_processed, "release_date": release_date, - "oz_updates_count": len(changelog.get("oz_updates", []) or []), + "oz_updates_count": len(oz_updates), + "oz_updates": oz_updates, }, indent=2, ) @@ -201,7 +210,14 @@ def main() -> int: print(f"New stable release: {version}") print(f" Released: {release_date or 'unknown'}") print(f" Last processed: {previous}") - print(f" Oz updates: {len(changelog.get('oz_updates', []) or [])}") + print(f" Oz updates: {len(oz_updates)}") + for bullet in oz_updates: + print(f" - {bullet}") + if oz_updates: + print( + " These are platform-side changes the audit cannot see. " + "Triage them against the worthiness criteria like changelog bullets." + ) print("Proceed with the audit.") else: print(f"No new release. Current stable {version} was already processed.") diff --git a/.agents/skills/missing_docs/scripts/test_audit_docs.py b/.agents/skills/missing_docs/scripts/test_audit_docs.py index 3636069fb..ce5f108ef 100755 --- a/.agents/skills/missing_docs/scripts/test_audit_docs.py +++ b/.agents/skills/missing_docs/scripts/test_audit_docs.py @@ -223,5 +223,131 @@ def test_map_hygiene_flags_unknown_gated_flag(self): self.assertEqual(gated_findings[0]["entry"], "oz bad") +class TestChangelogTriage(unittest.TestCase): + """Regression cases for the drift-watch triage baseline and section guard. + + All three failures these cover are silent: the run exits 0 and reports + nothing, which is indistinguishable from "nothing shipped". + """ + + ENTRY = ( + "### 2099.01.02 (v0.2099.01.02.00.00)\n\n" + "**New features**\n\n" + "* A tracked bullet.\n\n" + "**Automation Platform updates**\n\n" + "* A bullet under the post-rename platform heading.\n\n" + "**Bug fixes**\n\n" + "* Deliberately untracked.\n\n" + "**Sparkles**\n\n" + "* A bullet under a heading the parser has never seen.\n" + ) + + def _entries(self, text): + with tempfile.TemporaryDirectory() as d: + changelog = Path(d) / "src" / "content" / "docs" / "changelog" + changelog.mkdir(parents=True) + (changelog / "2099.mdx").write_text(text, encoding="utf-8") + return audit_docs.parse_changelog_entries(Path(d)) + + def test_normalize_release_version(self): + """The marker and the snapshot store versions in different shapes.""" + self.assertEqual( + audit_docs.normalize_release_version("v0.2026.08.18.02.52.stable_00"), + "2026.08.18") + self.assertEqual( + audit_docs.normalize_release_version("2026.08.19"), "2026.08.19") + self.assertIsNone(audit_docs.normalize_release_version(None)) + self.assertIsNone(audit_docs.normalize_release_version("not-a-version")) + + def test_baseline_uses_the_earlier_marker(self): + """A snapshot refresh must not skip a release that was never triaged. + + Bookkeeping PRs regenerate surface_snapshot.json, advancing its + changelog pointer without triaging anything. Taking the snapshot alone + as the baseline drops every entry in between. + """ + self.assertEqual( + audit_docs.changelog_review_baseline( + "2026.08.19", "v0.2026.08.18.02.52.stable_00"), + "2026.08.18") + # Either side missing falls back to the other rather than to "all seen". + self.assertEqual( + audit_docs.changelog_review_baseline("2026.08.19", None), "2026.08.19") + self.assertEqual( + audit_docs.changelog_review_baseline(None, "v0.2026.08.18.02.52.stable_00"), + "2026.08.18") + # No markers at all means nothing has been triaged; review everything. + self.assertIsNone(audit_docs.changelog_review_baseline(None, None)) + + def test_desynced_markers_still_surface_the_release(self): + """End to end: the snapshot ahead of the marker must not hide bullets.""" + entries = self._entries(self.ENTRY) + ahead = audit_docs.changelog_review_findings(entries, "2099.01.02") + self.assertEqual(ahead, [], "snapshot-only baseline hides the entry") + recovered = audit_docs.changelog_review_findings( + entries, "2099.01.02", "2099.01.01") + self.assertEqual(len(recovered), 2, recovered) + + def test_tracked_untracked_and_unknown_sections(self): + entry = self._entries(self.ENTRY)[0] + categories = sorted(i["category"] for i in entry["items"]) + self.assertEqual(categories, ["automation platform updates", "new features"]) + # Bug fixes are skipped on purpose, so they must not read as a rename. + self.assertEqual(entry["unknown_sections"], ["sparkles"]) + + def test_unknown_section_guard_ignores_already_triaged_history(self): + """Old launch posts use prose headings; policing them fails every run.""" + entries = self._entries(self.ENTRY) + self.assertEqual( + audit_docs.unreviewed_unknown_sections(entries, "2099.01.02"), [], + "entries at or below the baseline are already triaged") + self.assertEqual( + audit_docs.unreviewed_unknown_sections(entries, "2099.01.01"), + ["sparkles"], + "an unrecognized heading in a pending entry must fail loud") + + def test_marker_reader_survives_every_bad_shape(self): + """A malformed marker must degrade to "never triaged", never raise. + + Raising here aborts diff-mode triage entirely, which is a worse failure + than the desync this reader exists to fix. Valid JSON of the wrong + top-level type is the case that bites: `[]` reaches `.get` and throws. + """ + cases = { + "list": "[]", + "bare string": '"v0.2026.08.18.02.52.stable_00"', + "number": "42", + "null": "null", + "truncated json": '{"last_processed_version":', + "empty file": "", + "object, wrong key": '{"version": "v0.2026.08.18.02.52.stable_00"}', + } + for label, payload in cases.items(): + with self.subTest(shape=label), tempfile.TemporaryDirectory() as d: + refs = Path(d) + (refs / "last_release_processed.json").write_text( + payload, encoding="utf-8") + self.assertIsNone( + audit_docs.read_last_processed_release( + refs / "surface_snapshot.json"), + f"{label} marker should fall back to None") + + # A missing marker is the ordinary first-run case, not an error. + with tempfile.TemporaryDirectory() as d: + self.assertIsNone(audit_docs.read_last_processed_release( + Path(d) / "surface_snapshot.json")) + + # The well-formed case still reads through. + with tempfile.TemporaryDirectory() as d: + refs = Path(d) + (refs / "last_release_processed.json").write_text( + '{"last_processed_version": "v0.2026.08.18.02.52.stable_00"}', + encoding="utf-8") + self.assertEqual( + audit_docs.read_last_processed_release( + refs / "surface_snapshot.json"), + "2026.08.18") + + if __name__ == "__main__": unittest.main(verbosity=2)