From f2bd78635a492e5c018cd3893dea2ead0e03a6c7 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 16:48:38 -0400 Subject: [PATCH 1/3] build_search_index.py: make the hot-token path deterministic (reproducible chain step 7) Two builds of identical inputs produced different indexes (915 vs 901 shard files, 138 vs 137 hot tokens, 217 shared files with different bytes). Causes, all in the hot-token path: - est (avg posting width) was sampled with LIMIT 500000 and no ORDER BY, so the hot threshold moved between runs -> now averaged over all rows; - the hot set had no ORDER BY, so collision-suffixed hot/ keys were assigned in scan order -> ORDER BY token; - the per-shard 'heaviest token' promotion used ORDER BY n DESC LIMIT 1, so ties were arbitrary -> ORDER BY n DESC, token ASC; - hot_topk's per-pid static score was a parallel float sum -> sum(c ORDER BY field). Verified: two builds byte-identical across all 1,029 files (build_stats.json excluded: timestamps). Layout changes vs the old builder (1,025 shards) because the estimate is now over the full table; the reader contract is unchanged (hot_tokens.json remains the locator). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/build_search_index.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tools/build_search_index.py b/tools/build_search_index.py index 7f5c192c..553fe094 100644 --- a/tools/build_search_index.py +++ b/tools/build_search_index.py @@ -320,9 +320,13 @@ def flush() -> None: # Cap is enforced against parquet FILE bytes (what a browser actually # transfers), which is stricter than the contract's "uncompressed". cap_bytes = int(args.shard_cap_mb * 1024 * 1024) + # Reproducibility: the estimate is taken over ALL rows. A `LIMIT 500000` + # sample with no ORDER BY picked different rows on every run (parallel + # scan order), moving the hot threshold and changing the hot set/sub-shard + # layout between otherwise identical builds (found 2026-08-28, two builds + # of the same inputs: 915 vs 901 shard files). est = con.sql(f""" - SELECT avg(len(token) + len(pid) + len(field) + 4) FROM - (SELECT * FROM read_parquet('{rows_glob}') LIMIT 500000) + SELECT avg(len(token) + len(pid) + len(field) + 4) FROM read_parquet('{rows_glob}') """).fetchone()[0] or 30.0 # Planning ratio is only a first guess (observed file ratios vary by # token entropy); every hot token's sub-files are VERIFIED against the @@ -333,7 +337,8 @@ def flush() -> None: FROM read_parquet('{rows_glob}') GROUP BY token HAVING count(*) * {est} * {compress_ratio} > {cap_bytes} - """).fetchall() + ORDER BY token + """).fetchall() # ORDER BY: hot/ key collision suffixes are assigned in this order hot_dir = out_root / "hot" hot_manifest: dict[str, dict] = {} if hot: @@ -433,8 +438,8 @@ def write_hot_token(token: str, n: int) -> None: FROM read_parquet('{rows_glob}') WHERE shard = {shard} AND token NOT IN (SELECT token FROM hot_tokens) - GROUP BY token ORDER BY n DESC LIMIT 1 - """).fetchone() + GROUP BY token ORDER BY n DESC, token ASC LIMIT 1 + """).fetchone() # token ASC: a tie on n must promote the same token every run if heaviest is None: break con.execute("INSERT INTO hot_tokens VALUES (?)", [heaviest[0]]) @@ -502,7 +507,7 @@ def write_hot_token(token: str, n: int) -> None: JOIN field_avg fa USING (field) WHERE r.token IN ({hot_list_sql}) ), contrib AS ( - SELECT token, pid, + SELECT token, pid, field, (CASE field WHEN 'sample.label' THEN 3.0 WHEN 'concept.label' THEN 2.5 @@ -515,7 +520,9 @@ def write_hot_token(token: str, n: int) -> None: ), per_pid AS ( -- §5: rank per PID by the SUM of field-weighted -- contributions, not per (pid, field) posting. - SELECT token, pid, sum(c) AS static_score + -- sum(... ORDER BY field): floating-point addition order + -- must not depend on parallel aggregation order (reproducible bytes). + SELECT token, pid, sum(c ORDER BY field) AS static_score FROM contrib GROUP BY token, pid ), ranked AS ( SELECT *, row_number() OVER ( From 89ef0de74ba1f03c3920d89385a051e85b6b5e15 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 16:55:22 -0400 Subject: [PATCH 2/3] build_search_index.py: refuse a non-empty output index dir (--force replaces it); top_df tie-break (Codex round 1) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/build_search_index.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/build_search_index.py b/tools/build_search_index.py index 553fe094..39653795 100644 --- a/tools/build_search_index.py +++ b/tools/build_search_index.py @@ -42,6 +42,7 @@ import argparse import json import os +import shutil import sys import tempfile import time @@ -172,6 +173,7 @@ def main() -> int: # 256 default: the 202608 corpus yields ~570 MB of non-hot base rows; # 64 shards made ~9 MB base files against the 5 MB cap. 256 → ~2.2 MB # average with headroom for growth (v1.5 event/site fields). + ap.add_argument("--force", action="store_true", help="replace a non-empty output index directory") ap.add_argument("--shards", type=int, default=256) ap.add_argument("--shard-cap-mb", type=float, default=5.0) ap.add_argument("--batch-rows", type=int, default=200_000) @@ -179,6 +181,15 @@ def main() -> int: t0 = time.time() out_root = Path(args.outdir) / f"{args.tag}_search_index_v1" + # Reproducibility: the directory's byte inventory must come from THIS build only. + # A previous build could leave obsolete hot/ keys, higher _pN sub-files, or + # base shards beyond --shards, which no manifest would mention. Refuse a + # non-empty target unless --force, which removes it first. + if out_root.exists() and any(out_root.iterdir()): + if not args.force: + print(f'ERROR: {out_root} exists and is not empty; pass --force to replace it', file=sys.stderr) + return 2 + shutil.rmtree(out_root) out_root.mkdir(parents=True, exist_ok=True) con = duckdb.connect() @@ -574,7 +585,7 @@ def write_hot_token(token: str, n: int) -> None: top_df = con.sql(f""" SELECT token, count(*) AS df FROM read_parquet('{rows_glob}') - GROUP BY token ORDER BY df DESC LIMIT 20 + GROUP BY token ORDER BY df DESC, token ASC LIMIT 20 """).fetchall() total_uncompressed = con.sql(f""" SELECT sum(len(token) + len(pid) + len(field) + 4) From 70579393fca62a06807f8b7870bbc6234d1b46f3 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 28 Aug 2026 16:58:39 -0400 Subject: [PATCH 3/3] build_search_index.py: --tag must be one filename component; output target must resolve under --outdir (Codex round 2) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LtTxB4jfTZgaTR7CK4zKqy --- tools/build_search_index.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/build_search_index.py b/tools/build_search_index.py index 39653795..39887692 100644 --- a/tools/build_search_index.py +++ b/tools/build_search_index.py @@ -42,6 +42,7 @@ import argparse import json import os +import re import shutil import sys import tempfile @@ -180,7 +181,16 @@ def main() -> int: args = ap.parse_args() t0 = time.time() - out_root = Path(args.outdir) / f"{args.tag}_search_index_v1" + # --tag names a directory under --outdir and nothing else: one filename + # component, so --force can never remove anything outside --outdir. + if not re.fullmatch(r"[A-Za-z0-9._-]+", args.tag) or args.tag in (".", ".."): + print(f"ERROR: --tag must be a single filename component, got {args.tag!r}", file=sys.stderr) + return 2 + outdir = Path(args.outdir).resolve() + out_root = outdir / f"{args.tag}_search_index_v1" + if out_root.is_symlink() or out_root.resolve().parent != outdir: + print(f"ERROR: refusing output target {out_root} (symlink or outside --outdir)", file=sys.stderr) + return 2 # Reproducibility: the directory's byte inventory must come from THIS build only. # A previous build could leave obsolete hot/ keys, higher _pN sub-files, or # base shards beyond --shards, which no manifest would mention. Refuse a