From 690fb45957490fb4500d5ff3d08fcb57138b35e2 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 13:09:08 +0200 Subject: [PATCH 01/11] feat(memory): rank vault search instead of listing by date MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an FTS5 index over the vault so a search returns the page a question is about first, rather than every page that matched sorted by date. Folds umlauts, so "Kase" finds "Käse". Fuses a trigram index behind an opt-in flag, which is what reaches "Geburtstagsfeier" from "Feier". Each hit reports which of the query keywords it contains, so a caller can tell "found it" from "found the nearest page". Nothing calls this yet; the CLI still uses the regex walk. --- pyproject.toml | 6 + stacklets/memory/fts_index.py | 525 +++++++++++++++++++++++ tests/stacklets/test_memory_fts_index.py | 298 +++++++++++++ 3 files changed, 829 insertions(+) create mode 100644 stacklets/memory/fts_index.py create mode 100644 tests/stacklets/test_memory_fts_index.py diff --git a/pyproject.toml b/pyproject.toml index 4e4fc2b7..b4ce9a0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,6 +187,12 @@ extraPaths = ["lib", "stacklets/messages", "stacklets/messages/bot", "stacklets/ root = "stacklets/photos" extraPaths = ["lib", "stacklets/photos", "stacklets/core/bot-runner", "stacklets"] +[[tool.basedpyright.executionEnvironments]] +root = "tools/retrieval-lab" +# The lab drives both search engines head to head, so it bootstraps the +# memory stacklet's path the same way the CLI plugins there do. +extraPaths = ["lib", "stacklets/memory"] + [[tool.basedpyright.executionEnvironments]] root = "tests" extraPaths = ["lib", ".", "stacklets", "stacklets/core/bot-runner", "stacklets/agent/bot", "stacklets/core/bot", "stacklets/core/tools-server", "stacklets/docs/bot", "stacklets/memory/bot", "stacklets/memory/bot/cli", "stacklets/messages/bot"] diff --git a/stacklets/memory/fts_index.py b/stacklets/memory/fts_index.py new file mode 100644 index 00000000..8a0452e1 --- /dev/null +++ b/stacklets/memory/fts_index.py @@ -0,0 +1,525 @@ +"""A ranked full-text index over the memory vault. + +`search_memory` in `lib.py` walks every `*.md` in the vault with a +regex and sorts the matches by frontmatter date. That has no notion of +a better match: a shopping list that needs "batteries for the camping +lamp" and the page about the camping trip come back in whatever order +they were written, and the model upstream has to read both. It also +strips frontmatter before matching, so who a page is about and what it +is tagged with are invisible to the query, and it is byte-literal, so a +family typing "Kase" never reaches "Käse". + +This module is the ranked alternative. One SQLite file next to the +vault holds the pages and an FTS5 index over them; BM25 orders the +results; the tokenizer folds diacritics. Frontmatter becomes columns, +which makes persons and tags both searchable and weightable instead of +noise to be stripped. + +The index is a derived cache. It sits beside the vault working copy, +is never committed, and can be deleted at any time -- `build_index` +rebuilds whatever is missing and reconciles whatever moved, so the +authoritative answer is always the git checkout. + + build_index(vault, db) -> Stats reconcile the index with the vault + search(db, keywords, ...) -> [Hit] rank pages against model keywords + +Two design points worth knowing before reading the code: + +**Keywords, not a regex.** The caller passes the 2-4 words the model +produced, not a pattern. They are untrusted text, so every token is +quoted before it reaches FTS5 -- otherwise a model answering `NOT` or +`C++` would either invert the query or fail to parse it. + +**Every hit reports its own coverage.** `Hit.matched` names which of +the query's keywords that page actually contains. A BM25 score is only +comparable within one result set, so it cannot answer "is this fact in +the vault at all"; coverage can, and that is the input a caller needs +to decide between answering and saying it did not find anything. +""" + +from __future__ import annotations + +import hashlib +import re +import sqlite3 +import sys +import unicodedata +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Optional, Sequence + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from lib import _fm_list, _norm_tag, body_only # noqa: E402 +from stack.frontmatter import parse as parse_frontmatter # noqa: E402 + + +# Prefix matching is what lets "Geburtstag" reach "Geburtstagsfeier", +# but on a two-letter token it degenerates into matching most of the +# vault. Below this length a token has to match a whole word. +MIN_PREFIX_LEN = 3 + +# BM25 column weights: title, persons, tags, body. A word in the title +# or in the tags says the page is *about* that thing; the same word in +# the body may be a passing mention. Persons sits with tags because a +# name in the frontmatter is a fact about the page, not prose. +WEIGHTS = (8.0, 4.0, 4.0, 1.0) + + +@dataclass(frozen=True) +class Hit: + """One ranked page, in the shape the CLI formatter already prints.""" + + rel: str + title: str + date: str + persons: List[str] + tags: List[str] + excerpt: str + score: float + matched: tuple[str, ...] + + +@dataclass(frozen=True) +class Stats: + """What one reconcile pass did, for callers that log or test it.""" + + added: int + updated: int + deleted: int + total: int + + +# ── text normalisation ────────────────────────────────────────────────── + +def fold(text: str) -> str: + """Lower-case and strip combining marks, as `remove_diacritics 2` does. + + The tokenizer folds the indexed side; this folds the Python side so + that coverage (`Hit.matched`) agrees with what the index matched. + Decomposing first is what makes it work for both spellings of an + umlaut -- a precomposed "ü" and a "u" plus a combining diaeresis + are the same word to a family and must be the same token here. + """ + decomposed = unicodedata.normalize("NFD", text.lower()) + return "".join(c for c in decomposed if not unicodedata.combining(c)) + + +def tokens(text: str) -> List[str]: + """Split folded text into the alphanumeric runs unicode61 would produce.""" + return [t for t in re.split(r"[^0-9a-z]+", fold(text)) if t] + + +# ── schema ────────────────────────────────────────────────────────────── + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS docs( + id INTEGER PRIMARY KEY, + path TEXT UNIQUE, + title TEXT, + persons TEXT, + tags TEXT, + date TEXT, + mtime REAL, + sha1 TEXT, + body TEXT, + persons_norm TEXT, + tags_norm TEXT +); + +CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5( + title, persons, tags, body, + content='docs', content_rowid='id', + tokenize="unicode61 remove_diacritics 2" +); + +CREATE VIRTUAL TABLE IF NOT EXISTS tri USING fts5( + title, persons, tags, body, + content='docs', content_rowid='id', + tokenize="trigram remove_diacritics 1" +); +""" + +# The word index and the substring index, in the order results are fused. +INDEXES = ("fts", "tri") + +# Reciprocal rank fusion constant. 60 is the value from the original +# paper and the one the retrieval handover names for tier 2; nothing +# here is tuned to this corpus. +RRF_K = 60 + + +def _connect(db: Path) -> sqlite3.Connection: + db.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA) + return conn + + +# ── indexing ──────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class _Doc: + """A vault page reduced to the columns the index stores.""" + + path: str + title: str + persons: str + tags: str + date: str + mtime: float + sha1: str + body: str + persons_norm: str + tags_norm: str + + +def _read(md_path: Path, rel: str, mtime: float) -> Optional[_Doc]: + """Turn one file into a row, or None when it cannot be read. + + An unreadable page is skipped rather than fatal: the vault is a + working copy that a sync may be rewriting underneath us, and one + bad file must not cost the family every other search result. + """ + try: + text = md_path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return None + fm = parse_frontmatter(text) if text.startswith("---\n") else {} + persons = _fm_list(fm, "persons") + tags = _fm_list(fm, "tags") + body = body_only(text) + return _Doc( + path=rel, + title=str(fm.get("title") or md_path.stem), + persons=" ".join(persons), + tags=" ".join(tags), + date=str(fm.get("date") or ""), + mtime=mtime, + sha1=hashlib.sha1(text.encode("utf-8")).hexdigest(), + body=body, + # Wrapped in pipes so a SQL `LIKE '%|homer|%'` cannot match + # "homerette" -- the same reason `search_memory` normalises + # scope prefixes with a trailing slash. + persons_norm="|" + "|".join(p.lower() for p in persons) + "|", + tags_norm="|" + "|".join(_norm_tag(t) for t in tags) + "|", + ) + + +def _fts_delete(conn: sqlite3.Connection, row: sqlite3.Row) -> None: + """Retract a row from both indexes using its *old* column values. + + External-content FTS5 keeps no copy of the text, so it cannot work + out what to remove on its own. Handing it the current values is the + documented protocol; handing it the new ones would leave the old + terms in the index, and the page would keep answering searches for + a sentence it no longer contains. + """ + for table in INDEXES: + conn.execute( + f"INSERT INTO {table}({table}, rowid, title, persons, tags, body) " + "VALUES('delete', ?, ?, ?, ?, ?)", + (row["id"], row["title"], row["persons"], row["tags"], + row["body"]), + ) + + +def _fts_insert(conn: sqlite3.Connection, doc_id: int, doc: _Doc) -> None: + for table in INDEXES: + conn.execute( + f"INSERT INTO {table}(rowid, title, persons, tags, body)" + " VALUES(?,?,?,?,?)", + (doc_id, doc.title, doc.persons, doc.tags, doc.body), + ) + + +def build_index(vault: Path, db: Path) -> Stats: + """Reconcile the index with the vault, and report what changed. + + Cheap when nothing moved, which is the common case: this runs on + every search, so the no-change path compares mtime against the + stored value and reads no file at all. A page whose mtime differs + is hashed before it is re-indexed, because git checkouts and clone + refreshes rewrite mtimes on files whose content is identical. + """ + conn = _connect(db) + try: + known = { + row["path"]: row + for row in conn.execute( + "SELECT id, path, title, persons, tags, body, mtime, sha1 " + "FROM docs" + ) + } + added = updated = 0 + seen: set[str] = set() + + for md_path in sorted(vault.rglob("*.md")): + if not md_path.is_file(): + continue + rel = str(md_path.relative_to(vault)) + seen.add(rel) + mtime = md_path.stat().st_mtime + row = known.get(rel) + if row is not None and row["mtime"] == mtime: + continue + + doc = _read(md_path, rel, mtime) + if doc is None: + continue + if row is None: + cur = conn.execute( + "INSERT INTO docs(path, title, persons, tags, date, mtime," + " sha1, body, persons_norm, tags_norm)" + " VALUES(?,?,?,?,?,?,?,?,?,?) RETURNING id", + (doc.path, doc.title, doc.persons, doc.tags, doc.date, + doc.mtime, doc.sha1, doc.body, doc.persons_norm, + doc.tags_norm), + ) + _fts_insert(conn, int(cur.fetchone()[0]), doc) + added += 1 + continue + + if row["sha1"] == doc.sha1: + # Same bytes, new mtime. Store the timestamp so the next + # scan short-circuits, but leave the index alone. + conn.execute("UPDATE docs SET mtime=? WHERE id=?", + (mtime, row["id"])) + continue + + _fts_delete(conn, row) + conn.execute( + "UPDATE docs SET title=?, persons=?, tags=?, date=?, mtime=?," + " sha1=?, body=?, persons_norm=?, tags_norm=? WHERE id=?", + (doc.title, doc.persons, doc.tags, doc.date, doc.mtime, + doc.sha1, doc.body, doc.persons_norm, doc.tags_norm, + row["id"]), + ) + _fts_insert(conn, int(row["id"]), doc) + updated += 1 + + gone = [row for path, row in known.items() if path not in seen] + for row in gone: + _fts_delete(conn, row) + conn.execute("DELETE FROM docs WHERE id=?", (row["id"],)) + + conn.commit() + total = conn.execute("SELECT COUNT(*) FROM docs").fetchone()[0] + return Stats(added=added, updated=updated, deleted=len(gone), + total=int(total)) + finally: + conn.close() + + +# ── querying ──────────────────────────────────────────────────────────── + +def _match_expression(keywords: Sequence[str]) -> str: + """Render model keywords as an FTS5 MATCH expression, or "" for none. + + Every token is wrapped in double quotes before it is joined. That is + not cosmetic: unquoted, a model's `NOT`, `OR`, `*` or stray `"` + would be read as the query language rather than as words, which + turns a bad keyword into a silently inverted search or a syntax + error. Quoted, the worst case is a word that matches nothing. + """ + parts: List[str] = [] + for keyword in keywords: + for token in tokens(keyword): + star = "*" if len(token) >= MIN_PREFIX_LEN else "" + parts.append(f'"{token}"{star}') + return " OR ".join(parts) + + +def _trigram_expression(keywords: Sequence[str]) -> str: + """The same keywords as substring searches, for the trigram index. + + No trailing `*`: a trigram index already matches anywhere inside a + word, which is the point of consulting it. Tokens under three + characters are dropped rather than passed through, because a + trigram index cannot match them at all and they would only cost a + scan. + """ + parts: List[str] = [] + for keyword in keywords: + for token in tokens(keyword): + if len(token) >= MIN_PREFIX_LEN: + parts.append(f'"{token}"') + return " OR ".join(parts) + + +def covers(keyword: str, doc_tokens: set[str]) -> bool: + """Does this page contain the keyword, under the same rules as MATCH? + + Public because coverage has to be computable for pages this module + did not rank -- the retrieval lab scores the regex engine's hits the + same way, and a second implementation of the rule would make that + comparison meaningless. + """ + kw_tokens = tokens(keyword) + if not kw_tokens: + return False + for token in kw_tokens: + if len(token) >= MIN_PREFIX_LEN: + if any(d.startswith(token) for d in doc_tokens): + return True + elif token in doc_tokens: + return True + return False + + +def _scope_clause(scopes: Sequence[str]) -> tuple[str, List[str]]: + """`rel LIKE 'family/%'` for each prefix, OR-combined. + + Prefixes gain a trailing slash first, so `marge` cannot reach into + `margery/` -- same rule as `search_memory`, kept identical because + the archivist relies on it to keep an unknown sender out of the + personal buckets. + """ + prefixes = [s if s.endswith("/") else f"{s}/" for s in scopes] + clause = " OR ".join("docs.path LIKE ?" for _ in prefixes) + return f"({clause})", [f"{p}%" for p in prefixes] + + +def _axis_clause(column: str, values: Iterable[str]) -> tuple[str, List[str]]: + """OR within one frontmatter axis, against the pipe-wrapped column.""" + wanted = list(values) + clause = " OR ".join(f"docs.{column} LIKE ?" for _ in wanted) + return f"({clause})", [f"%|{v}|%" for v in wanted] + + +def _filters( + persons: Optional[Sequence[str]], + tags: Optional[Sequence[str]], + scopes: Optional[Sequence[str]], +) -> tuple[List[str], List[object]]: + """The frontmatter and scope narrowing, shared by both indexes.""" + where: List[str] = [] + params: List[object] = [] + if persons: + clause, values = _axis_clause( + "persons_norm", (p.lower() for p in persons)) + where.append(clause) + params += values + if tags: + clause, values = _axis_clause("tags_norm", (_norm_tag(t) for t in tags)) + where.append(clause) + params += values + if scopes: + clause, values = _scope_clause(scopes) + where.append(clause) + params += values + return where, params + + +def _ranked(conn: sqlite3.Connection, table: str, expression: str, + filters: tuple[List[str], List[object]], + limit: int) -> List[sqlite3.Row]: + """One index's best `limit` rows for this expression, best first.""" + where, filter_params = filters + sql = ( + "SELECT docs.path, docs.title, docs.date, docs.persons, docs.tags," + " docs.body," + f" snippet({table}, 3, '', '', ' … ', 16) AS excerpt," + f" bm25({table}, ?, ?, ?, ?) AS score" + f" FROM {table} JOIN docs ON docs.id = {table}.rowid" + f" WHERE {' AND '.join([f'{table} MATCH ?', *where])}" + " ORDER BY score LIMIT ?" + ) + try: + return conn.execute( + sql, [*WEIGHTS, expression, *filter_params, limit]).fetchall() + except sqlite3.OperationalError: + # A MATCH expression this module built should always parse. + # Staying quiet here keeps the failure mode identical to the + # regex engine's, which returns [] on a pattern it cannot + # compile rather than taking the caller down. + return [] + + +def _fuse(lists: Sequence[Sequence[sqlite3.Row]], + limit: int) -> List[tuple[sqlite3.Row, float]]: + """Reciprocal rank fusion over one or more ranked lists. + + RRF combines rankings without needing their scores to mean the same + thing, which matters here: a BM25 over words and a BM25 over + character trigrams are not on one scale and averaging them would be + arithmetic on unrelated numbers. Each list contributes `1/(k+rank)`, + so a page both indexes like beats a page only one of them found. + """ + scores: dict[str, float] = {} + rows: dict[str, sqlite3.Row] = {} + for ranking in lists: + for position, row in enumerate(ranking, start=1): + path = row["path"] + scores[path] = scores.get(path, 0.0) + 1.0 / (RRF_K + position) + rows.setdefault(path, row) + ordered = sorted(scores.items(), key=lambda kv: -kv[1])[:limit] + return [(rows[path], score) for path, score in ordered] + + +def search( + db: Path, + keywords: Sequence[str], + persons: Optional[Sequence[str]] = None, + tags: Optional[Sequence[str]] = None, + scopes: Optional[Sequence[str]] = None, + limit: int = 20, + substrings: bool = False, +) -> List[Hit]: + """Rank the vault against `keywords`, best first. + + The filters carry the same contract as `search_memory`: persons and + tags are OR within an axis and AND across axes, `scopes=None` means + the whole vault and `scopes=[]` means nothing at all. An empty + keyword list returns nothing rather than everything -- a caller who + got no keywords out of the model has no query, and returning the + newest twenty pages would look like an answer. + + `substrings` also consults the trigram index and fuses the two + rankings. The word index matches from the front of a word, which + reaches "Geburtstagsfeier" from "Geburtstag" but not from "Feier"; + German puts the noun the family asks with at either end of a + compound, so the substring index covers the half prefixes cannot. + It costs a second query and a larger index, which is why it is a + parameter rather than the default until the measurement says + otherwise. + """ + if scopes is not None and not scopes: + return [] + expression = _match_expression(keywords) + if not expression: + return [] + + filters = _filters(persons, tags, scopes) + conn = _connect(db) + try: + word_hits = _ranked(conn, "fts", expression, filters, limit) + if not substrings: + scored = [(row, -float(row["score"])) for row in word_hits] + else: + tri_expression = _trigram_expression(keywords) + tri_hits = (_ranked(conn, "tri", tri_expression, filters, limit) + if tri_expression else []) + scored = _fuse([word_hits, tri_hits], limit) + finally: + conn.close() + + hits: List[Hit] = [] + for row, score in scored: + doc_tokens = set(tokens( + " ".join((row["title"], row["persons"], row["tags"], row["body"])))) + hits.append(Hit( + rel=row["path"], + title=row["title"], + date=row["date"] or "", + persons=row["persons"].split() if row["persons"] else [], + tags=row["tags"].split() if row["tags"] else [], + excerpt=(row["excerpt"] or "").strip(), + # Higher is better either way: bm25() is negative and was + # flipped, RRF is already a positive sum. The two are not + # on one scale, so a score is only comparable inside one + # result set -- which is why coverage exists below. + score=score, + matched=tuple(k for k in keywords if covers(k, doc_tokens)), + )) + return hits diff --git a/tests/stacklets/test_memory_fts_index.py b/tests/stacklets/test_memory_fts_index.py new file mode 100644 index 00000000..4f2d3415 --- /dev/null +++ b/tests/stacklets/test_memory_fts_index.py @@ -0,0 +1,298 @@ +"""The ranked search index behind `stack memory search`. + +Today's engine is a regex walk: it matches lines and sorts what it finds +by date, so "who was worried about bugs on the trip" reaches the right +page only when the family happened to write those words. This index is +the ranked alternative -- one SQLite FTS5 table over the vault, BM25 for +the order, the frontmatter promoted to columns that can be weighted and +filtered instead of stripped as noise. + +These tests drive it the way the CLI will: build an index over a vault +directory, hand it the 2-4 keywords the model produces, read the hits +back. They pin the promises the regex engine already makes (frontmatter +field names are not content, persons/tags/scopes narrow the same way) +plus the three this index adds: diacritics fold for a German-speaking +family, better matches come first, and every hit says which of the +query's keywords it actually contains -- the raw material for deciding +whether an answer is in the vault at all. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_REPO_ROOT / "lib")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory")) + +import fts_index # noqa: E402 + + +# ── fixtures ──────────────────────────────────────────────────────────── + +def page(vault: Path, rel: str, *, title: str, body: str, + persons: list[str] | None = None, + tags: list[str] | None = None, + date: str = "2026-09-01") -> Path: + """Write one vault page, frontmatter and all, the way the archivist does.""" + path = vault / rel + path.parent.mkdir(parents=True, exist_ok=True) + fm = [f"title: {title}", f"date: {date}", "type: note"] + if persons: + fm.append("persons:") + fm += [f" - {p}" for p in persons] + if tags: + fm.append("tags:") + fm += [f" - {t}" for t in tags] + path.write_text("---\n" + "\n".join(fm) + "\n---\n\n" + body + "\n", + encoding="utf-8") + return path + + +@pytest.fixture +def vault(tmp_path: Path) -> Path: + """A small vault with the shapes the engine has to tell apart. + + Two pages mention camping; only one is about it. One page is + German, one sits in a personal bucket, one belongs to nobody in + particular. That is enough to ask ranking, folding and scope + questions of. + """ + v = tmp_path / "vault" + page(v, "family/camping/about.md", + title="Camping trip", + body="The tent leaks. We bought a new one before the trip.", + persons=["homer", "bart"], tags=["camping"]) + page(v, "family/groceries/todos.md", + title="Shopping list", + body="- [ ] milk\n- [ ] batteries for the camping lamp", + persons=["marge"], tags=["groceries"]) + page(v, "family/birthday/about.md", + title="Omas Geburtstagsfeier", + body="Wir backen einen Kuchen mit Käse und grünem Zuckerguss.", + persons=["marge", "lisa"], tags=["birthday"]) + page(v, "marge/notes/gift-ideas.md", + title="Gift ideas", + body="A telescope for Lisa. Do not tell her.", + persons=["marge"], tags=["private"]) + return v + + +@pytest.fixture +def index(vault: Path, tmp_path: Path) -> Path: + db = tmp_path / "vault-index.sqlite3" + fts_index.build_index(vault, db) + return db + + +def rels(hits) -> list[str]: + return [h.rel for h in hits] + + +# ── what the index promises ───────────────────────────────────────────── + +def test_a_word_from_a_page_body_finds_that_page(index): + """The floor: anything written in the vault can be searched for.""" + assert rels(fts_index.search(index, ["telescope"])) == [ + "marge/notes/gift-ideas.md" + ] + + +def test_frontmatter_field_names_are_not_searchable_content(index): + """`date` is structure, not something the family wrote. + + The regex engine strips frontmatter for exactly this reason: a + query for "date" or "tags" would otherwise match every page in the + vault. Promoting the *values* to columns must not promote the + *keys* with them. + """ + assert fts_index.search(index, ["date"]) == [] + assert fts_index.search(index, ["type"]) == [] + + +def test_frontmatter_values_are_searchable(index): + """Who and what a page is about is part of the page. + + The regex engine throws this away, so "groceries" only finds pages + that happen to say the word in prose. Here the persons and tags + columns are indexed, which is the upgrade. + """ + hits = fts_index.search(index, ["groceries"]) + assert "family/groceries/todos.md" in rels(hits) + + +def test_german_diacritics_fold_to_their_base_letter(index): + """A family that types "Kase" on a phone keyboard still finds "Käse". + + `unicode61 remove_diacritics 2` is the tokenizer setting that does + this, and it has to work in both directions -- the query may carry + the umlaut and the page may not, or the reverse. + """ + assert rels(fts_index.search(index, ["Kase"])) == [ + "family/birthday/about.md" + ] + assert rels(fts_index.search(index, ["grunem"])) == [ + "family/birthday/about.md" + ] + assert rels(fts_index.search(index, ["Käse"])) == [ + "family/birthday/about.md" + ] + + +def test_a_compound_reaches_its_base_word(index): + """"Geburtstag" has to reach a page that only says "Geburtstagsfeier". + + German compounds are the reason a plain token match is not enough + for this family. Prefix matching covers the direction that matters + most -- the short word the family asks with, against the long word + the page happens to use. + """ + assert rels(fts_index.search(index, ["Geburtstag"])) == [ + "family/birthday/about.md" + ] + + +def test_the_back_half_of_a_compound_needs_the_substring_index(index): + """The half prefix matching cannot reach. + + German puts the word the family asks with at either end of a + compound. "Geburtstag" finds "Geburtstagsfeier" because the page + word starts with the query word; "Feier" does not, because nothing + in a word index starts there. The regex engine gets this case right + by accident -- it matches substrings -- so shipping the word index + alone would be a regression for a German-speaking family. + """ + assert fts_index.search(index, ["Feier"]) == [] + assert rels(fts_index.search(index, ["Feier"], substrings=True)) == [ + "family/birthday/about.md" + ] + + +def test_the_substring_index_still_ranks_the_right_page_first(index): + """Fusing a second ranking must not scramble the first one. + + A substring index matches inside words, so it finds more and means + less. If switching it on cost the ordering the word index gets + right, it would be trading one failure for another. + """ + hits = fts_index.search(index, ["camping"], substrings=True) + assert rels(hits)[0] == "family/camping/about.md" + + +def test_the_page_a_query_is_about_outranks_a_passing_mention(index): + """This is the whole point of the change. + + Two pages contain "camping": the trip page and a shopping list that + needs batteries for a camping lamp. The regex engine returns both + in date order and leaves the choosing to the model. BM25 puts the + trip first, because the word carries the page's title and tag + rather than one line of a list. + """ + hits = fts_index.search(index, ["camping"]) + assert rels(hits)[0] == "family/camping/about.md" + assert "family/groceries/todos.md" in rels(hits) + + +def test_a_hit_says_which_of_the_query_keywords_it_matched(index): + """The signal for "is the answer even in here". + + A question whose keywords all land on one page is a different + situation from one where the best hit only matched a single generic + word, and the caller cannot tell those apart from a BM25 score + alone -- the score is only comparable inside one result set, so it + says nothing about whether the fact is in the vault. Each hit + reports its own coverage so that decision can be made on evidence. + """ + hits = {h.rel: h for h in + fts_index.search(index, ["camping", "tent", "telescope"])} + assert set(hits["family/camping/about.md"].matched) == {"camping", "tent"} + assert set(hits["marge/notes/gift-ideas.md"].matched) == {"telescope"} + assert set(hits["family/groceries/todos.md"].matched) == {"camping"} + + +def test_model_supplied_keywords_are_never_read_as_query_syntax(index): + """The keywords come from a model, so they are untrusted input. + + FTS5 has an operator language -- `NOT`, `OR`, `*`, quotes, column + filters. A model that answers `C++` or `NOT` or a stray quote must + produce a search, not a syntax error and not an inverted query. + """ + for hostile in (["NOT"], ['tent" OR "x'], ["C++"], ["*"], ["-"], + ["body:tent"], ["("]): + fts_index.search(index, hostile) # must not raise + + # Read as the operator it looks like, `NOT tent` would exclude the + # one page the family is asking about. Read as a word, it is just + # another term, and the tent page still comes back. + assert "family/camping/about.md" in rels( + fts_index.search(index, ["NOT", "tent"])) + # `title:tent` is a column filter in FTS5's grammar, and as one it + # finds nothing: no page is titled "tent". Quoted, it is two words, + # one of which is on the camping page. + assert "family/camping/about.md" in rels( + fts_index.search(index, ["title:tent"])) + + +def test_a_query_that_matches_nothing_returns_no_hits(index): + """No results is an answer, not a failure.""" + assert fts_index.search(index, ["snowmobile"]) == [] + + +def test_filters_narrow_the_same_way_the_regex_engine_does(index): + """Parity with `search_memory`, because callers already rely on it. + + Persons and tags are OR within an axis and AND across axes. Scopes + are path prefixes, where `None` means the whole vault and an empty + list means nothing at all -- that is how the archivist denies an + unknown sender access to personal buckets. + """ + assert rels(fts_index.search(index, ["telescope"], persons=["homer"])) == [] + assert rels(fts_index.search(index, ["telescope"], persons=["marge"])) == [ + "marge/notes/gift-ideas.md" + ] + assert rels(fts_index.search(index, ["telescope"], tags=["camping"])) == [] + + scoped = rels(fts_index.search(index, ["milk", "tent"], scopes=["family"])) + assert scoped and all(r.startswith("family/") for r in scoped) + assert fts_index.search(index, ["telescope"], scopes=[]) == [] + + +def test_an_edited_page_is_reindexed_and_a_deleted_one_disappears(vault, index): + """The index is a cache over a git checkout that moves under it. + + A rebuild has to pick up an edit, drop a removed page, and add a new + one, without the stale text surviving in the index -- a search that + still returns yesterday's sentence is worse than no index. + """ + (vault / "marge/notes/gift-ideas.md").write_text( + "---\ntitle: Gift ideas\ndate: 2026-09-02\n---\n\nA microscope instead.\n", + encoding="utf-8") + (vault / "family/groceries/todos.md").unlink() + page(vault, "bart/notes/skateboard.md", + title="Skateboard", body="The deck cracked.", persons=["bart"]) + + fts_index.build_index(vault, index) + + assert fts_index.search(index, ["telescope"]) == [] + assert rels(fts_index.search(index, ["microscope"])) == [ + "marge/notes/gift-ideas.md" + ] + assert fts_index.search(index, ["milk"]) == [] + assert rels(fts_index.search(index, ["skateboard"])) == [ + "bart/notes/skateboard.md" + ] + + +def test_an_unchanged_vault_costs_no_rewrites(vault, index): + """Rebuild runs on every search, so a no-change scan has to be cheap. + + The stats say what the scan did. Nothing changed, so nothing should + have been written -- if this reports work, the change detection is + broken and every search pays a full reindex. + """ + stats = fts_index.build_index(vault, index) + assert (stats.added, stats.updated, stats.deleted) == (0, 0, 0) + assert stats.total == 4 From 32ceaba52cb4f60ed3f1050dd5f92ee8629706c3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 13:09:13 +0200 Subject: [PATCH 02/11] test(memory): add a bench that compares the search engines An offline A/B over a fabricated 177-page vault and 47 questions, including 12 whose answer is not in the vault at all. Reports recall, ranking and latency per kind of question, and how well each candidate confidence signal tracks whether the top hit was actually right. Runs in about two seconds with no model and no containers, so an engine decision costs seconds rather than LLM calls. Questions are written from a neutral truth line rather than from the page they have to find, and both engines get the same mechanically extracted keywords. --- tools/retrieval-lab/.gitignore | 1 + tools/retrieval-lab/README.md | 83 +++++ tools/retrieval-lab/corpus.yaml | 531 +++++++++++++++++++++++++++++++ tools/retrieval-lab/evaluate.py | 522 ++++++++++++++++++++++++++++++ tools/retrieval-lab/explain.py | 41 +++ tools/retrieval-lab/generate.py | 111 +++++++ tools/retrieval-lab/goldset.yaml | 283 ++++++++++++++++ 7 files changed, 1572 insertions(+) create mode 100644 tools/retrieval-lab/.gitignore create mode 100644 tools/retrieval-lab/README.md create mode 100644 tools/retrieval-lab/corpus.yaml create mode 100644 tools/retrieval-lab/evaluate.py create mode 100644 tools/retrieval-lab/explain.py create mode 100644 tools/retrieval-lab/generate.py create mode 100644 tools/retrieval-lab/goldset.yaml diff --git a/tools/retrieval-lab/.gitignore b/tools/retrieval-lab/.gitignore new file mode 100644 index 00000000..89f9ac04 --- /dev/null +++ b/tools/retrieval-lab/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/tools/retrieval-lab/README.md b/tools/retrieval-lab/README.md new file mode 100644 index 00000000..082cf039 --- /dev/null +++ b/tools/retrieval-lab/README.md @@ -0,0 +1,83 @@ +# retrieval-lab + +An offline A/B bench for `stack memory search`. It runs the engines +against the same questions over the same fabricated vault and prints +what each one found, how fast, and how well its own confidence tracked +whether it was right. + +No model, no containers, no network. One run is about two seconds, which +is the point: the agent rig measures how the family experiences an +answer, and this measures the retrieval underneath it. Use this one +first, because it changes an engine decision in seconds rather than in +LLM calls. + +## Run it + +```bash +python3 tools/retrieval-lab/generate.py # render the vault +uv run --extra test python tools/retrieval-lab/evaluate.py # measure +uv run --extra test python tools/retrieval-lab/explain.py compound_tail +``` + +`generate.py` needs `pyyaml`, so run it under `uv run --extra test` too +if your system python3 lacks it. Output lands in `out/`, which is +gitignored and disposable. + +## What is in here + +| File | What it is | +|---|---| +| `corpus.yaml` | 27 answer pages plus the vocabulary for 150 distractors | +| `goldset.yaml` | 47 questions: 35 answerable, 12 with no answer in the vault | +| `generate.py` | renders `corpus.yaml` into `out/vault/` | +| `evaluate.py` | runs every engine over every question, prints the tables | +| `explain.py` | one question kind, side by side, for reading a regression | + +## The engines it compares + +| Arm | What it is | +|---|---| +| `regex` | today's `search_memory`: keywords OR'd into a regex, sorted by date | +| `fts5` | SQLite FTS5 over the vault, BM25 ranking, `unicode61 remove_diacritics 2` | +| `fts5+tri` | the above fused by RRF with a trigram (substring) index | + +All three live behind `stacklets/memory/fts_index.py` and +`stacklets/memory/lib.py`. Nothing is reimplemented here, so a +measurement is about the shipping code and not about a copy of it. + +## How the corpus avoids flattering the engine + +A gold set written while looking at the pages it has to find will share +their vocabulary, and then any lexical engine looks better than it is. +Three rules keep that honest: + +1. **Every fact page carries a `truth` line** stating what happened, in + neutral words. The page body says the same thing the way somebody + would actually have typed it. +2. **The questions in `goldset.yaml` are written from the truth lines + only.** If you add a question, read the truth and not the body. +3. **Keywords are extracted mechanically** from the question by dropping + stopwords. Both engines get the identical list. A hand-written + keyword list is a thumb on the scale even when nobody means it to be. + +Questions are tagged by kind (`paraphrase`, `compound_tail`, `fold`, +`absent`, ...) so a result can be traced to a case rather than being one +number that moved. `absent` questions have no answer in the vault at +all; getting those wrong is how a family ends up with a confident answer +about something nobody ever wrote down. + +## What it does not measure + +- **The model's rewrite.** Production `--nl` asks a model for keywords; + this uses mechanical extraction so the engine is the only variable. + Whether a model's keywords change the ranking is an agent-rig + question. +- **Cross-language retrieval.** A German question reaches German pages. + No lexical engine translates, and mixing the two in would measure + translation rather than ranking. +- **What the family finally reads.** Retrieval feeding a good answer is + a separate step, measured in `tools/agent-lab/rig/`. + +Sample sizes are tens of questions, not thousands. `evaluate.py` prints +95% intervals on the confidence rates for that reason; read those before +treating a two-point difference as real. diff --git a/tools/retrieval-lab/corpus.yaml b/tools/retrieval-lab/corpus.yaml new file mode 100644 index 00000000..07264c46 --- /dev/null +++ b/tools/retrieval-lab/corpus.yaml @@ -0,0 +1,531 @@ +# The fabricated vault the retrieval lab measures against. +# +# Each entry under `facts` is one page the family wrote, plus a `truth` +# line stating what actually happened. The page prose and the truth line +# are written to disagree in wording on purpose: the truth is what a +# family member remembers, the body is what somebody typed at the time. +# `goldset.yaml` is written from the truth lines alone, never from the +# bodies, so a query is not quietly reverse-engineered from the text it +# is supposed to find. +# +# German pages carry real umlauts, because that is what a family types. +# Queries in the gold set sometimes do not -- a phone keyboard, a hurry, +# an English layout -- and whether the engine survives that is one of +# the things being measured. +# +# Simpsons only, half German, as the rest of the demo data is. Nothing +# here comes from a real household. + +persons: [homer, marge, bart, lisa, maggie] + +facts: + # ── camping ────────────────────────────────────────────────────────── + - id: mosquitoes + scope: family/camping + slug: lake-weekend + lang: en + date: 2026-07-14 + persons: [marge, lisa] + tags: [camping, travel] + title: Lake weekend + truth: Lisa did not want to sleep outside because of the insects at the lake. + body: | + Two nights at Lake Springfield. Marge drove, we were there by four. + + Lisa refused to sleep in the tent after dark. The mosquitoes near + the reeds were unbearable and she had bites on both arms by the + second evening. She slept in the car instead. + + Next time: a net for the tent door, and a pitch away from the water. + + - id: stove-broken + scope: family/camping + slug: gear-check + lang: en + date: 2026-06-02 + persons: [homer, bart] + tags: [camping, todos] + title: Gear check before the trip + truth: The camping stove stopped working and Homer arranged to use the neighbour's. + body: | + Went through the boxes in the garage. + + The stove will not light any more. The regulator is corroded and a + new one costs more than the thing is worth. Ned has a spare + two-burner he never uses, so we take his. + + Tent, cooler box and both sleeping bags are fine. + + - id: campsite-booking + scope: family/camping + slug: booking + lang: de + date: 2026-05-20 + persons: [marge] + tags: [camping, travel] + title: Stellplatz reserviert + truth: The campsite is booked for the second week of August with a deposit already paid. + body: | + Der Platz am Waldrand ist für die zweite Augustwoche reserviert. + + Anzahlung ist raus, 80 Euro, der Rest wird vor Ort bezahlt. Die + Bestätigung liegt im Ordner. Hunde sind erlaubt, Lagerfeuer nur + auf der Wiese hinten. + + # ── birthday ───────────────────────────────────────────────────────── + - id: oma-party + scope: family/birthday + slug: geburtstagsfeier + lang: de + date: 2026-09-05 + persons: [marge, lisa] + tags: [birthday, family] + title: Omas Geburtstagsfeier + truth: The grandmother's party is on a Sunday afternoon at home with fourteen guests. + body: | + Die Geburtstagsfeier steigt am Sonntagnachmittag bei uns. + + Vierzehn Leute haben zugesagt. Marge bäckt den Kuchen, Lisa macht + die Deko. Anfang um drei, damit die Kinder nicht zu spät ins Bett + kommen. + + - id: cake-allergy + scope: family/birthday + slug: kuchen + lang: de + date: 2026-09-06 + persons: [marge] + tags: [birthday, health] + title: Kuchen für Tante Hilde + truth: One guest cannot eat nuts, so the cake has to be made without them. + body: | + Tante Hilde verträgt keine Haselnüsse, auch nicht in Spuren. + + Also der Zitronenkuchen statt der Nusstorte. Rezept steht im roten + Heft. Marzipan geht auch nicht, da ist oft Nussmehl drin. + + - id: gift-telescope + scope: marge/notes + slug: gift-ideas-autumn + lang: en + date: 2026-08-30 + persons: [marge] + tags: [private, gifts] + title: Gift ideas + truth: Marge plans to buy Lisa a telescope and is keeping it secret from her. + body: | + For Lisa: the refractor from the shop on Elm Street, 120 dollars, + they hold it until the end of the month. + + Do not leave this page open. She reads over my shoulder. + + # ── groceries and household ────────────────────────────────────────── + - id: milk-brand + scope: family/groceries + slug: einkauf + lang: de + date: 2026-09-10 + persons: [marge, homer] + tags: [groceries, todos] + title: Einkaufsliste + truth: The family buys lactose free milk because Bart gets stomach ache otherwise. + body: | + - [ ] Milch, die laktosefreie, Bart bekommt sonst Bauchweh + - [ ] Butter + - [x] Kaffee + - [ ] Spülmittel + - [ ] Batterien für die Campinglampe + + - id: dishwasher + scope: family/house + slug: spuelmaschine + lang: de + date: 2026-04-18 + persons: [homer] + tags: [house, repairs] + title: Spülmaschine läuft aus + truth: The dishwasher leaked and a repair man replaced a seal for ninety euro. + body: | + Wasser unter der Maschine, kam aus der Tür. + + Der Monteur war Dienstag da, hat die Dichtung getauscht. 90 Euro + bar. Er meinte, das Ding hält noch zwei Jahre, dann lohnt sich die + Reparatur nicht mehr. + + - id: boiler-service + scope: family/house + slug: heizung + lang: de + date: 2026-03-11 + persons: [homer, marge] + tags: [house, repairs] + title: Heizung gewartet + truth: The heating was serviced in spring and the next service is due in a year. + body: | + Wartung erledigt. Der Techniker hat den Brenner gereinigt und den + Druck nachgefüllt. + + Nächster Termin in zwölf Monaten, er meldet sich von selbst. + Rechnung 140 Euro, liegt bei den Hausunterlagen. + + # ── school ─────────────────────────────────────────────────────────── + - id: parent-evening + scope: family/school + slug: elternabend + lang: de + date: 2026-09-02 + persons: [marge, bart] + tags: [school] + title: Elternabend + truth: The parents evening is on a Thursday at seven in Bart's classroom. + body: | + Donnerstag um neunzehn Uhr, Raum 2b. + + Thema ist die Klassenfahrt im Frühjahr und der neue Mathelehrer. + Marge geht hin, Homer hat Schicht. + + - id: bart-detention + scope: bart/notes + slug: school-trouble + lang: en + date: 2026-09-08 + persons: [bart] + tags: [school] + title: After school + truth: Bart has to stay behind after school for a week for writing on the board. + body: | + Skinner kept me back. A week of it, every day until four, because + of what I wrote on the blackboard. + + Do not tell dad before Friday. + + - id: lisa-saxophone + scope: lisa/notes + slug: music + lang: en + date: 2026-08-22 + persons: [lisa] + tags: [school, music] + title: Saxophone lessons + truth: Lisa's music lesson moved to Wednesday and the fee rises in September. + body: | + Mr Largo moved the slot. Wednesdays now, half past four, same room. + + The fee goes up in September, 45 a month instead of 38. He says it + is the first rise in four years. + + # ── health ─────────────────────────────────────────────────────────── + - id: maggie-vaccination + scope: family/health + slug: impfung + lang: de + date: 2026-06-25 + persons: [maggie, marge] + tags: [health] + title: Impftermin Maggie + truth: Maggie had a vaccination in June and the next one is due in October. + body: | + Zweite Impfung erledigt, Maggie hat kaum geweint. + + Die nächste steht im Oktober an, die Praxis schickt eine + Erinnerung. Impfpass liegt wieder in der Schublade im Flur. + + - id: homer-back + scope: homer/notes + slug: ruecken + lang: de + date: 2026-07-30 + persons: [homer] + tags: [health] + title: Rücken + truth: Homer's back pain came from lifting and the doctor prescribed physiotherapy. + body: | + Seit dem Umräumen im Keller zieht es unten links. + + Doktor Hibbert sagt nichts Ernstes, aber sechs Mal Krankengymnastik. + Rezept ist eingelöst, erster Termin nächsten Montag. + + - id: dentist + scope: family/health + slug: zahnarzt + lang: de + date: 2026-05-14 + persons: [bart, marge] + tags: [health] + title: Zahnarzt + truth: Bart needs a brace and the health insurance pays only about half of it. + body: | + Kontrolle war unauffällig, aber der Kiefer steht schief. + + Die Spange kostet knapp 1900 Euro, die Kasse übernimmt etwa die + Hälfte. Zweitmeinung wäre gut, bevor wir unterschreiben. + + # ── car ────────────────────────────────────────────────────────────── + - id: car-inspection + scope: family/car + slug: tuev + lang: de + date: 2026-02-09 + persons: [homer] + tags: [car] + title: TÜV + truth: The car failed its inspection over the brakes and passed after the repair. + body: | + Durchgefallen. Bremsscheiben hinten waren unter dem Mindestmaß. + + Werkstatt hat sie getauscht, 320 Euro, danach Nachprüfung ohne + Mängel. Plakette gilt jetzt bis Februar in zwei Jahren. + + - id: winter-tyres + scope: family/car + slug: reifen + lang: de + date: 2026-04-02 + persons: [homer] + tags: [car] + title: Reifen gewechselt + truth: The winter tyres are worn down and have to be replaced before next winter. + body: | + Sommerreifen sind drauf. + + Die Winterreifen haben nur noch drei Millimeter Profil, das reicht + für eine Saison nicht mehr. Vor Oktober neue kaufen, sonst wird es + teuer wie immer im November. + + # ── travel ─────────────────────────────────────────────────────────── + - id: passport-expiry + scope: family/travel + slug: reisepaesse + lang: de + date: 2026-01-20 + persons: [marge, homer, bart, lisa] + tags: [travel, documents] + title: Reisepässe + truth: Bart's passport expires in May and has to be renewed before the summer. + body: | + Durchgesehen: Homer und Marge laufen bis 2029, Lisa bis 2028. + + Barts Pass läuft im Mai ab. Termin im Bürgerbüro dauert + erfahrungsgemäß sechs Wochen, also im Februar kümmern. + + - id: flight-delay-refund + scope: family/travel + slug: flug + lang: de + date: 2026-08-11 + persons: [homer] + tags: [travel, money] + title: Flug verspätet + truth: A delayed flight entitles the family to compensation nobody has claimed yet. + body: | + Vier Stunden später gelandet, Grund war die Crew. + + Ab drei Stunden gibt es 400 Euro pro Person. Formular ist + ausgedruckt, noch nicht abgeschickt. Frist läuft drei Jahre, aber + liegen lassen hilft nicht. + + # ── money ──────────────────────────────────────────────────────────── + - id: insurance-switch + scope: family/money + slug: versicherung + lang: de + date: 2026-03-28 + persons: [marge] + tags: [money, documents] + title: Hausratversicherung + truth: The contents insurance must be cancelled three months before the year ends. + body: | + Der Vertrag läuft zum Jahresende aus, Kündigung muss drei Monate + vorher raus. + + Ein Angebot liegt vor, 60 Euro im Jahr günstiger bei gleicher + Deckung. Das Fahrrad ist dort mitversichert, im alten Vertrag nicht. + + - id: savings-plan + scope: family/money + slug: sparen + lang: en + date: 2026-06-14 + persons: [marge, homer] + tags: [money] + title: Savings for the children + truth: The family puts fifty a month aside for each child by standing order. + body: | + Two accounts opened at the branch on Main Street, one per child. + + Fifty a month each, standing order from the joint account on the + first. Maggie gets hers when she starts school. + + # ── pets ───────────────────────────────────────────────────────────── + - id: dog-food + scope: family/pets + slug: hund + lang: de + date: 2026-07-05 + persons: [bart, lisa] + tags: [pets] + title: Futter für den Hund + truth: The dog gets different food because the old one upset his stomach. + body: | + Das Trockenfutter verträgt er nicht, seit Wochen Durchfall. + + Der Tierarzt hat Schonkost empfohlen, Huhn und Reis, danach langsam + auf die Sorte mit Lamm umstellen. Keine Reste vom Tisch, auch nicht + heimlich, Bart. + + # ── house and hobbies ──────────────────────────────────────────────── + - id: garden-shed + scope: family/house + slug: gartenhaus + lang: de + date: 2026-05-02 + persons: [homer] + tags: [house, garden] + title: Gartenhaus streichen + truth: The garden shed needs painting and the paint is already in the cellar. + body: | + Das Holz ist an der Wetterseite grau geworden. + + Lasur steht im Keller, zwei Eimer, Farbton Nussbaum. Vorher + abschleifen, sonst hält es keine zwei Jahre. Braucht ein trockenes + Wochenende. + + - id: bike-repair + scope: bart/notes + slug: fahrrad + lang: de + date: 2026-08-19 + persons: [bart] + tags: [repairs] + title: Fahrrad + truth: Bart's bike has a snapped gear cable and the shop wants twenty euro for it. + body: | + Schaltung geht nicht mehr, der Zug ist gerissen. + + Im Laden wollen sie zwanzig Euro. Homer sagt, das macht er selbst, + mal sehen wie lange das Rad dann in der Garage steht. + + # ── documents ──────────────────────────────────────────────────────── + - id: electricity-meter + scope: family/house + slug: stromzaehler + lang: de + date: 2026-01-03 + persons: [homer] + tags: [house, documents] + title: Zählerstand + truth: The electricity reading at new year was noted for the annual bill. + body: | + Stand am 1. Januar: 48231 Kilowattstunden. + + Letztes Jahr waren es 44980, also gut 3200 im Jahr. Abschlag bleibt + erst mal gleich, die Abrechnung kommt im März. + + - id: rental-deposit + scope: family/money + slug: kaution + lang: de + date: 2026-02-27 + persons: [marge] + tags: [money, documents] + title: Kaution alte Wohnung + truth: The old flat's deposit came back short, with a deduction for the floor. + body: | + Überweisung ist da, aber nicht vollständig. + + Der Vermieter hat 350 Euro einbehalten für den Parkettschaden im + Wohnzimmer. Das Übergabeprotokoll liegt im Ordner, da steht der + Kratzer drin, also wohl berechtigt. + + - id: library-card + scope: lisa/notes + slug: library + lang: en + date: 2026-09-01 + persons: [lisa] + tags: [school] + title: Library + truth: Lisa owes a late fee and cannot borrow anything until it is paid. + body: | + Three books went back two weeks late. It is four dollars twenty and + they will not let me take anything out until it is settled. + + The astronomy one was worth it. + +# Distractor pages. Same domains, same vocabulary, no gold answers -- +# they exist so that a query has plenty of plausible wrong pages to +# outrank. `count` pages are generated by cycling the openers and +# details with a fixed seed, so the corpus is reproducible. +noise: + seed: 20260916 + count: 150 + topics: + - scope: family/camping + tags: [camping] + lang: de + openers: ["Kurz notiert vom Wochenende.", "Noch offen für die Fahrt.", + "Packliste durchgegangen.", "Wetterbericht angeschaut."] + details: ["Schlafsack muss gelüftet werden.", "Der Kocher ist voll.", + "Zeltplane hat einen Fleck.", "Karte liegt im Auto.", + "Taschenlampe braucht Batterien.", "Campingstuhl wackelt."] + - scope: family/house + tags: [house, repairs] + lang: de + openers: ["Im Haus zu erledigen.", "Notiz an die Kühlschranktür.", + "Nach dem Regen angeschaut.", "Kurz mit dem Nachbarn geredet."] + details: ["Der Wasserhahn tropft leicht.", "Fenster im Bad klemmt.", + "Rasen muss gemäht werden.", "Keller riecht muffig.", + "Glühbirne im Flur ist durch.", "Türklingel geht sporadisch."] + - scope: family/school + tags: [school] + lang: de + openers: ["Aus der Schule mitgebracht.", "Zettel im Ranzen gefunden.", + "Kurzer Anruf aus dem Sekretariat.", "Termin notiert."] + details: ["Sportzeug muss mit.", "Die Arbeit wird zurückgegeben.", + "Ausflug kostet acht Euro.", "Bücher einbinden.", + "Foto wird gemacht.", "Vertretung in Englisch."] + - scope: family/groceries + tags: [groceries, todos] + lang: de + openers: ["Für den Wocheneinkauf.", "Vorrat durchgesehen.", + "Schnell vor dem Laden notiert.", "Nach dem Kochen gemerkt."] + details: ["Mehl ist fast leer.", "Nudeln reichen noch.", + "Öl nachkaufen.", "Zwiebeln vergessen.", + "Käse war im Angebot.", "Brot beim Bäcker holen."] + - scope: family/health + tags: [health] + lang: de + openers: ["Nach dem Termin notiert.", "Aus der Praxis mitgenommen.", + "Am Telefon besprochen.", "Kurz aufgeschrieben."] + details: ["Blutdruck war in Ordnung.", "Rezept liegt im Auto.", + "Termin verschoben auf nächste Woche.", + "Salbe zweimal täglich.", "Krankmeldung ist raus.", + "Überweisung mitgenommen."] + - scope: family/car + tags: [car] + lang: de + openers: ["Nach der Fahrt notiert.", "In der Werkstatt gehört.", + "Beim Tanken gemerkt.", "Auf dem Parkplatz gesehen."] + details: ["Scheibenwischer schmiert.", "Ölstand kontrolliert.", + "Kofferraum ausgeräumt.", "Kennzeichen hinten lose.", + "Tank war fast leer.", "Innenraum ausgesaugt."] + - scope: family/travel + tags: [travel] + lang: en + openers: ["Noted after the call.", "From the confirmation mail.", + "Quick note before we forget.", "Checked this morning."] + details: ["The train leaves from platform four.", + "Breakfast is not included.", "Check in opens at two.", + "The rental desk closes at eight.", + "Bring the printed voucher.", "Parking is extra."] + - scope: family/money + tags: [money] + lang: en + openers: ["From the statement.", "After the phone call.", + "Filed for later.", "Checked the account."] + details: ["The standing order went out.", + "The invoice is due at the end of the month.", + "They refunded the difference.", + "The rate changes in January.", + "Keep the receipt for the warranty.", + "The fee was waived this time."] diff --git a/tools/retrieval-lab/evaluate.py b/tools/retrieval-lab/evaluate.py new file mode 100644 index 00000000..8a2b4f99 --- /dev/null +++ b/tools/retrieval-lab/evaluate.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Measure both search engines against the same questions. + + python3 tools/retrieval-lab/generate.py + uv run --extra test python tools/retrieval-lab/evaluate.py + +Two things get measured, because the retrieval handover asks for both +and they fail independently. + +**Query quality.** Does the page that answers the question come back, +and does it come back first? recall@1, recall@5 and MRR over the gold +set, broken down by the kind of question, so "it got better" can be +traced to which cases got better. + +**Confidence correctness.** A search that returns the right page and a +search that returns the nearest wrong page look identical to whatever +reads the results, and the second one is how a family gets a confident +answer about something nobody ever wrote down. So each result set also +carries candidate confidence signals, and this harness asks which of +them actually separates "the top hit answers the question" from "it does +not" -- by AUC, by where a threshold would have to sit, and by what that +threshold costs in wrongly suppressed real answers. + +Fairness rules, since the whole point is a comparison: + +- Both engines get the same keywords, extracted mechanically from the + question by dropping stopwords. No model, no hand-tuning -- a keyword + list written by hand would be written, however unconsciously, to suit + whichever engine the author is hoping wins. +- Coverage is computed the same way for both, from the page on disk, so + the confidence comparison is about ranking rather than about one + engine having a signal the other cannot produce. +- Both are capped at the same result limit, which is what the CLI's + default `--limit` gives an agent today. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional, Sequence + +import yaml + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +sys.path.insert(0, str(REPO / "lib")) +sys.path.insert(0, str(REPO / "stacklets" / "memory")) + +import fts_index # noqa: E402 +from lib import keywords_to_regex, search_memory # noqa: E402 + +LIMIT = 20 + +# Function words in both languages the family writes in. Dropping them +# is the entire keyword extraction: mechanical, so neither engine gets a +# query shaped in its favour. +STOPWORDS = set(""" +a an and are as at be been but by can could did do does for from had has have +how i in is it its of on or our should that the their them they this to too +up us was we were what when where which who why will with would you your + +aber alle als am an auch auf aus bei beim bis da damit dann das dass dem den +der des dessen die dies diese diesem diesen dieser dieses doch dort du ein +eine einem einen einer eines er es euch euer für hat hatte hatten hier ich +ihm ihn ihr ihre im in ist ja kann kein keine keinen können mal man mehr +mein mit muss müssen nach nicht noch nun nur ob oder ohne schon sein seine +sich sie sind so soll sollen um und uns unser vom von vor war waren was +weil welche welchem welchen welcher welches wem wen wenn wer werden wie +wieso wir wird wo wofür wohin wurde wurden zu zum zur über +""".split()) + + +# ── the query the engines actually receive ────────────────────────────── + +def keywords_for(question: str) -> list[str]: + """Content words of the question, in order, deduplicated. + + Stands in for the model's rewrite step. Using the real model here + would make the engine comparison depend on which words it happened + to pick that run, and the handover's own warning about A/B hygiene + applies: freeze everything that is not the thing under test. + """ + seen: list[str] = [] + for raw in question.replace("?", " ").split(): + word = raw.strip(",.:;!\"'()").lower() + if not word or word in STOPWORDS or len(word) < 2: + continue + if word not in seen: + seen.append(word) + return seen + + +# ── backends ──────────────────────────────────────────────────────────── + +@dataclass +class Result: + """One backend's answer to one question.""" + + ranked: list[str] + scores: list[float] + seconds: float + + +Backend = Callable[[Sequence[str]], Result] + + +def regex_backend(vault: Path) -> Backend: + """Today's engine: OR the keywords into a regex, sort by date. + + This is `search_memory` unchanged, driven exactly as `stack memory + search --nl` drives it, so the baseline is the shipping behaviour + and not a reconstruction of it. + """ + def run(keywords: Sequence[str]) -> Result: + started = time.perf_counter() + hits = search_memory(keywords_to_regex(list(keywords)), vault, + limit=LIMIT) + elapsed = time.perf_counter() - started + return Result([h["rel"] for h in hits], [], elapsed) + return run + + +def fts5_backend(db: Path, substrings: bool = False) -> Backend: + def run(keywords: Sequence[str]) -> Result: + started = time.perf_counter() + hits = fts_index.search(db, list(keywords), limit=LIMIT, + substrings=substrings) + elapsed = time.perf_counter() - started + return Result([h.rel for h in hits], [h.score for h in hits], elapsed) + return run + + +# ── scoring ───────────────────────────────────────────────────────────── + +@dataclass +class Measured: + """Everything one (question, backend) pair produced.""" + + question: str + klass: str + lang: str + gold: list[str] + ranked: list[str] + seconds: float + rank: Optional[int] # 1-based rank of the first gold page + signals: dict[str, float] = field(default_factory=dict) + + @property + def answerable(self) -> bool: + return bool(self.gold) + + @property + def top_is_gold(self) -> bool: + return bool(self.ranked) and self.ranked[0] in self.gold + + +def first_gold_rank(ranked: Sequence[str], gold: Sequence[str]) -> Optional[int]: + for position, rel in enumerate(ranked, start=1): + if rel in gold: + return position + return None + + +def page_tokens(vault: Path, rel: str, cache: dict[str, set[str]]) -> set[str]: + """Every token on a page, the way the index would have tokenised it.""" + if rel not in cache: + text = (vault / rel).read_text(encoding="utf-8", errors="ignore") + cache[rel] = set(fts_index.tokens(text)) + return cache[rel] + + +def signals_for(result: Result, keywords: Sequence[str], vault: Path, + cache: dict[str, set[str]]) -> dict[str, float]: + """Candidate answers to "should anything be said about this at all". + + None of these is the confidence number yet. They are the cheap + things a caller could compute from a result set, measured side by + side so the one that actually tracks correctness can be chosen from + evidence rather than picked because it sounds principled. + + top_coverage share of the query's keywords present on the top hit. + best_coverage the same, over the whole result set. + margin how far the top hit's score sits above the + runner-up, relative to the top score. Ranking + engines only, and zero when there is no runner-up: + one lonely hit is not evidence of anything, and + scoring it against nothing would make every + single-hit query look maximally certain. + top_score the top hit's own score. Ranking engines only. + coverage_margin the two multiplied, on the theory that confidence + needs both "this page has the words" and "no other + page is as good". A candidate like the rest; the + AUC decides whether it earns its place. + hit_share how much of the result limit came back. A query + that fills the page with matches is usually a query + whose words are too common to mean anything. + """ + if not result.ranked: + return {"top_coverage": 0.0, "best_coverage": 0.0, "margin": 0.0, + "top_score": 0.0, "coverage_margin": 0.0, "hit_share": 0.0} + + coverages = [ + sum(1 for k in keywords + if fts_index.covers(k, page_tokens(vault, rel, cache))) + / max(len(keywords), 1) + for rel in result.ranked + ] + signals = { + "top_coverage": coverages[0], + "best_coverage": max(coverages), + "hit_share": len(result.ranked) / LIMIT, + "margin": 0.0, + "top_score": 0.0, + } + if len(result.scores) >= 2 and result.scores[0]: + top, second = result.scores[0], result.scores[1] + signals["top_score"] = top + signals["margin"] = (top - second) / top + elif result.scores: + signals["top_score"] = result.scores[0] + signals["coverage_margin"] = signals["top_coverage"] * signals["margin"] + return signals + + +def auc(positives: Sequence[float], negatives: Sequence[float]) -> float: + """Probability a positive scores above a negative, ties counting half. + + The Mann-Whitney form, written out rather than imported: the lab has + no scientific-stack dependency and this is six lines. 0.5 means the + signal carries no information about whether the answer was found. + """ + if not positives or not negatives: + return float("nan") + wins = 0.0 + for p in positives: + for n in negatives: + wins += 1.0 if p > n else 0.5 if p == n else 0.0 + return wins / (len(positives) * len(negatives)) + + +def wilson(hits: int, total: int) -> tuple[float, float]: + """A 95% interval for a rate measured on very few queries. + + The gold set has tens of questions, not thousands, so a rate like + "17% of absent facts still get answered" is two queries out of + twelve and could as honestly be 5% or 45%. The Wilson interval says + so, where the bare percentage invites a decision the sample cannot + support. 1.96 is the normal quantile for 95%. + """ + if total == 0: + return (0.0, 0.0) + z = 1.96 + p = hits / total + denominator = 1 + z * z / total + centre = (p + z * z / (2 * total)) / denominator + spread = (z * ((p * (1 - p) / total + z * z / (4 * total * total)) ** 0.5) + / denominator) + return (max(0.0, centre - spread), min(1.0, centre + spread)) + + +@dataclass +class Threshold: + """Where a confidence cut would have to sit, and what it would cost.""" + + value: float + balanced_accuracy: float + false_confident_absent: float + false_confident_wrong: float + false_abstain: float + + +def best_threshold(rows: Sequence[Measured], signal: str) -> Threshold: + """The cut that best separates "found it" from "did not", and its bill. + + Balanced accuracy rather than plain accuracy because the gold set is + not balanced and a cut that answers everything would otherwise look + respectable. The three costs are reported separately because they + are not interchangeable: wrongly answering a question the vault + cannot answer is the failure that produces an invented fact, and it + deserves to be read on its own. + """ + values = sorted({r.signals.get(signal, 0.0) for r in rows}) + candidates = values + [max(values) + 1e-9] if values else [0.0] + found = [r for r in rows if r.top_is_gold] + not_found = [r for r in rows if not r.top_is_gold] + absent = [r for r in rows if not r.answerable] + wrong = [r for r in rows if r.answerable and not r.top_is_gold] + + best = Threshold(0.0, 0.0, 1.0, 1.0, 0.0) + for cut in candidates: + tpr = (sum(1 for r in found if r.signals.get(signal, 0.0) >= cut) + / len(found)) if found else 0.0 + tnr = (sum(1 for r in not_found if r.signals.get(signal, 0.0) < cut) + / len(not_found)) if not_found else 0.0 + balanced = (tpr + tnr) / 2 + if balanced > best.balanced_accuracy: + best = Threshold( + value=cut, + balanced_accuracy=balanced, + false_confident_absent=( + sum(1 for r in absent if r.signals.get(signal, 0.0) >= cut) + / len(absent)) if absent else 0.0, + false_confident_wrong=( + sum(1 for r in wrong if r.signals.get(signal, 0.0) >= cut) + / len(wrong)) if wrong else 0.0, + false_abstain=1.0 - tpr, + ) + return best + + +# ── reporting ─────────────────────────────────────────────────────────── + +def quality(rows: Sequence[Measured]) -> dict: + answerable = [r for r in rows if r.answerable] + if not answerable: + return {} + return { + "queries": len(answerable), + "recall@1": sum(1 for r in answerable if r.rank == 1) / len(answerable), + "recall@5": sum(1 for r in answerable + if r.rank and r.rank <= 5) / len(answerable), + "mrr": sum(1 / r.rank for r in answerable if r.rank) / len(answerable), + } + + +def table(title: str, headers: Sequence[str], + rows: Sequence[Sequence[object]]) -> str: + """Fixed-width columns. Every cell is rendered with `str`.""" + widths = [max(len(str(h)), *(len(str(r[i])) for r in rows)) if rows + else len(str(h)) for i, h in enumerate(headers)] + line = " ".join(str(h).ljust(w) for h, w in zip(headers, widths)) + out = [f"\n{title}", line, " ".join("-" * w for w in widths)] + out += [" ".join(str(c).ljust(w) for c, w in zip(r, widths)) for r in rows] + return "\n".join(out) + + +def pct(value: float) -> str: + return f"{value * 100:.0f}%" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vault", default=str(HERE / "out" / "vault")) + parser.add_argument("--json", default=str(HERE / "out" / "results.json")) + ns = parser.parse_args() + + vault = Path(ns.vault) + if not vault.exists(): + sys.exit(f"no vault at {vault} -- run generate.py first") + + fact_paths = yaml.safe_load( + (vault.parent / "fact-paths.yaml").read_text(encoding="utf-8")) + gold_spec = yaml.safe_load( + (HERE / "goldset.yaml").read_text(encoding="utf-8")) + + db = vault.parent / "vault-index.sqlite3" + if db.exists(): + db.unlink() + built = time.perf_counter() + stats = fts_index.build_index(vault, db) + build_seconds = time.perf_counter() - built + + warm = time.perf_counter() + fts_index.build_index(vault, db) + rescan_seconds = time.perf_counter() - warm + + backends: dict[str, Backend] = { + "regex": regex_backend(vault), + "fts5": fts5_backend(db), + "fts5+tri": fts5_backend(db, substrings=True), + } + + cache: dict[str, set[str]] = {} + measured: dict[str, list[Measured]] = {name: [] for name in backends} + for query in gold_spec["queries"]: + keywords = keywords_for(query["q"]) + gold = [fact_paths[fid] for fid in query.get("gold") or []] + for name, backend in backends.items(): + result = backend(keywords) + measured[name].append(Measured( + question=query["q"], klass=query["class"], lang=query["lang"], + gold=gold, ranked=result.ranked, seconds=result.seconds, + rank=first_gold_rank(result.ranked, gold), + signals=signals_for(result, keywords, vault, cache), + )) + + # ── query quality ─────────────────────────────────────────────────── + print(f"corpus: {stats.total} pages index build: " + f"{build_seconds * 1000:.0f} ms no-change rescan: " + f"{rescan_seconds * 1000:.0f} ms") + + overall = {name: quality(rows) for name, rows in measured.items()} + print(table( + "Query quality (answerable questions only)", + ["backend", "queries", "recall@1", "recall@5", "MRR", "p50 ms", + "p95 ms"], + [[name, overall[name]["queries"], pct(overall[name]["recall@1"]), + pct(overall[name]["recall@5"]), f"{overall[name]['mrr']:.2f}", + f"{statistics.median(r.seconds for r in rows) * 1000:.1f}", + f"{sorted(r.seconds for r in rows)[int(len(rows) * 0.95) - 1] * 1000:.1f}"] + for name, rows in measured.items()])) + + classes = [c for c in dict.fromkeys(q["class"] for q in gold_spec["queries"]) + if c != "absent"] + per_class = [] + for klass in classes: + row = [klass] + for name in backends: + rows = [r for r in measured[name] if r.klass == klass] + q = quality(rows) + row.append(f"{pct(q['recall@1'])} / {pct(q['recall@5'])}") + per_class.append(row + [str(len([q for q in gold_spec["queries"] + if q["class"] == klass]))]) + print(table("recall@1 / recall@5 by question kind", + ["kind", *backends, "n"], per_class)) + + # ── confidence correctness ────────────────────────────────────────── + signal_names = ["top_coverage", "best_coverage", "margin", "top_score", + "coverage_margin", "hit_share"] + + def auc_for(rows: Sequence[Measured], signal: str) -> float: + return auc([r.signals.get(signal, 0.0) for r in rows if r.top_is_gold], + [r.signals.get(signal, 0.0) for r in rows + if not r.top_is_gold]) + + auc_rows = [] + for signal in signal_names: + row = [signal] + for name in backends: + value = auc_for(measured[name], signal) + row.append("--" if value != value else f"{value:.2f}") + auc_rows.append(row) + print(table( + 'Confidence signals: AUC for "the top hit answers the question"' + "\n(0.5 = tells you nothing; below 0.5 = the signal runs backwards)", + ["signal", *backends], auc_rows)) + + # The AUC winner is not automatically the signal to ship. AUC ranks + # separation across every possible cut; what a family lives with is + # one cut, and two signals with the same AUC can put their mistakes + # in very different places. So every signal gets its bill printed. + thresholds: dict[str, dict] = {} + cut_rows = [] + for name in backends: + rows = measured[name] + for signal in signal_names: + if len({r.signals.get(signal, 0.0) for r in rows}) < 2: + continue + cut = best_threshold(rows, signal) + thresholds.setdefault(name, {})[signal] = vars(cut) + absent_n = sum(1 for r in rows if not r.answerable) + low, high = wilson( + round(cut.false_confident_absent * absent_n), absent_n) + cut_rows.append([ + name, signal, f"{cut.value:.2f}", + pct(cut.balanced_accuracy), + f"{pct(cut.false_confident_absent)} [{pct(low)}-{pct(high)}]", + pct(cut.false_confident_wrong), pct(cut.false_abstain)]) + print(table( + "Every signal at its best cut, and what that cut costs", + ["backend", "signal", "cut", "balanced acc", + "answers an absent fact", "answers with wrong page", + "suppresses a right answer"], cut_rows)) + + # Answering with no gate at all is the situation today: any hit is + # treated as an answer. Reported so the gate has something to beat. + ungated = [] + absent_total = sum(1 for r in measured["regex"] if not r.answerable) + for name in backends: + absent = [r for r in measured[name] if not r.answerable] + answered = sum(1 for r in absent if r.ranked) + low, high = wilson(answered, len(absent)) + ungated.append([ + name, f"{pct(answered / len(absent))} [{pct(low)}-{pct(high)}]", + f"{statistics.median(len(r.ranked) for r in absent):.0f}"]) + print(table("Without a confidence gate: what an absent fact returns today" + f"\n(n={absent_total} absent questions, 95% interval)", + ["backend", "returns at least one page", "median hits"], + ungated)) + + # Validity check, not a result. `top_coverage` is a fraction of the + # query's keywords, so it falls as a question gets longer. If the + # absent questions were systematically wordier than the answerable + # ones, the signal would be measuring question length and the AUC + # above would be an artefact of how the gold set was written. + lengths = { + "answerable": [len(keywords_for(q["q"])) for q in gold_spec["queries"] + if q.get("gold")], + "absent": [len(keywords_for(q["q"])) for q in gold_spec["queries"] + if not q.get("gold")], + } + print(table( + "Validity check: is the coverage signal just measuring question length?", + ["questions", "n", "median keywords", "mean keywords"], + [[kind, str(len(values)), f"{statistics.median(values):.1f}", + f"{statistics.mean(values):.2f}"] + for kind, values in lengths.items()])) + + Path(ns.json).write_text(json.dumps({ + "corpus_pages": stats.total, + "index_build_ms": build_seconds * 1000, + "rescan_ms": rescan_seconds * 1000, + "quality": overall, + "thresholds": thresholds, + "queries": {name: [ + {"q": r.question, "class": r.klass, "lang": r.lang, + "gold": r.gold, "rank": r.rank, "top": r.ranked[0] if r.ranked + else None, "hits": len(r.ranked), "ms": r.seconds * 1000, + "signals": r.signals} + for r in rows] for name, rows in measured.items()}, + }, indent=2, ensure_ascii=False), encoding="utf-8") + print(f"\nfull per-query detail: {ns.json}") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-lab/explain.py b/tools/retrieval-lab/explain.py new file mode 100644 index 00000000..b9282e61 --- /dev/null +++ b/tools/retrieval-lab/explain.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Print one question kind's results side by side, for reading a regression. + + uv run --extra test python tools/retrieval-lab/inspect.py compound_tail + +The summary tables say a kind got worse. This says which question, what +each engine put first, and where the right page ended up -- which is the +difference between a finding and a number that moved. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def main() -> None: + kinds = sys.argv[1:] or ["compound_tail"] + data = json.loads( + (HERE / "out" / "results.json").read_text(encoding="utf-8")) + for kind in kinds: + print(f"\n=== {kind} ===") + regex = {r["q"]: r for r in data["queries"]["regex"] + if r["class"] == kind} + fts5 = {r["q"]: r for r in data["queries"]["fts5"] + if r["class"] == kind} + for question, rx in regex.items(): + ft = fts5[question] + print(f"\n{question}") + print(f" gold {', '.join(rx['gold']) or '(none)'}") + print(f" regex rank={rx['rank']} hits={rx['hits']:>2} " + f"top={rx['top']}") + print(f" fts5 rank={ft['rank']} hits={ft['hits']:>2} " + f"top={ft['top']}") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-lab/generate.py b/tools/retrieval-lab/generate.py new file mode 100644 index 00000000..77207c2f --- /dev/null +++ b/tools/retrieval-lab/generate.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Render corpus.yaml into a vault directory the search engines can index. + +The output is disposable: delete `out/` and run this again. The spec is +the source of truth, which is the same arrangement `tools/family-memories` +uses, and for the same reason -- a corpus you cannot regenerate is a +corpus nobody can check. + + python3 tools/retrieval-lab/generate.py [--out DIR] + +Fact pages come out as written. Noise pages are cycled from the topic +vocabularies with a fixed seed, so two runs produce byte-identical files +and a measurement taken last week still means something today. +""" + +from __future__ import annotations + +import argparse +import random +from pathlib import Path + +import yaml + +HERE = Path(__file__).resolve().parent + + +def render(title: str, date: str, persons: list[str], tags: list[str], + body: str) -> str: + """One vault page, in the frontmatter shape the archivist writes.""" + lines = ["---", f"title: {title}", f"date: {date}", "type: note"] + if persons: + lines.append("persons:") + lines += [f" - {p}" for p in persons] + if tags: + lines.append("tags:") + lines += [f" - {t}" for t in tags] + lines += ["---", "", body.rstrip(), ""] + return "\n".join(lines) + + +def write_facts(spec: dict, out: Path) -> dict[str, str]: + """Write the answer pages. Returns {fact id: vault-relative path}.""" + paths: dict[str, str] = {} + for fact in spec["facts"]: + rel = f"{fact['scope']}/{fact['slug']}.md" + path = out / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + render(fact["title"], str(fact["date"]), fact.get("persons", []), + fact.get("tags", []), fact["body"]), + encoding="utf-8") + paths[fact["id"]] = rel + return paths + + +def write_noise(spec: dict, out: Path) -> int: + """Fill the vault with plausible wrong answers. + + Without these the corpus is thirty pages and every query is easy: + ranking only earns its keep when there is something to rank against. + The pages are deliberately dull and repetitive -- that is what a real + vault of short notes looks like, and it is the condition under which + a lexical engine either finds the one page that matters or does not. + """ + noise = spec["noise"] + rng = random.Random(noise["seed"]) + people = spec["persons"] + topics = noise["topics"] + written = 0 + for i in range(noise["count"]): + topic = topics[i % len(topics)] + opener = topic["openers"][rng.randrange(len(topic["openers"]))] + details = rng.sample(topic["details"], 2) + month = 1 + (i % 9) + day = 1 + (i * 7) % 27 + rel = f"{topic['scope']}/notes/{i:03d}.md" + path = out / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + render(opener.rstrip("."), f"2026-{month:02d}-{day:02d}", + [people[rng.randrange(len(people))]], topic["tags"], + opener + "\n\n" + "\n".join(details)), + encoding="utf-8") + written += 1 + return written + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", default=str(HERE / "out" / "vault"), + help="directory to render the vault into") + ns = parser.parse_args() + + spec = yaml.safe_load((HERE / "corpus.yaml").read_text(encoding="utf-8")) + out = Path(ns.out) + if out.exists(): + for stale in sorted(out.rglob("*.md")): + stale.unlink() + out.mkdir(parents=True, exist_ok=True) + + facts = write_facts(spec, out) + noise = write_noise(spec, out) + (out.parent / "fact-paths.yaml").write_text( + yaml.safe_dump(facts, allow_unicode=True, sort_keys=True), + encoding="utf-8") + + print(f"{len(facts)} fact pages + {noise} noise pages -> {out}") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-lab/goldset.yaml b/tools/retrieval-lab/goldset.yaml new file mode 100644 index 00000000..ea704591 --- /dev/null +++ b/tools/retrieval-lab/goldset.yaml @@ -0,0 +1,283 @@ +# Questions, and the pages that answer them. +# +# Written from the `truth` lines in corpus.yaml, never from the page +# bodies. That is the whole discipline here: if the queries were written +# while looking at the text they have to find, they would share its +# vocabulary and any lexical engine would look better than it is. +# +# `gold` lists the fact ids that genuinely answer the question. A query +# with no `gold` is a negative control: the fact is not in the vault, and +# the right behaviour is to find nothing convincing rather than to return +# the nearest neighbour with a straight face. +# +# Classes, and what each one is for: +# +# keyword a word that is literally on the page. The floor -- +# both engines should get these. +# paraphrase the question and the page share no content word. +# What ranking is supposed to buy. +# compound_head the query is the front of a German compound on the +# page (Geburtstag -> Geburtstagsfeier). Prefix +# matching should cover this. +# compound_tail the query is the back of one (Nüsse -> Haselnüsse), +# or the page has the back and the query the whole +# word. Prefix matching cannot cover this; the +# handover's trigram fallback is the answer if it hurts. +# fold the query drops an umlaut (TUV for TÜV). The +# tokenizer's remove_diacritics should cover this. +# translit the query spells the umlaut out (Ruecken for Rücken). +# Folding does not cover this. Measured to find out +# whether it matters. +# vague several pages could answer; the right ones must rank +# above the rest. +# scope the answer sits in a personal bucket while the +# question is asked of the whole vault. +# absent not in the vault at all. + +queries: + # ── keyword: the floor ─────────────────────────────────────────────── + - q: Welche Bremsscheiben waren zu dünn + lang: de + class: keyword + gold: [car-inspection] + + - q: Wie oft hat Homer Krankengymnastik verschrieben bekommen + lang: de + class: keyword + gold: [homer-back] + + - q: Where is the telescope for Lisa from + lang: en + class: keyword + gold: [gift-telescope] + + - q: Wo liegt der Impfpass + lang: de + class: keyword + gold: [maggie-vaccination] + + - q: Wann ist der Elternabend + lang: de + class: keyword + gold: [parent-evening] + + - q: When is the saxophone lesson + lang: en + class: keyword + gold: [lisa-saxophone] + + # ── paraphrase: no shared content word ─────────────────────────────── + - q: Who was worried about bugs on the trip + lang: en + class: paraphrase + gold: [mosquitoes] + + - q: What did we arrange to use from next door for the camping trip + lang: en + class: paraphrase + gold: [stove-broken] + + - q: Wer aus der Familie hat Probleme mit dem Magen + lang: de + class: paraphrase + gold: [milk-brand] + + - q: What does Lisa owe money for + lang: en + class: paraphrase + gold: [library-card] + + - q: Was hat Homer sich beim Tragen getan + lang: de + class: paraphrase + gold: [homer-back] + + - q: Welches Tier hat Verdauungsprobleme + lang: de + class: paraphrase + gold: [dog-food] + + - q: Wann müssen wir die Police kündigen + lang: de + class: paraphrase + gold: [insurance-switch] + + # ── compound_head: prefix matching should carry these ──────────────── + - q: Wann ist Omas Geburtstag + lang: de + class: compound_head + gold: [oma-party] + + - q: Was ist am Sonntag geplant + lang: de + class: compound_head + gold: [oma-party] + + - q: Wie ist der Zähler abgelesen worden + lang: de + class: compound_head + gold: [electricity-meter] + + - q: Was ist am Parkett passiert + lang: de + class: compound_head + gold: [rental-deposit] + + # ── compound_tail: prefix matching cannot ──────────────────────────── + - q: Wer verträgt keine Nüsse + lang: de + class: compound_tail + gold: [cake-allergy] + + - q: Was kostet die Zahnspange + lang: de + class: compound_tail + gold: [dentist] + + - q: Wann ist die Feier + lang: de + class: compound_tail + gold: [oma-party] + + - q: Für welchen Schaden wurde Geld einbehalten + lang: de + class: compound_tail + gold: [rental-deposit] + + - q: Wofür brauchen wir Batterien für die Lampe + lang: de + class: compound_tail + gold: [milk-brand] + + # ── fold: the umlaut is dropped ────────────────────────────────────── + - q: Warum ist das Auto durch den TUV gefallen + lang: de + class: fold + gold: [car-inspection] + + - q: Wie hoch war der Zahlerstand im Januar + lang: de + class: fold + gold: [electricity-meter] + + - q: Wann laufen die Reisepasse ab + lang: de + class: fold + gold: [passport-expiry] + + # ── translit: the umlaut is spelled out ────────────────────────────── + - q: Was war mit der Spuelmaschine + lang: de + class: translit + gold: [dishwasher] + + - q: Was hat Homer am Ruecken + lang: de + class: translit + gold: [homer-back] + + - q: Wie viel Profil haben die Winterreifen im Fruehjahr + lang: de + class: translit + gold: [winter-tyres] + + # ── vague: several candidates, the right ones must rank ────────────── + - q: Wie viel hat die letzte Reparatur gekostet + lang: de + class: vague + gold: [dishwasher, car-inspection, bike-repair, boiler-service] + + - q: Was liegt im Ordner + lang: de + class: vague + gold: [campsite-booking, rental-deposit] + + # German, like the pages that answer it. An English question against + # German prose measures translation, which no lexical engine does and + # which is not what this comparison is about. + - q: Was müssen wir noch kaufen + lang: de + class: vague + gold: [milk-brand, winter-tyres] + + - q: Wann ist der nächste Termin beim Arzt + lang: de + class: vague + gold: [maggie-vaccination, homer-back] + + # ── scope: the answer is in a personal bucket ──────────────────────── + - q: What are we giving Lisa + lang: en + class: scope + gold: [gift-telescope] + + - q: Why is Bart staying late at school + lang: en + class: scope + gold: [bart-detention] + + - q: Warum steht Barts Rad in der Garage + lang: de + class: scope + gold: [bike-repair] + + # ── absent: nothing in the vault answers these ─────────────────────── + - q: Wann ist der Zahnarzttermin für Lisa + lang: de + class: absent + gold: [] + + - q: Bei welcher Gesellschaft ist das Auto versichert + lang: de + class: absent + gold: [] + + - q: Which hotel did we book in Italy + lang: en + class: absent + gold: [] + + - q: Wann hat Maggie Geburtstag + lang: de + class: absent + gold: [] + + - q: What is the wifi password + lang: en + class: absent + gold: [] + + - q: Wie viel wiegt der Hund + lang: de + class: absent + gold: [] + + - q: Wann hat Homer die nächste Schicht + lang: de + class: absent + gold: [] + + - q: How much did the flight ticket cost + lang: en + class: absent + gold: [] + + - q: Wann wurde das Auto gekauft + lang: de + class: absent + gold: [] + + - q: What is the neighbour's phone number + lang: en + class: absent + gold: [] + + - q: Wann läuft der Handyvertrag aus + lang: de + class: absent + gold: [] + + - q: Welche Schule besucht Lisa + lang: de + class: absent + gold: [] From 64a1a281587285a7dd0bc3e07faec73b240566c1 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 13:09:17 +0200 Subject: [PATCH 03/11] docs(brain): record what the retrieval PoC measured The right page comes first 2.6x as often and a search costs a tenth of the time. Two results change the plan: the trigram index is required rather than optional, because German compounds regress without it, and the confidence signal to ship is keyword coverage, not the BM25 score. Paraphrase recall of 29% is the number tier 2 has to beat. --- docs/design/brain/retrieval-engine-poc.md | 158 ++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 docs/design/brain/retrieval-engine-poc.md diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md new file mode 100644 index 00000000..d8d72d56 --- /dev/null +++ b/docs/design/brain/retrieval-engine-poc.md @@ -0,0 +1,158 @@ +# Retrieval engine PoC: what the numbers say + +**Status:** probe complete, nothing wired into the CLI yet. +**Answers:** `docs/design/handover/memory-retrieval-upgrade.md` +**Bench:** `tools/retrieval-lab/` (177-page fabricated vault, 47 questions) +**Date:** 2026-09-16 + +The handover proposed SQLite FTS5 + BM25 as tier 1 and named a trigram +companion as the fallback if German compounds hurt. This measured both +against the shipping regex engine before writing any of it into +`stack memory search`, on a corpus large enough for ranking to matter. + +Every number below comes from `tools/retrieval-lab/evaluate.py`, which +runs all three engines over one question set in about two seconds. + +## Query quality + +| Engine | recall@1 | recall@5 | MRR | p50 latency | +|---|---|---|---|---| +| regex (today) | 23% | 57% | 0.36 | 9.8 ms | +| fts5 | 57% | 69% | 0.61 | 0.8 ms | +| fts5 + trigram | **60%** | **71%** | **0.64** | 1.3 ms | + +35 answerable questions. The right page comes first 2.6x as often, and +a search costs a tenth of what it does today -- on 177 pages. The regex +engine reads every file on every query, so its 9.8 ms grows with the +vault while FTS5's does not. + +By question kind, recall@1 / recall@5: + +| Kind | n | regex | fts5 | fts5+tri | +|---|---|---|---|---| +| keyword | 6 | 17% / 67% | 83% / 83% | 83% / 83% | +| paraphrase | 7 | 0% / 0% | 29% / 29% | 29% / 29% | +| compound_head | 4 | 75% / 75% | 100% / 100% | 100% / 100% | +| compound_tail | 5 | 60% / 100% | 40% / 80% | 60% / 100% | +| fold (TUV for TÜV) | 3 | 0% / 33% | 100% / 100% | 100% / 100% | +| translit (Ruecken for Rücken) | 3 | 0% / 33% | 33% / 33% | 33% / 33% | +| vague | 4 | 25% / 100% | 50% / 75% | 50% / 75% | +| scope | 3 | 0% / 67% | 33% / 67% | 33% / 67% | + +### The trigram index is not optional for a German family + +This is the finding that changes the plan. FTS5 matches from the front +of a word, so "Geburtstag" reaches "Geburtstagsfeier" but "Feier" does +not. The regex engine matches *substrings*, so it gets that case right +by accident -- and German puts the noun the family asks with at either +end of a compound. + +Shipping FTS5 alone would therefore be a **regression** on compound +tails: 60% to 40% recall@1, and one question ("Wann ist die Feier", +against a page titled *Omas Geburtstagsfeier*) went from one correct hit +to zero hits at all. Adding the trigram index and fusing with RRF +restores it to 60% / 100% while keeping every other gain. + +The handover had this as a stretch item, conditional on compounds +disappointing. They did. It belongs in the first slice. + +### Paraphrases are where tier 1 runs out + +29% recall@1 on questions that share no content word with their page, +up from 0%. Better, still the weakest class, and no amount of lexical +tuning fixes it: "wer hat Probleme mit dem Magen" cannot reach a page +that says "Bauchweh" by matching characters. This is the tier-2 trigger +the handover named, now with a number on it. + +### A German gap the handover did not name + +`translit`: a query spelling the umlaut out ("Ruecken", "Spuelmaschine") +against a page that uses it ("Rücken", "Spülmaschine"). 33% for every +engine. `remove_diacritics` folds ü to u, not to ue, so the two spellings +stay different words. Cheap to fix in normalisation if it matters -- +worth raising with the family before adding machinery for it, since it +depends on how they actually type. + +## Confidence correctness + +The question behind this half: when the search returns something, can +anything downstream tell whether it is the answer or just the nearest +page? Today nothing can, and the cost is measurable. + +**Without any gate, 92% [65-99%] of questions whose answer is not in the +vault still return at least one page** -- a median of 6 of them. That is +the raw material for a confident answer about something nobody wrote +down. + +Five candidate signals, scored by how well each separates "the top hit +answers the question" from "it does not" (AUC; 0.5 is a coin flip): + +| Signal | regex | fts5 | fts5+tri | +|---|---|---|---| +| **top_coverage** (share of query keywords on the top hit) | 0.73 | **0.85** | 0.80 | +| best_coverage (same, best hit in the set) | 0.54 | 0.83 | 0.79 | +| top_score (the BM25 itself) | 0.50 | 0.75 | 0.68 | +| margin (top score over runner-up) | 0.50 | 0.57 | 0.58 | +| coverage x margin | 0.50 | 0.58 | 0.61 | +| hit_share (how full the result page is) | 0.25 | 0.40 | 0.41 | + +**The score is not the confidence signal.** The obvious move -- trust +BM25, or trust the gap to the runner-up -- is the weak one: margin lands +at 0.57, barely above chance. What tracks correctness is how much of the +question the top page actually contains. That is also the signal the +regex engine could compute today, which makes it a change that does not +depend on the index landing first. + +`hit_share` below 0.5 means it runs backwards, and informatively: a +query that fills the result page is usually a query whose words are too +common to mean anything. + +At its best cut (0.40 coverage) on the fts5 arm: + +| | | +|---|---| +| balanced accuracy | 80% | +| answers a fact that is not in the vault | 17% [5-45%], down from 92% | +| answers with the wrong page | 40% | +| suppresses a right answer | 10% | + +So a coverage gate would cut confidently-wrong answers on absent facts +by roughly five to one, and pay 10% of correct answers for it. On 12 +absent questions the interval is wide; the direction is clear, the +precise rate is not. + +Note the trigram arm costs a little confidence (0.85 to 0.80) for the +recall it buys: matching inside words finds more pages and means less +per match. Worth watching, not worth reversing. + +### Validity check + +`top_coverage` is a fraction of the query's keywords, so it falls as a +question gets longer. If the absent questions were wordier than the +answerable ones, the AUC would be measuring question length. They are +not: median 3.0 keywords either way, mean 2.94 against 3.08. + +## What this changes in the plan + +1. **Build tier 1 as FTS5 + trigram, fused with RRF**, not FTS5 alone. + The trigram table moves from stretch goal to first slice, because + without it the change is a regression for German compounds. +2. **Ship a confidence signal with the results, and make it coverage**, + not the score. `Hit.matched` already carries it. +3. **Tier 2 (embeddings) stays open**, with paraphrase recall at 29% as + the number it has to beat. +4. **Decide whether transliterated umlauts matter** before building for + them. + +## What has not been measured + +The agent has not run against this. Everything above is the engine in +isolation: whether ranked results and a coverage number actually cut the +agent's tool iterations, or change what it says when it finds nothing, +is a `tools/agent-lab/rig/` question and the next step. The handover's +scenarios 1-6 are written for that rig and still stand. + +Also untested: the model's own keyword rewrite (this used mechanical +stopword removal so the engine was the only variable), cross-language +questions, and anything at real vault scale -- 177 pages is enough for +ranking to bite, not enough to say anything about a vault of thousands. From 1b2e8684a69daaf68f65c2de6c1398a62d5df241 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 13:17:19 +0200 Subject: [PATCH 04/11] fix(memory): keep the search index in step with the vault The index is a cache over the vault checkout, so a search has to know whether it is still current before it answers. Nothing writes into that checkout outside git, so a HEAD that has not moved means no page has changed: the index stores the HEAD it was built from and skips the reconcile when they match. Constant time instead of statting every file, which reaches 337 ms on a vault of 8000 pages. A directory that is not a checkout keeps scanning, so --vault overrides still pick up edits. Also: the HEAD is stored with the rows it describes, so a reconcile that dies halfway is redone rather than left half applied; a page deleted mid-scan is skipped instead of failing the search; and a second indexer waits for the file instead of erroring. --- docs/design/brain/retrieval-engine-poc.md | 61 ++++++++++ stacklets/memory/fts_index.py | 64 ++++++++-- tests/stacklets/test_memory_fts_index.py | 141 ++++++++++++++++++++++ tools/retrieval-lab/README.md | 1 + tools/retrieval-lab/freshness.py | 110 +++++++++++++++++ tools/retrieval-lab/generate.py | 4 + 6 files changed, 373 insertions(+), 8 deletions(-) create mode 100644 tools/retrieval-lab/freshness.py diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index d8d72d56..ac7ed573 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -132,6 +132,67 @@ question gets longer. If the absent questions were wordier than the answerable ones, the AUC would be measuring question length. They are not: median 3.0 keywords either way, mean 2.94 against 3.08. +## Keeping the index current + +An index is a cache, and a cache that quietly falls behind is worse +than no cache: the family gets yesterday's answer with today's +confidence. Two links in the chain, with different owners. + +**Forgejo to the clone** is `refresh_vault_if_stale`, which already +exists and does not change. It compares local HEAD to remote HEAD and +pulls only on a difference. When the remote is unreachable it says so +and reads proceed against the stale clone -- the existing contract. + +**The clone to the index** is new, and leans on a fact that is already +load-bearing elsewhere in this stacklet: *nothing writes into the vault +working copy outside git*. `update_memory` reads the canonical file +from Forgejo, commits there, and fast-forwards the clone; +`propagate_write` already uses that clone's HEAD as its "has it landed" +token. So a HEAD that has not moved is proof that no page has changed. + +The index stores the HEAD it was built from. Each search compares, and +skips the reconcile entirely when they match: + +| Vault pages | HEAD gate | Scan every file | Cold rebuild | +|---|---|---|---| +| 177 | 14.4 ms | 8.2 ms | 64 ms | +| 1000 | 15.7 ms | 42.5 ms | 252 ms | +| 3000 | 15.0 ms | 124.7 ms | 651 ms | +| 8000 | 14.9 ms | 337.5 ms | 1810 ms | + +Both columns are real `build_index` calls where nothing changed; the +scan column is the same tree with `.git` hidden so the gate cannot +fire. Reproduce with `tools/retrieval-lab/freshness.py`. + +Below roughly 400 pages the gate is the *slower* of the two, because a +`git rev-parse` is a subprocess and statting 177 files is not. It is +still the right choice: 15 ms is constant and 337 ms is not, and the +number that matters is the one at the size a vault grows into. The +obvious further saving is to pass HEAD in rather than re-read it -- +`refresh_vault_if_stale` already computed it moments earlier -- which +is a one-line change when the CLI is wired up. + +Four properties make the staleness safe rather than merely fast: + +- **A vault that is not a checkout always scans.** `--vault` overrides, + fixtures and the lab have no HEAD to trust, so nothing + short-circuits and an edit is picked up. +- **The HEAD is stored in the same transaction as the rows it + describes.** A reconcile that dies halfway rolls back both, and the + next search redoes it. The index is never half updated while + claiming to be current. +- **An unreadable page is skipped, not fatal.** A sync can delete a + file between the walk listing it and the reconcile reading it. +- **Concurrent indexers wait instead of failing.** The CLI on the host + and the archivist in its container both reconcile; SQLite's default + is to error the instant the file is busy, so the connection sets a + busy timeout. + +What this deliberately does *not* do is make a hand-edited working copy +visible. In production nothing hand-edits it. That is the whole reason +the gate is sound, and the cost of it is written into the test that +pins the behaviour. + ## What this changes in the plan 1. **Build tier 1 as FTS5 + trigram, fused with RRF**, not FTS5 alone. diff --git a/stacklets/memory/fts_index.py b/stacklets/memory/fts_index.py index 8a0452e1..37945ada 100644 --- a/stacklets/memory/fts_index.py +++ b/stacklets/memory/fts_index.py @@ -49,7 +49,7 @@ from typing import Iterable, List, Optional, Sequence sys.path.insert(0, str(Path(__file__).resolve().parent)) -from lib import _fm_list, _norm_tag, body_only # noqa: E402 +from lib import _fm_list, _norm_tag, body_only, vault_local_head # noqa: E402 from stack.frontmatter import parse as parse_frontmatter # noqa: E402 @@ -112,6 +112,8 @@ def tokens(text: str) -> List[str]: # ── schema ────────────────────────────────────────────────────────────── SCHEMA = """ +CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE IF NOT EXISTS docs( id INTEGER PRIMARY KEY, path TEXT UNIQUE, @@ -152,6 +154,12 @@ def _connect(db: Path) -> sqlite3.Connection: db.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(db) conn.row_factory = sqlite3.Row + # Two processes reconcile this file: the CLI on the host and the + # archivist in its container. SQLite serialises writers and, left + # alone, gives up the instant the file is busy -- which would turn + # "the other one is mid-reconcile" into an error rather than a + # short wait. + conn.execute("PRAGMA busy_timeout = 5000") conn.executescript(SCHEMA) return conn @@ -236,14 +244,43 @@ def _fts_insert(conn: sqlite3.Connection, doc_id: int, doc: _Doc) -> None: def build_index(vault: Path, db: Path) -> Stats: """Reconcile the index with the vault, and report what changed. - Cheap when nothing moved, which is the common case: this runs on - every search, so the no-change path compares mtime against the - stored value and reads no file at all. A page whose mtime differs - is hashed before it is re-indexed, because git checkouts and clone - refreshes rewrite mtimes on files whose content is identical. + This runs before every search, so the cost when nothing changed is + the cost that matters. It is answered in two steps. + + **The checkout's HEAD, when there is one.** The vault is a clone, + and every change to it arrives as a commit: `update_memory` writes + through Forgejo and fast-forwards this copy, and nothing else + writes these files at all. So a HEAD that has not moved is proof + that no page has changed, in constant time. Measured on a vault of + 8000 pages: 14 ms to read HEAD against 350 ms to stat every file. + + **A scan, when there is no HEAD to trust.** A `--vault` override or + a fixture directory is not a clone, so there is nothing to + short-circuit on. Then mtime decides, and a page whose mtime moved + is hashed before it is re-indexed, because a checkout rewrites + mtimes on files whose content is identical. + + The HEAD is stored in the same transaction as the rows it + describes. A reconcile that dies halfway leaves both behind, and + the next search does the work again -- the index is never half + updated while claiming to be current. + + What this cannot do is make the *clone* current; that is + `refresh_vault_if_stale`, and the caller's concern. When the remote + is unreachable the index faithfully reflects a stale vault, which + is the existing contract for reads and not something an index can + fix. """ conn = _connect(db) try: + head = vault_local_head(vault) + if head is not None: + stored = conn.execute( + "SELECT value FROM meta WHERE key = 'head'").fetchone() + if stored is not None and stored["value"] == head: + total = conn.execute("SELECT COUNT(*) FROM docs").fetchone()[0] + return Stats(added=0, updated=0, deleted=0, total=int(total)) + known = { row["path"]: row for row in conn.execute( @@ -255,11 +292,17 @@ def build_index(vault: Path, db: Path) -> Stats: seen: set[str] = set() for md_path in sorted(vault.rglob("*.md")): - if not md_path.is_file(): + try: + if not md_path.is_file(): + continue + mtime = md_path.stat().st_mtime + except OSError: + # A sync can delete a page between the walk listing it + # and this reading it. One page missing from the + # results beats an error instead of the other results. continue rel = str(md_path.relative_to(vault)) seen.add(rel) - mtime = md_path.stat().st_mtime row = known.get(rel) if row is not None and row["mtime"] == mtime: continue @@ -303,6 +346,11 @@ def build_index(vault: Path, db: Path) -> Stats: _fts_delete(conn, row) conn.execute("DELETE FROM docs WHERE id=?", (row["id"],)) + if head is not None: + conn.execute( + "INSERT INTO meta(key, value) VALUES('head', ?)" + " ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (head,)) conn.commit() total = conn.execute("SELECT COUNT(*) FROM docs").fetchone()[0] return Stats(added=added, updated=updated, deleted=len(gone), diff --git a/tests/stacklets/test_memory_fts_index.py b/tests/stacklets/test_memory_fts_index.py index 4f2d3415..c8f0b74e 100644 --- a/tests/stacklets/test_memory_fts_index.py +++ b/tests/stacklets/test_memory_fts_index.py @@ -19,7 +19,11 @@ from __future__ import annotations +import sqlite3 +import subprocess import sys +import threading +import time from pathlib import Path import pytest @@ -286,6 +290,143 @@ def test_an_edited_page_is_reindexed_and_a_deleted_one_disappears(vault, index): ] +# ── staying current ───────────────────────────────────────────────────── + +@pytest.fixture +def git_vault(vault: Path, git_commit) -> Path: + """The vault as what it actually is in production: a git checkout. + + The memory stacklet keeps a clone of the Forgejo vault repo, and + every change to it arrives as a commit -- `update_memory` writes + through Forgejo and fast-forwards this copy. Nothing edits these + files in place, which is the fact the freshness check below leans + on. + """ + subprocess.run(["git", "init", "-q", "-b", "main", str(vault)], check=True) + for key, value in (("user.email", "test@famstack.local"), + ("user.name", "Test"), ("commit.gpgsign", "false")): + subprocess.run(["git", "-C", str(vault), "config", key, value], + check=True) + git_commit(vault, "family/about.md", + "---\ntitle: Family\ndate: 2026-09-01\n---\n\nThe household.\n", + "seed") + return vault + + +def test_a_page_that_arrives_in_a_commit_becomes_searchable( + git_vault, tmp_path, git_commit): + """The case that matters: somebody wrote something, and we can find it. + + A pull is how every change reaches this checkout, so "the index is + current" means "the index has caught up with HEAD". + """ + db = tmp_path / "index.sqlite3" + fts_index.build_index(git_vault, db) + assert fts_index.search(db, ["kayak"]) == [] + + git_commit(git_vault, "family/camping/kayak.md", + "---\ntitle: Kayak\ndate: 2026-09-14\n---\n\nThe kayak leaks.\n", + "add kayak note") + fts_index.build_index(git_vault, db) + + assert rels(fts_index.search(db, ["kayak"])) == ["family/camping/kayak.md"] + + +def test_an_unmoved_head_costs_nothing_to_confirm(git_vault, tmp_path): + """A search must not pay for a scan when nothing can have changed. + + Statting every file to learn that none of them moved is work + proportional to the vault, on every single search. The checkout's + HEAD answers the same question in constant time, and it is a sound + substitute precisely because nothing writes into this tree outside + git: no commit, no change. + + The edit below is therefore deliberately invisible. That is the + trade: a hand-edited working copy is not picked up until something + commits. In production nothing hand-edits it, and `--vault` + overrides are not checkouts at all, so they keep scanning. + """ + db = tmp_path / "index.sqlite3" + fts_index.build_index(git_vault, db) + + (git_vault / "family/camping/about.md").write_text( + "---\ntitle: Camping trip\ndate: 2026-09-01\n---\n\nA kayak now.\n", + encoding="utf-8") + + stats = fts_index.build_index(git_vault, db) + assert (stats.added, stats.updated, stats.deleted) == (0, 0, 0) + assert fts_index.search(db, ["kayak"]) == [] + + +def test_a_vault_that_is_not_a_checkout_is_always_rescanned(vault, tmp_path): + """The fallback the lab and `--vault` overrides run on. + + A directory that is not a clone has no HEAD to compare, so there is + nothing to short-circuit on and the scan is the only way to know. + """ + db = tmp_path / "index.sqlite3" + fts_index.build_index(vault, db) + + (vault / "family/camping/about.md").write_text( + "---\ntitle: Camping trip\ndate: 2026-09-01\n---\n\nA kayak now.\n", + encoding="utf-8") + + stats = fts_index.build_index(vault, db) + assert stats.updated == 1 + assert rels(fts_index.search(db, ["kayak"])) == ["family/camping/about.md"] + + +def test_one_unreadable_path_does_not_cost_every_other_result(vault, tmp_path): + """The vault moves underneath a scan, so a bad path must not be fatal. + + A sync can delete a file between the moment the walk lists it and + the moment it is read. One page that cannot be resolved is a page + missing from the results; it is not a reason for the family to get + an error instead of the other twelve. + """ + (vault / "family/dangling.md").symlink_to(vault / "nowhere.md") + + db = tmp_path / "index.sqlite3" + stats = fts_index.build_index(vault, db) + + assert stats.total == 4 + assert rels(fts_index.search(db, ["telescope"])) == [ + "marge/notes/gift-ideas.md" + ] + + +def test_a_second_process_indexing_does_not_fail_the_search(vault, tmp_path): + """The CLI and the archivist both reconcile, sometimes at once. + + SQLite serialises writers, and its default is to give up the + instant the file is busy. Without a wait, one search landing while + the other is mid-reconcile is an error rather than a short pause. + """ + db = tmp_path / "index.sqlite3" + fts_index.build_index(vault, db) + + holder = sqlite3.connect(db) + holder.execute("BEGIN EXCLUSIVE") + done: list[object] = [] + + def reconcile() -> None: + page(vault, "bart/notes/skateboard.md", + title="Skateboard", body="The deck cracked.") + done.append(fts_index.build_index(vault, db)) + + worker = threading.Thread(target=reconcile) + worker.start() + time.sleep(0.2) + holder.rollback() + holder.close() + worker.join(timeout=10) + + assert done, "the second indexer gave up instead of waiting" + assert rels(fts_index.search(db, ["skateboard"])) == [ + "bart/notes/skateboard.md" + ] + + def test_an_unchanged_vault_costs_no_rewrites(vault, index): """Rebuild runs on every search, so a no-change scan has to be cheap. diff --git a/tools/retrieval-lab/README.md b/tools/retrieval-lab/README.md index 082cf039..2f59dada 100644 --- a/tools/retrieval-lab/README.md +++ b/tools/retrieval-lab/README.md @@ -32,6 +32,7 @@ gitignored and disposable. | `generate.py` | renders `corpus.yaml` into `out/vault/` | | `evaluate.py` | runs every engine over every question, prints the tables | | `explain.py` | one question kind, side by side, for reading a regression | +| `freshness.py` | what the "is the index current" check costs as a vault grows | ## The engines it compares diff --git a/tools/retrieval-lab/freshness.py b/tools/retrieval-lab/freshness.py new file mode 100644 index 00000000..f4b5bee7 --- /dev/null +++ b/tools/retrieval-lab/freshness.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""What it costs to find out whether the index is still current. + + uv run --extra test python tools/retrieval-lab/freshness.py + +The index is a cache over a git checkout, so every search has to answer +"has anything changed" before it answers the question. There are two +ways to ask, and they scale differently: + + scan stat every *.md and compare against the stored mtime + head read the checkout's git HEAD and compare against the one + stored in the index + +The scan is O(pages). The head read is O(1) and is sound only because +nothing in famstack writes into the vault working copy outside git -- +`update_memory` commits to Forgejo and fast-forwards the clone, so a +file cannot change without HEAD moving. + +Both columns below are real `build_index` calls on a vault where +nothing changed: one on the checkout, one on the same tree with its +`.git` hidden so the short-circuit cannot fire. The third column is +what a cold rebuild costs, for the case where the index is deleted. +Read the curve rather than the 177-page corpus, where everything is +fast enough to hide the difference. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +sys.path.insert(0, str(REPO / "lib")) +sys.path.insert(0, str(REPO / "stacklets" / "memory")) + +import fts_index # noqa: E402 + +SIZES = (177, 1000, 3000, 8000) +REPEATS = 5 + + +def git(*args: str, cwd: Path) -> str: + return subprocess.run(["git", *args], cwd=cwd, capture_output=True, + text=True, check=True).stdout.strip() + + +def build_vault(pages: int, root: Path) -> Path: + """Render a vault of roughly `pages` files and commit it.""" + vault = root / f"vault-{pages}" + subprocess.run( + [sys.executable, str(HERE / "generate.py"), "--out", str(vault), + "--noise", str(max(pages - 27, 0))], + check=True, capture_output=True) + git("init", "-q", cwd=vault) + git("add", "-A", cwd=vault) + git("-c", "user.email=lab@famstack.dev", "-c", "user.name=lab", + "commit", "-qm", "corpus", cwd=vault) + return vault + + +def best_of(fn, repeats: int = REPEATS) -> float: + """Milliseconds for the fastest of `repeats` runs. + + Fastest rather than mean: this is measuring a floor cost that a + search pays, and the slow runs are the machine doing something + else, not the code doing more work. + """ + timings = [] + for _ in range(repeats): + started = time.perf_counter() + fn() + timings.append((time.perf_counter() - started) * 1000) + return min(timings) + + +def main() -> None: + root = HERE / "out" / "freshness" + if root.exists(): + shutil.rmtree(root) + root.mkdir(parents=True) + + print(f"{'pages':>6} {'head gate (ms)':>15} {'scan (ms)':>10} " + f"{'cold rebuild (ms)':>18}") + for pages in SIZES: + vault = build_vault(pages, root) + db = root / f"index-{pages}.sqlite3" + + cold_ms = best_of(lambda: fts_index.build_index(vault, db), repeats=1) + gated_ms = best_of(lambda: fts_index.build_index(vault, db)) + + # Same tree, no checkout to ask: the fallback path, and what + # every search would cost without the gate. + dot_git = vault / ".git" + dot_git.rename(vault.parent / f"git-{pages}") + try: + scan_ms = best_of(lambda: fts_index.build_index(vault, db)) + finally: + (vault.parent / f"git-{pages}").rename(dot_git) + + total = fts_index.build_index(vault, db).total + print(f"{total:>6} {gated_ms:>15.1f} {scan_ms:>10.1f} " + f"{cold_ms:>18.0f}") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-lab/generate.py b/tools/retrieval-lab/generate.py index 77207c2f..39a000b4 100644 --- a/tools/retrieval-lab/generate.py +++ b/tools/retrieval-lab/generate.py @@ -89,9 +89,13 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", default=str(HERE / "out" / "vault"), help="directory to render the vault into") + parser.add_argument("--noise", type=int, default=None, + help="override the distractor count, for scale tests") ns = parser.parse_args() spec = yaml.safe_load((HERE / "corpus.yaml").read_text(encoding="utf-8")) + if ns.noise is not None: + spec["noise"]["count"] = ns.noise out = Path(ns.out) if out.exists(): for stale in sorted(out.rglob("*.md")): From 5aacfc6c9e6f14c37a8d232bb3be713911adb5b7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 15:39:45 +0200 Subject: [PATCH 05/11] test(memory): run the search engines through the agent The engine bench measures whether retrieval finds the right page, which is not the same as whether the family gets a better answer. This runs seven questions a single lookup cannot answer through the real agent on each backend, over a vault the size of a real one. The rig gains a --corpus flag, because thirteen pages cannot tell two engines apart, and lab-api gains a --backend flag serving the real engine rather than a copy. Result: ranking wins one answer out of seven and costs nothing; the confidence gate wins none and is dropped. A fixed result limit loses questions that need every matching page rather than the best one. --- docs/design/brain/retrieval-engine-poc.md | 121 ++++++++++++++-- pyproject.toml | 6 + tools/agent-lab/rig/lab-api.py | 101 +++++++++++--- tools/agent-lab/rig/rig.py | 12 +- tools/retrieval-lab/agent_ab.py | 160 ++++++++++++++++++++++ tools/retrieval-lab/generate.py | 16 ++- tools/retrieval-lab/replies.py | 51 +++++++ 7 files changed, 436 insertions(+), 31 deletions(-) create mode 100644 tools/retrieval-lab/agent_ab.py create mode 100644 tools/retrieval-lab/replies.py diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index ac7ed573..504a0ebc 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -195,16 +195,121 @@ pins the behaviour. ## What this changes in the plan -1. **Build tier 1 as FTS5 + trigram, fused with RRF**, not FTS5 alone. - The trigram table moves from stretch goal to first slice, because - without it the change is a regression for German compounds. -2. **Ship a confidence signal with the results, and make it coverage**, - not the score. `Hit.matched` already carries it. -3. **Tier 2 (embeddings) stays open**, with paraphrase recall at 29% as - the number it has to beat. -4. **Decide whether transliterated umlauts matter** before building for +Revised after the agentic run, which is the measurement that counts. + +1. **Ranking earns its place, narrowly.** One correctness win out of + seven at agent level, on the question the bench predicted, and no + measured cost in iterations. Not the landslide the engine numbers + suggested, because the agent was already covering for a lot of what + the old engine got wrong. +2. **Drop the confidence gate.** Its engine-level case was the best + number in this document and it bought nothing once a model was + reading the results. Keep `Hit.matched` as a signal on the + results; do not gate on it. +3. **A fixed result limit is the wrong shape for aggregate questions.** + Ranking plus `--limit 5` cost the agent a repair bill it would have + seen from the unranked dump. Fix before shipping, or aggregate + questions get quietly worse. +4. **Tier 2 (embeddings) stays open**, and the agentic run strengthens + its case rather than the index's: `expiring` failed on all three + backends because no engine reaches "läuft ab" from a page that says + "Kündigung muss drei Monate vorher raus". Paraphrase recall of 29% + is the number to beat. +5. **Decide whether transliterated umlauts matter** before building for them. +## The agentic test, and its kill criterion + +Everything above measures retrieval in isolation, which is an +intermediate result. An agent that can search twice and read a page +closes part of the gap without any of this. And at a vault of a few +hundred pages we sit in the tier where "BM25 Wins at Scale" puts the +file-system agent *ahead* of BM25; lexical retrieval wins there on cost +(39x fewer query tokens), not on accuracy. On local inference that cost +is the family's waiting time, so it still matters, but it is a +different argument from the one the handover made. + +So the deciding test is the agent answering questions that one lookup +cannot: several pages combined, arithmetic across them, or the +discipline to say a thing is not written down. Three arms, same +questions, 420-page vault: `regex`, `fts5`, `fts5+gate`. + +**Written before the run, so it cannot be adjusted to fit the result:** + +Ship the index if it does at least one of + +- answers a complex question correctly that `regex` gets wrong, or +- cuts tool iterations (`llm_calls`) on questions both get right, or +- stops an invented answer on an absent fact that `regex` invents. + +Otherwise drop the index, the trigram table and the RRF fusion, and +keep only the two cheap fixes: diacritic folding inside the existing +regex walk, and the coverage gate, which is a pure function over hits +and needs no index at all. + +## What the agent actually did + +420-page vault, seven questions, three backends, one run each. Model +`Qwen3.6-35B-A3B-UD-MLX-4bit` on the house oMLX. Replies in +`tools/retrieval-lab/out/agent-ab.json`, readable with `replies.py`. + +| Question | regex | fts5 | fts5+gate | +|---|---|---|---| +| camping-todo (multi-page) | partial, 3 calls | partial, 4 | **best**, 3 | +| repair-total (arithmetic) | **3 of 4 bills**, 7 | 2 of 4, 8 | 2 of 4, 6 | +| nut-cake (constraint) | **correct, 2** | correct, 3 | correct, 3 | +| expiring (temporal) | missed everything, 5 | noise, 9 | missed, 5 | +| feier (compound) | **wrong**, 4 | **correct**, 3 | **correct**, 3 | +| absent-birthday | declined, 12 | **declined, 8** | infra error, 11 | +| absent-ticket | declined, 5 | **declined better, 4** | declined, 10 | +| **total** | **38 calls, 274 s** | 39 calls, 324 s | 41 calls, 299 s | + +Against the criterion written before the run: + +1. **Answers a complex question correctly that regex gets wrong: yes, + once.** `feier`. Regex reported no guest count; both ranked arms + answered "vierzehn Leute, ab 15 Uhr" and cited the page, in fewer + calls. Diagnosis: the regex engine *can* find + `geburtstagsfeier.md`, because matching substrings is what it does. + Sorting by date then buried it below newer noise, outside the top + five. So this is a **ranking** win, not the compound-matching win + the engine bench predicted. Same symptom, different cause. +2. **Cuts tool iterations: no.** 38 calls against 39 and 41. The + per-question spread is noise at one run each. +3. **Stops an invented answer: no.** Nothing invented anything. All + three declined both absent facts correctly, and the regex arm + declined as cleanly as the gated one. + +### The gate earned nothing here + +Its engine-level case was strong: without it, 92% of unanswerable +questions still return pages. At agent level that never became a wrong +answer, because the model reads the pages and declines on its own. The +gate cost three extra calls across the set and produced the run's only +hard failure, an oMLX prefill guard rejection. **Drop it.** The +engine-level number was measuring a risk the reasoning layer was +already absorbing. + +### Ranking plus a hard limit loses aggregate questions + +`repair-total` is the one to keep. The regex engine returns everything +that matched, so the agent saw three of the four repair bills. The +ranked arms return the best five, and the fourth bill ranked sixth, so +they saw two and confidently answered 230 Euro. None of the three got +the right total. + +Ranking helps "which page answers this" and hurts "find every page +like this". That is not an argument against ranking; it is an argument +that a fixed `--limit 5` is the wrong shape for aggregate questions. + +### Honest limits on all of the above + +One run per cell, seven questions, a non-deterministic model. The call +counts are within noise and should not be read as a result. The one +correctness difference is more trustworthy because the bench predicted +that exact question would separate the engines, but a single run is a +single run. Repeats would be the next thing, not more questions. + ## What has not been measured The agent has not run against this. Everything above is the engine in diff --git a/pyproject.toml b/pyproject.toml index b4ce9a0d..c69a0eb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,6 +187,12 @@ extraPaths = ["lib", "stacklets/messages", "stacklets/messages/bot", "stacklets/ root = "stacklets/photos" extraPaths = ["lib", "stacklets/photos", "stacklets/core/bot-runner", "stacklets"] +[[tool.basedpyright.executionEnvironments]] +root = "tools/agent-lab/rig" +# lab-api serves the real search engines rather than copies, so it puts +# the memory stacklet on its path the way the CLI plugins there do. +extraPaths = ["lib", "stacklets/memory", "stacklets/memory/cli"] + [[tool.basedpyright.executionEnvironments]] root = "tools/retrieval-lab" # The lab drives both search engines head to head, so it bootstraps the diff --git a/tools/agent-lab/rig/lab-api.py b/tools/agent-lab/rig/lab-api.py index 63f43e7c..77e23cda 100644 --- a/tools/agent-lab/rig/lab-api.py +++ b/tools/agent-lab/rig/lab-api.py @@ -41,6 +41,11 @@ _list_edit_transform = _prod.apply_list_edit _list_edit_batch = _prod.apply_list_edits +# The ranked engine, for the `--backend fts5` arm. Imported from the +# stacklet rather than reimplemented, for the same reason the list-edit +# transform is: an A/B against a copy of the engine measures the copy. +import fts_index # noqa: E402 + def _body_only(text: str) -> str: """Strip YAML frontmatter, so field names do not match every page.""" @@ -77,6 +82,70 @@ def _rewrite_keywords(query: str) -> list[str]: return query.split() +# Coverage below this and the gated backend says it found nothing, +# rather than handing over its best guess. Measured in +# tools/retrieval-lab; see docs/design/brain/retrieval-engine-poc.md. +GATE = 0.40 + + +def _block(vault: Path, rel: str, pattern) -> str: + """One result, in the shape the agent has always been shown. + + Identical across backends on purpose. The A/B is about which pages + come back and in what order; changing how they are rendered would + change the model's reading of them too, and then the comparison + would be measuring two things at once. + """ + body = _body_only((vault / rel).read_text()) + lines = [ln for ln in body.splitlines() if pattern.search(ln)] + return f"— vault/{rel}\n" + "\n".join(f" {ln}" for ln in lines[:3]) + + +def _keywords_for(ns) -> list[str]: + """The search terms, resolved the same way for every backend.""" + if ns.nl and len(ns.query.split()) > 1: + return _rewrite_keywords(ns.query) + return ns.query.split() + + +def _search_regex(ns, vault: Path, pattern) -> tuple[str, int]: + """Today's engine: every file matched, newest first.""" + hits = [] + for path in sorted(vault.rglob("*.md")): + rel = path.relative_to(vault) + if ns.scope and not str(rel).startswith(ns.scope): + continue + text = path.read_text() + if any(pattern.search(ln) for ln in _body_only(text).splitlines()): + m = re.search(r"^date:\s*(\S+)", text, re.MULTILINE) + hits.append((m.group(1) if m else "", _block(vault, str(rel), pattern))) + if not hits: + return "no results\n", 1 + hits.sort(key=lambda h: h[0], reverse=True) + return "\n".join(h[1] for h in hits[: ns.limit]) + "\n", 0 + + +def _search_ranked(ns, vault: Path, keywords, pattern, + gated: bool) -> tuple[str, int]: + """The ranked engine: best match first, optionally behind a gate.""" + db = vault.parent / "vault-index.sqlite3" + fts_index.build_index(vault, db) + hits = fts_index.search( + db, keywords, scopes=[ns.scope] if ns.scope else None, + limit=ns.limit, substrings=True) + if not hits: + return "no results\n", 1 + if gated: + covered = len(hits[0].matched) / max(len(keywords), 1) + if covered < GATE: + # The honest answer when the best page carries almost none + # of what was asked. Says so plainly rather than handing + # over a neighbour for the model to answer from. + return ("no confident match: the closest pages do not contain " + "what you asked about\n"), 1 + return "\n".join(_block(vault, h.rel, pattern) for h in hits) + "\n", 0 + + def memory_search(argv: list[str]) -> tuple[str, int]: parser = argparse.ArgumentParser(prog="stack memory search", add_help=False) parser.add_argument("query") @@ -90,9 +159,10 @@ def memory_search(argv: list[str]) -> tuple[str, int]: except SystemExit: return parser.format_usage(), 2 + keywords = _keywords_for(ns) if ns.nl and len(ns.query.split()) > 1: - terms = _rewrite_keywords(ns.query) - pattern = re.compile("|".join(re.escape(t) for t in terms), re.IGNORECASE) + pattern = re.compile("|".join(re.escape(t) for t in keywords), + re.IGNORECASE) else: # The real CLI treats the query as a regex (lib.py search engine). # Keep that contract; fall back to a literal match on a bad regex. @@ -102,24 +172,10 @@ def memory_search(argv: list[str]) -> tuple[str, int]: pattern = re.compile(re.escape(ns.query), re.IGNORECASE) vault = Path(ARGS.vault) - hits = [] - for path in sorted(vault.rglob("*.md")): - rel = path.relative_to(vault) - if ns.scope and not str(rel).startswith(ns.scope): - continue - text = path.read_text() - body = _body_only(text) - lines = [ln for ln in body.splitlines() if pattern.search(ln)] - if lines: - m = re.search(r"^date:\s*(\S+)", text, re.MULTILINE) - date = m.group(1) if m else "" - hits.append((date, f"— vault/{rel}\n" - + "\n".join(f" {ln}" for ln in lines[:3]))) - if not hits: - return "no results\n", 1 - # The real CLI sorts by frontmatter date, newest first, then limits. - hits.sort(key=lambda h: h[0], reverse=True) - return "\n".join(h[1] for h in hits[: ns.limit]) + "\n", 0 + if ARGS.backend == "regex": + return _search_regex(ns, vault, pattern) + return _search_ranked(ns, vault, keywords, pattern, + gated=ARGS.backend == "fts5+gate") def memory_person(argv: list[str]) -> tuple[str, int]: @@ -330,6 +386,11 @@ def main(): parser.add_argument("--llm", default="http://localhost:8888/v1") parser.add_argument("--key", default="none") parser.add_argument("--model", default=None) + parser.add_argument("--backend", default="regex", + choices=["regex", "fts5", "fts5+gate"], + help="search engine to serve: the shipping regex " + "walk, the ranked index, or the ranked index " + "behind a confidence gate") ARGS = parser.parse_args() socketserver.ThreadingTCPServer.allow_reuse_address = True server = socketserver.ThreadingTCPServer(("127.0.0.1", ARGS.listen), Handler) diff --git a/tools/agent-lab/rig/rig.py b/tools/agent-lab/rig/rig.py index aeae9476..b3152525 100644 --- a/tools/agent-lab/rig/rig.py +++ b/tools/agent-lab/rig/rig.py @@ -64,8 +64,14 @@ def cmd_reset(_args): # The brief reads git history from the vault. Make the demo vault a # real git repo, with the same shape as the production clone. + # + # `--corpus` seeds from somewhere else. Thirteen pages is enough to + # exercise the scenarios but not to tell two search engines apart: + # an agent that can read every page finds everything regardless of + # ranking. Comparing engines needs a vault the size of a real one. vault = STATE / "vault" - shutil.copytree(RIG / "demo-vault", vault) + shutil.copytree(Path(_args.corpus) if _args.corpus else RIG / "demo-vault", + vault) env = {"GIT_AUTHOR_NAME": "homer", "GIT_AUTHOR_EMAIL": "homer@demo.invalid", "GIT_COMMITTER_NAME": "homer", "GIT_COMMITTER_EMAIL": "homer@demo.invalid"} for cmd in (["git", "init", "-q", "-b", "main"], @@ -160,7 +166,9 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="cmd", required=True) sub.add_parser("build") - sub.add_parser("reset") + p_reset = sub.add_parser("reset") + p_reset.add_argument("--corpus", default=None, metavar="DIR", + help="seed the vault from DIR instead of demo-vault") p_turn = sub.add_parser("turn") p_turn.add_argument("message") p_turn.add_argument("--session", default="rig:main") diff --git a/tools/retrieval-lab/agent_ab.py b/tools/retrieval-lab/agent_ab.py new file mode 100644 index 00000000..48464a64 --- /dev/null +++ b/tools/retrieval-lab/agent_ab.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Run the same hard questions through the agent on each search backend. + + uv run --extra test python tools/retrieval-lab/agent_ab.py + +The engine bench measures whether retrieval hands over the right page. +That is an intermediate result. What decides whether any of this is +worth shipping is whether the *family* gets a better answer, and an +agent that can search twice and read a page closes a lot of the gap on +its own. So these questions are deliberately ones a single lookup +cannot answer: they need several pages combined, arithmetic across +them, or the discipline to say "that is not written down anywhere". + +Three backends, served by three lab-api instances on three ports so a +turn can pick one without a restart: + + regex today's engine: every file matched, newest first + fts5 the ranked index, best match first + fts5+gate the same, but silent when the best hit carries almost + none of what was asked + +Prerequisites, all started by hand first (see the rig README): + + rig.py reset --corpus + proxy.py --target + lab-api.py --listen 42011 --backend regex ... and 42013, 42014 + +Every turn runs in its own session, so no answer is helped by a +previous one, and `--no-log` keeps 20-odd engine turns out of the +agent improvement log. The summary goes to `out/agent-ab.json`, replies +included, because the numbers cannot tell you whether an answer was +actually right. That part is a human reading them. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +RIG = REPO / "tools" / "agent-lab" / "rig" / "rig.py" + +BACKENDS = {"regex": 42011, "fts5": 42013, "fts5+gate": 42014} + +# Each question needs more than one page, or needs the agent to decline. +# `expect` is what a correct answer has to contain, for a human reading +# the replies afterwards; nothing here is scored automatically, because +# an answer can carry the right number and still be wrong about why. +SCENARIOS = [ + { + "id": "camping-todo", + "q": "Was müssen wir vor der Campingfahrt noch erledigen?", + "kind": "multi-page", + "expect": "Kocher defekt (Neds leihen), Anzahlung ist raus / Rest vor " + "Ort, Batterien für die Campinglampe", + }, + { + "id": "repair-total", + "q": "Wie viel haben wir dieses Jahr insgesamt für Reparaturen bezahlt?", + "kind": "arithmetic", + "expect": "90 + 320 + 20 + 140 = 570 Euro (Spülmaschine, TÜV, " + "Fahrrad, Heizung)", + }, + { + "id": "nut-cake", + "q": "Können wir Tante Hilde die Nusstorte servieren?", + "kind": "constraint", + "expect": "Nein, Haselnussallergie, auch keine Spuren; " + "Zitronenkuchen stattdessen", + }, + { + "id": "expiring", + "q": "Was läuft demnächst ab oder muss rechtzeitig erneuert werden?", + "kind": "temporal", + "expect": "Barts Pass (Mai), Hausratversicherung (Kündigung 3 Monate " + "vor Jahresende), Winterreifen (vor Oktober), Heizung " + "(12 Monate)", + }, + { + "id": "feier", + "q": "Wann ist die Feier und wie viele Leute kommen?", + "kind": "compound", + "expect": "Sonntagnachmittag, Anfang um drei, vierzehn Zusagen", + }, + { + "id": "absent-birthday", + "q": "Wann hat Maggie Geburtstag?", + "kind": "absent", + "expect": "must say it is not in the vault; must not borrow Omas " + "Geburtstagsfeier", + }, + { + "id": "absent-ticket", + "q": "Wie viel hat das Flugticket gekostet?", + "kind": "absent-adjacent", + "expect": "must say the price is not recorded; the 400 Euro on that " + "page is compensation, not the ticket", + }, +] + + +def run_turn(question: str, backend: str, port: int, session: str) -> dict: + started = time.monotonic() + result = subprocess.run( + [sys.executable, str(RIG), "turn", question, + "--session", session, "--no-log", + "--env", f"STACK_API_ADDR=host.docker.internal:{port}"], + capture_output=True, text=True) + wall = time.monotonic() - started + + # rig.py prints the reply, then the metrics object last. + summary: dict = {} + head, brace, tail = result.stdout.rpartition("\n{\n") + if brace: + try: + summary = json.loads("{\n" + tail) + except ValueError: + head = result.stdout + else: + head = result.stdout + return { + "backend": backend, + "reply": head.strip(), + "wall_s": summary.get("wall_s", round(wall, 2)), + "llm_calls": summary.get("llm_calls"), + "rows": summary.get("rows", []), + "exit": result.returncode, + "stderr": result.stderr[-400:] if result.returncode else "", + } + + +def main() -> None: + out: list[dict] = [] + for scenario in SCENARIOS: + for backend, port in BACKENDS.items(): + session = f"ab:{scenario['id']}:{backend.replace('+', '-')}" + print(f"[{backend:>9}] {scenario['id']}", flush=True) + turn = run_turn(scenario["q"], backend, port, session) + out.append({**scenario, **turn}) + print(f" {turn['llm_calls']} calls, " + f"{turn['wall_s']}s", flush=True) + + target = HERE / "out" / "agent-ab.json" + target.write_text(json.dumps(out, indent=2, ensure_ascii=False), + encoding="utf-8") + + print("\n" + "=" * 70) + print(f"{'question':<18} {'backend':>9} {'calls':>5} {'wall_s':>7}") + for row in out: + print(f"{row['id']:<18} {row['backend']:>9} " + f"{str(row['llm_calls']):>5} {row['wall_s']:>7}") + print(f"\nreplies: {target}") + + +if __name__ == "__main__": + main() diff --git a/tools/retrieval-lab/generate.py b/tools/retrieval-lab/generate.py index 39a000b4..fb102cf1 100644 --- a/tools/retrieval-lab/generate.py +++ b/tools/retrieval-lab/generate.py @@ -91,6 +91,10 @@ def main() -> None: help="directory to render the vault into") parser.add_argument("--noise", type=int, default=None, help="override the distractor count, for scale tests") + parser.add_argument("--include", default=None, metavar="DIR", + help=("copy another vault's pages in first, so the " + "agent rig keeps its own scenario pages while " + "gaining a corpus big enough to rank over")) ns = parser.parse_args() spec = yaml.safe_load((HERE / "corpus.yaml").read_text(encoding="utf-8")) @@ -102,13 +106,23 @@ def main() -> None: stale.unlink() out.mkdir(parents=True, exist_ok=True) + included = 0 + if ns.include: + source = Path(ns.include) + for page in sorted(source.rglob("*.md")): + target = out / page.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(page.read_bytes()) + included += 1 + facts = write_facts(spec, out) noise = write_noise(spec, out) (out.parent / "fact-paths.yaml").write_text( yaml.safe_dump(facts, allow_unicode=True, sort_keys=True), encoding="utf-8") - print(f"{len(facts)} fact pages + {noise} noise pages -> {out}") + carried = f"{included} carried + " if included else "" + print(f"{carried}{len(facts)} fact pages + {noise} noise pages -> {out}") if __name__ == "__main__": diff --git a/tools/retrieval-lab/replies.py b/tools/retrieval-lab/replies.py new file mode 100644 index 00000000..2c0a6302 --- /dev/null +++ b/tools/retrieval-lab/replies.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Print what the agent actually answered, per question, per backend. + + uv run --extra test python tools/retrieval-lab/replies.py [id ...] + +Call counts and wall time are the easy half. Whether the answer was +right is the half that decides anything, and no script can score it: +an answer can carry the correct total and still be wrong about which +bills it added. So this strips the runtime chatter and prints the +replies side by side for a human to read against `expect`. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def answer(reply: str) -> str: + """Drop nanobot's banner and thinking lines, keep the reply.""" + kept = [ + line for line in reply.splitlines() + if line.strip() + and not line.startswith(("✻", "🐈", "[llm-state]")) + ] + return "\n".join(kept).strip() + + +def main() -> None: + rows = json.loads( + (HERE / "out" / "agent-ab.json").read_text(encoding="utf-8")) + wanted = sys.argv[1:] + for scenario in dict.fromkeys(r["id"] for r in rows): + if wanted and scenario not in wanted: + continue + group = [r for r in rows if r["id"] == scenario] + print("\n" + "=" * 72) + print(f"{scenario} ({group[0]['kind']})") + print(f"Q: {group[0]['q']}") + print(f"EXPECT: {group[0]['expect']}") + for row in group: + print(f"\n--- {row['backend']} " + f"({row['llm_calls']} calls, {row['wall_s']}s) ---") + print(answer(row["reply"]) or "(no reply)") + + +if __name__ == "__main__": + main() From f9121010e6911ef21dc82faf772ca1dca3cda7c4 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 16:10:21 +0200 Subject: [PATCH 06/11] test(memory): repeat the agent A/B, and correct what it showed One turn per cell could not tell a real difference from a dice roll, and the conclusions drawn from it about individual questions were wrong. This repeats each cell three times, alternates which backend runs first so neither always pays the cold prefix cache, and records every search both arms were given. Result: no difference between the engines that survives the run-to-run spread. The one claimed correctness win was the old engine's single bad draw, correct 2 of 3 times on a repeat. The search log also shows why the engine gains do not reach the family: the agent searches one word at a time, so there is nothing for a ranking scheme to combine, and the two arms shared only 19% of their keywords. --- docs/design/brain/retrieval-engine-poc.md | 111 +++++++++++++------ tools/agent-lab/rig/lab-api.py | 39 ++++++- tools/retrieval-lab/agent_ab.py | 53 ++++++--- tools/retrieval-lab/compare.py | 128 ++++++++++++++++++++++ 4 files changed, 281 insertions(+), 50 deletions(-) create mode 100644 tools/retrieval-lab/compare.py diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index 504a0ebc..95321501 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -195,28 +195,35 @@ pins the behaviour. ## What this changes in the plan -Revised after the agentic run, which is the measurement that counts. - -1. **Ranking earns its place, narrowly.** One correctness win out of - seven at agent level, on the question the bench predicted, and no - measured cost in iterations. Not the landslide the engine numbers - suggested, because the agent was already covering for a lot of what - the old engine got wrong. +Revised after the repeated agentic run, which is the measurement that +counts. + +1. **Do not ship the index.** Three repeats, seven questions, two arms: + no difference in answers, iterations or wall time that survives the + run-to-run spread. The engine bench's 23% to 60% recall@1 is real + and does not reach the family, because the agent closes the gap by + iterating. 2. **Drop the confidence gate.** Its engine-level case was the best number in this document and it bought nothing once a model was - reading the results. Keep `Hit.matched` as a signal on the - results; do not gate on it. -3. **A fixed result limit is the wrong shape for aggregate questions.** - Ranking plus `--limit 5` cost the agent a repair bill it would have - seen from the unranked dump. Fix before shipping, or aggregate - questions get quietly worse. -4. **Tier 2 (embeddings) stays open**, and the agentic run strengthens - its case rather than the index's: `expiring` failed on all three - backends because no engine reaches "läuft ab" from a page that says - "Kündigung muss drei Monate vorher raus". Paraphrase recall of 29% - is the number to beat. -5. **Decide whether transliterated umlauts matter** before building for - them. + reading the results. The 92% figure was measuring a risk the + reasoning layer already absorbs. +3. **The agent searches one word at a time.** Median search: one + keyword. Any ranking scheme that earns its keep by combining + evidence across terms has nothing to work with. Changing *how the + agent queries* is a bigger lever than changing what answers it, and + it is free. +4. **Raise the prefill ceiling or trim the agent's context.** Five of + six runs of the multi-search question died on an oMLX memory guard. + That is a harder limit on complex questions than retrieval quality + is, and it is unrelated to any of this work. +5. **Tier 2 (embeddings) is the remaining lever on quality.** + `expiring` failed on every arm and every run, because no lexical + engine reaches "läuft ab" from a page saying "Kündigung muss drei + Monate vorher raus". Paraphrase recall of 29% is the number to beat. + +The two cheap fixes still stand on their own, and neither needs an +index: diacritic folding inside the existing regex walk, and keeping +`Hit.matched`-style coverage as a signal rather than a gate. ## The agentic test, and its kill criterion @@ -264,21 +271,59 @@ and needs no index at all. | absent-ticket | declined, 5 | **declined better, 4** | declined, 10 | | **total** | **38 calls, 274 s** | 39 calls, 324 s | 41 calls, 299 s | +That first pass ran one turn per cell, and every conclusion drawn from +it about individual questions was wrong. A second pass repeated each +cell three times, alternated which arm went first so neither always +paid the cold prefix cache, and logged every search both arms received. + +| | regex | fts5 | +|---|---|---| +| calls per run | 36 [35-41] | 37 [33-37] | +| wall seconds per run | 244 [238-279] | 261 [238-332] | + Against the criterion written before the run: -1. **Answers a complex question correctly that regex gets wrong: yes, - once.** `feier`. Regex reported no guest count; both ranked arms - answered "vierzehn Leute, ab 15 Uhr" and cited the page, in fewer - calls. Diagnosis: the regex engine *can* find - `geburtstagsfeier.md`, because matching substrings is what it does. - Sorting by date then buried it below newer noise, outside the top - five. So this is a **ranking** win, not the compound-matching win - the engine bench predicted. Same symptom, different cause. -2. **Cuts tool iterations: no.** 38 calls against 39 and 41. The - per-question spread is noise at one run each. -3. **Stops an invented answer: no.** Nothing invented anything. All - three declined both absent facts correctly, and the regex arm - declined as cleanly as the gated one. +1. **Answers a complex question correctly that regex gets wrong: no.** + `feier` was the claimed win. Over three runs the ranked arm is + correct 3 of 3 and the regex arm 2 of 3. The single run that + started all this was regex's one bad draw. At n=3 that is not a + difference, and the mechanism story built on top of it (date + sorting buries the page) was explaining noise. +2. **Cuts tool iterations: no.** The totals overlap. +3. **Stops an invented answer: no.** Nothing invented anything, on + either arm, in any run. + +### The comparison was weaker than it looked + +The agent writes its own query for each search, and the search logs +show the two arms were barely asked the same things: **19% keyword +vocabulary overlap**, 18 shared terms out of 96 distinct. A difference +between the arms would have been as easily explained by the agent +happening to ask one of them better questions. + +### Why the engine gains do not reach the agent + +The search log answers this. The **median search carries one keyword**. +The agent does not hand over the 2-4 term queries the bench fed the +engines; it sends a single word, looks, and sends another. BM25 ranks +by combining evidence across terms, and there is almost nothing to +combine. The agent harness turns search into grep no matter what is +underneath it, which is the same effect "Is Grep All You Need?" +reports. + +One mechanical difference did survive: the ranked arm dead-ends less +often, 16 empty results of 79 searches against 27 of 82. It did not +convert into fewer iterations or better answers. + +### An infrastructure limit, not a retrieval one + +`repair-total` hit an oMLX prefill guard rejection in **5 of 6 runs**, +on both arms: `predicted peak would exceed prefill safety cap 46.8GB +... kv_len=8192`. Questions that need several searches grow the +context past what the endpoint will prefill. The earlier claim that +regex "saw three of four repair bills" was reading whichever arm got +further before erroring. That question measures the endpoint, not the +engine, until the guard is raised or the context trimmed. ### The gate earned nothing here diff --git a/tools/agent-lab/rig/lab-api.py b/tools/agent-lab/rig/lab-api.py index 77e23cda..410934b7 100644 --- a/tools/agent-lab/rig/lab-api.py +++ b/tools/agent-lab/rig/lab-api.py @@ -173,9 +173,39 @@ def memory_search(argv: list[str]) -> tuple[str, int]: vault = Path(ARGS.vault) if ARGS.backend == "regex": - return _search_regex(ns, vault, pattern) - return _search_ranked(ns, vault, keywords, pattern, - gated=ARGS.backend == "fts5+gate") + text, code = _search_regex(ns, vault, pattern) + else: + text, code = _search_ranked(ns, vault, keywords, pattern, + gated=ARGS.backend == "fts5+gate") + _log_search(ns, keywords, text, code) + return text, code + + +def _log_search(ns, keywords, text: str, code: int) -> None: + """Record what was asked and what came back, for reading a run after. + + The agent picks its own query for each search, so two backends + never receive quite the same thing. Without this, a difference + between them cannot be told apart from the agent having asked + better questions of one of them, and the whole comparison rests on + trust. Appends rather than truncates, so an arm's whole run is in + one place. + """ + record = { + "backend": ARGS.backend, + "query": ns.query, + "nl": ns.nl, + "keywords": keywords, + "scope": ns.scope, + "limit": ns.limit, + "exit": code, + "pages": [ln[len("— vault/"):] for ln in text.splitlines() + if ln.startswith("— vault/")], + } + path = Path(ARGS.search_log) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=False) + "\n") def memory_person(argv: list[str]) -> tuple[str, int]: @@ -386,6 +416,9 @@ def main(): parser.add_argument("--llm", default="http://localhost:8888/v1") parser.add_argument("--key", default="none") parser.add_argument("--model", default=None) + parser.add_argument("--search-log", default=str( + Path(__file__).parent / "state" / "search-log.jsonl"), + help="append every search and its hits here, for run analysis") parser.add_argument("--backend", default="regex", choices=["regex", "fts5", "fts5+gate"], help="search engine to serve: the shipping regex " diff --git a/tools/retrieval-lab/agent_ab.py b/tools/retrieval-lab/agent_ab.py index 48464a64..9e1f088e 100644 --- a/tools/retrieval-lab/agent_ab.py +++ b/tools/retrieval-lab/agent_ab.py @@ -34,6 +34,7 @@ from __future__ import annotations +import argparse import json import subprocess import sys @@ -134,25 +135,49 @@ def run_turn(question: str, backend: str, port: int, session: str) -> dict: def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repeat", type=int, default=1, + help="runs per (question, backend); the model is " + "not deterministic, so one is a dice roll") + parser.add_argument("--backends", default=",".join(BACKENDS), + help="comma-separated subset to run") + parser.add_argument("--out", default=str(HERE / "out" / "agent-ab.json")) + ns = parser.parse_args() + + arms = [(name, BACKENDS[name]) for name in ns.backends.split(",") + if name in BACKENDS] + out: list[dict] = [] - for scenario in SCENARIOS: - for backend, port in BACKENDS.items(): - session = f"ab:{scenario['id']}:{backend.replace('+', '-')}" - print(f"[{backend:>9}] {scenario['id']}", flush=True) - turn = run_turn(scenario["q"], backend, port, session) - out.append({**scenario, **turn}) - print(f" {turn['llm_calls']} calls, " - f"{turn['wall_s']}s", flush=True) - - target = HERE / "out" / "agent-ab.json" + for run in range(ns.repeat): + # Whichever arm goes first pays the cold prefix cache for that + # question. Alternating means neither arm always pays it, so a + # wall-time difference is about the engine rather than about + # the running order. + ordered = arms if run % 2 == 0 else list(reversed(arms)) + for scenario in SCENARIOS: + for backend, port in ordered: + session = (f"ab{run}:{scenario['id']}:" + f"{backend.replace('+', '-')}") + print(f"[run {run} {backend:>9}] {scenario['id']}", flush=True) + turn = run_turn(scenario["q"], backend, port, session) + out.append({**scenario, **turn, "run": run}) + print(f" {turn['llm_calls']} calls, " + f"{turn['wall_s']}s", flush=True) + + target = Path(ns.out) target.write_text(json.dumps(out, indent=2, ensure_ascii=False), encoding="utf-8") print("\n" + "=" * 70) - print(f"{'question':<18} {'backend':>9} {'calls':>5} {'wall_s':>7}") - for row in out: - print(f"{row['id']:<18} {row['backend']:>9} " - f"{str(row['llm_calls']):>5} {row['wall_s']:>7}") + print(f"{'question':<18} {'backend':>9} {'calls':>16} {'wall_s':>16}") + for scenario in SCENARIOS: + for backend, _ in arms: + rows = [r for r in out + if r["id"] == scenario["id"] and r["backend"] == backend] + calls = [r["llm_calls"] for r in rows if r["llm_calls"]] + walls = [r["wall_s"] for r in rows if r["wall_s"]] + print(f"{scenario['id']:<18} {backend:>9} " + f"{str(calls):>16} {str(walls):>16}") print(f"\nreplies: {target}") diff --git a/tools/retrieval-lab/compare.py b/tools/retrieval-lab/compare.py new file mode 100644 index 00000000..67dbfe64 --- /dev/null +++ b/tools/retrieval-lab/compare.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Aggregate a repeated agent A/B, and check it was a fair fight. + + uv run --extra test python tools/retrieval-lab/compare.py \ + --results out/agent-ab-repeat.json \ + --logs ../agent-lab/rig/state/search-regex.jsonl \ + ../agent-lab/rig/state/search-fts5.jsonl + +Two halves. + +**The numbers, across repeats.** One run per cell cannot tell a real +difference from a dice roll, so this reports the spread rather than a +single value. A median that moves less than the run-to-run range has +not moved. + +**Whether the comparison was fair.** The agent writes its own query for +each search, so the two backends never receive quite the same input. +If one arm happened to be asked better questions, a difference between +them says nothing about the engines. The search logs make that +visible: how many searches each arm was given, and how much the +queries overlap. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +from collections import defaultdict +from pathlib import Path +from typing import Sequence + +HERE = Path(__file__).resolve().parent + + +def spread(values: Sequence[float]) -> str: + if not values: + return "-" + if len(values) == 1: + return f"{values[0]:g}" + return f"{statistics.median(values):g} [{min(values):g}-{max(values):g}]" + + +def numbers(rows: list[dict]) -> None: + backends = list(dict.fromkeys(r["backend"] for r in rows)) + ids = list(dict.fromkeys(r["id"] for r in rows)) + + print(f"\n{'question':<18} {'backend':>8} {'llm calls':>16} " + f"{'wall seconds':>20}") + print("-" * 70) + for qid in ids: + for backend in backends: + cell = [r for r in rows + if r["id"] == qid and r["backend"] == backend] + calls = [r["llm_calls"] for r in cell if r["llm_calls"]] + walls = [r["wall_s"] for r in cell if r["wall_s"]] + print(f"{qid:<18} {backend:>8} {spread(calls):>16} " + f"{spread(walls):>20}") + + print(f"\n{'backend':>8} {'total calls per run':>22} " + f"{'total wall per run':>22} {'errors':>7}") + for backend in backends: + per_run = defaultdict(lambda: [0, 0.0]) + errors = 0 + for r in rows: + if r["backend"] != backend: + continue + if r.get("exit"): + errors += 1 + per_run[r.get("run", 0)][0] += r["llm_calls"] or 0 + per_run[r.get("run", 0)][1] += r["wall_s"] or 0 + calls = [v[0] for v in per_run.values()] + walls = [round(v[1]) for v in per_run.values()] + print(f"{backend:>8} {spread(calls):>22} {spread(walls):>22} " + f"{errors:>7}") + + +def fairness(log_paths: list[Path]) -> None: + """Did both arms get asked comparable questions?""" + by_backend: dict[str, list[dict]] = defaultdict(list) + for path in log_paths: + if not path.exists(): + print(f"\n(no search log at {path})") + continue + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + record = json.loads(line) + by_backend[record["backend"]].append(record) + + if not by_backend: + return + + print(f"\n{'backend':>8} {'searches':>9} {'median terms':>13} " + f"{'empty results':>14} {'distinct queries':>17}") + for backend, records in by_backend.items(): + terms = [len(r.get("keywords") or []) for r in records] + empty = sum(1 for r in records if r["exit"] != 0) + distinct = len({r["query"].lower() for r in records}) + print(f"{backend:>8} {len(records):>9} " + f"{statistics.median(terms) if terms else 0:>13g} " + f"{empty:>14} {distinct:>17}") + + vocab = {b: {w.lower() for r in rs for w in (r.get("keywords") or [])} + for b, rs in by_backend.items()} + names = list(vocab) + if len(names) == 2: + a, b = vocab[names[0]], vocab[names[1]] + shared = len(a & b) / max(len(a | b), 1) + print(f"\nkeyword vocabulary overlap between arms: {shared:.0%} " + f"({len(a & b)} shared of {len(a | b)} distinct)") + print("Low overlap means the arms were asked different things and " + "the comparison is weaker than it looks.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results", + default=str(HERE / "out" / "agent-ab-repeat.json")) + parser.add_argument("--logs", nargs="*", default=[]) + ns = parser.parse_args() + + rows = json.loads(Path(ns.results).read_text(encoding="utf-8")) + numbers(rows) + fairness([Path(p) for p in ns.logs]) + + +if __name__ == "__main__": + main() From 2ca111657e5aed7a13b18e20a0880968ed3fb5c5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 16:47:03 +0200 Subject: [PATCH 07/11] feat(memory): find pages by their title, and despite an umlaut MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps in the vault search, both visible to anyone who types a question into chat. A page titled "Zahnarzttermin Lisa" could not be found by searching for Zahnarzttermin unless the body happened to repeat it. Frontmatter was stripped before matching so that a query for "date" would not hit every page through its date line, and that threw the title and tags out with the field names. The values are back; the keys are still out. Person names stay out too, because a name says who a page concerns rather than what it says, and --person already asks that. Searching "Kase" for a page that says "Käse" returned nothing at all, which reads like the vault has no such page rather than like the word was spelled differently. Both sides now fold before matching. Spelled out umlauts ("Kaese") are still a different word. Measured on the retrieval lab: recall@1 21% to 26%, recall@5 54% to 64%, no question kind worse. --- docs/design/brain/retrieval-engine-poc.md | 69 +++++++++- stacklets/memory/fts_index.py | 15 +- stacklets/memory/lib.py | 89 ++++++++++-- tests/stacklets/test_memory_search.py | 160 ++++++++++++++++++++++ tools/retrieval-lab/evaluate.py | 14 +- tools/retrieval-lab/goldset.yaml | 35 ++++- 6 files changed, 357 insertions(+), 25 deletions(-) diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index 95321501..878f864c 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -221,9 +221,15 @@ counts. engine reaches "läuft ab" from a page saying "Kündigung muss drei Monate vorher raus". Paraphrase recall of 29% is the number to beat. -The two cheap fixes still stand on their own, and neither needs an -index: diacritic folding inside the existing regex walk, and keeping -`Hit.matched`-style coverage as a signal rather than a gate. +6. **The two cheap fixes are in**, and neither needs an index. See + "The cheap fixes, measured" below: recall@1 21% to 26%, recall@5 + 54% to 64%, no class worse, about fifteen lines. + +**Coverage as a displayed signal is not done, on purpose.** It was +listed as a cheap fix, but the only thing measured about coverage is +that *gating* on it hurt. Showing it to the model is untested, and it +adds output the agent pays to read on its hot path. It needs a reason +before it needs an implementation. ## The agentic test, and its kill criterion @@ -355,6 +361,63 @@ correctness difference is more trustworthy because the bench predicted that exact question would separate the engines, but a single run is a single run. Repeats would be the next thing, not more questions. +## The cheap fixes, measured + +Two changes to the existing regex walk, about fifteen lines, no index, +no new dependency: fold combining marks on both sides before matching, +and match the title and tag *values* alongside the body. + +| | recall@1 | recall@5 | MRR | p50 | +|---|---|---|---|---| +| regex (before) | 21% | 54% | 0.33 | 10.5 ms | +| regex + both fixes | **26%** | **64%** | **0.41** | 12.0 ms | +| fts5 + trigram | 62% | 72% | 0.65 | 1.3 ms | + +By question kind, recall@1 / recall@5: + +| kind | regex | regex+cheap | +|---|---|---| +| keyword | 17% / 67% | **50% / 100%** | +| compound_head | 75% / 75% | 75% / **100%** | +| scope | 0% / 67% | 0% / **100%** | +| everything else | unchanged | unchanged | + +No class got worse. Most of the gain is the title, not the folding. + +### Two corrections to this document's own gold set + +**The `fold` class was never testing folding.** Adding a `regex+fold` +arm produced results byte-identical to plain regex, which is not what a +working fix looks like. The reason: all three questions turned on words +("TÜV", "Zählerstand", "Reisepässe") that appear *only in a page +title*, and the body-only engine cannot see titles however they are +spelled. Those are now filed as `frontmatter`, and four real +body-level fold questions replace them. The 0% to 100% jump this +document previously credited to diacritics belongs to indexing the +title. + +**Folding works; the bench protocol hides it.** Directly on the corpus, +`Nachprufung`, `Burgerburo` and `Uberweisung` go from **zero hits to +the right page**. The bench misses that because it ORs every content +word of a question into one pattern, so a common word like "stand" +drags in dozens of pages and the date sort scatters the real hit. The +agent does not query that way: the search log says its median query is +**one keyword**, which is exactly the case folding rescues. + +### Matching is not ranking + +The pattern across every cheap fix: they improve what is *found* +without improving what is *surfaced*. `frontmatter` questions still +sit at 0% recall@1 even once titles are searchable, because the right +page is now in the results and sorted by date along with everything +else. Recall@5 moves; recall@1 mostly does not. + +An early version of the frontmatter change also matched person names, +and that made things worse: one query went from four hits to twenty +and its answer from rank two to rank nine. A name says who a page +concerns, not what it says. `--person` already asks that question +properly. Persons stay out of the haystack. + ## What has not been measured The agent has not run against this. Everything above is the engine in diff --git a/stacklets/memory/fts_index.py b/stacklets/memory/fts_index.py index 37945ada..9dee775a 100644 --- a/stacklets/memory/fts_index.py +++ b/stacklets/memory/fts_index.py @@ -43,13 +43,14 @@ import re import sqlite3 import sys -import unicodedata from dataclasses import dataclass from pathlib import Path from typing import Iterable, List, Optional, Sequence sys.path.insert(0, str(Path(__file__).resolve().parent)) -from lib import _fm_list, _norm_tag, body_only, vault_local_head # noqa: E402 +from lib import ( # noqa: E402 + _fm_list, _norm_tag, body_only, strip_diacritics, vault_local_head, +) from stack.frontmatter import parse as parse_frontmatter # noqa: E402 @@ -96,12 +97,12 @@ def fold(text: str) -> str: The tokenizer folds the indexed side; this folds the Python side so that coverage (`Hit.matched`) agrees with what the index matched. - Decomposing first is what makes it work for both spellings of an - umlaut -- a precomposed "ü" and a "u" plus a combining diaeresis - are the same word to a family and must be the same token here. + The mark-stripping half is `lib.strip_diacritics`, shared with the + regex engine so both answer "Kase" and "Käse" the same way; the + case half is safe to add here because these are tokens rather than + a pattern. """ - decomposed = unicodedata.normalize("NFD", text.lower()) - return "".join(c for c in decomposed if not unicodedata.combining(c)) + return strip_diacritics(text).lower() def tokens(text: str) -> List[str]: diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index a31b3e0a..27431ad7 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -33,6 +33,7 @@ import re import subprocess import time +import unicodedata from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Callable, List, Optional @@ -1599,20 +1600,54 @@ def extract_summary_callout(text: str) -> str: return "\n".join(captured).strip() -def _excerpt(text: str, query: str, max_len: int = 200) -> str: +def strip_diacritics(text: str) -> str: + """Drop combining marks, so "Käse" and "Kase" are the same word. + + The family writes German; somebody searching from a phone keyboard + or in a hurry types the bare vowel. Byte-literal matching answers + that with nothing at all, which reads like the vault has no such + page rather than like the query was spelled differently. + + Case is deliberately left alone. The query is a regex, and + lower-casing one rewrites its operators: `\\W` becomes `\\w` and + stops meaning the opposite of what it said. Callers match + case-insensitively anyway. + + Decomposing first is what makes both spellings of an umlaut agree: + a precomposed "ü" and a "u" followed by a combining diaeresis are + the same letter to a reader and have to be the same here. + + What this is not is transliteration. "Kaese" stays a different word + from "Käse"; folding those together needs a German-specific table, + not a Unicode normalisation form. + """ + decomposed = unicodedata.normalize("NFD", text) + return "".join(c for c in decomposed if not unicodedata.combining(c)) + + +def _excerpt(text: str, query: str, max_len: int = 200, + fold_diacritics: bool = True) -> str: """First non-empty body line that mentions `query` (case-insensitive). Body starts after the *closing* `---` of the frontmatter block -- otherwise hits on `title:` or `persons:` lines would surface as excerpts, which is noisy and misleading. + + Folds the same way the match did, so a page found through "Kase" + still shows the line saying "Käse". A hit whose excerpt came back + empty is a hit the reader cannot judge. """ - needle = query.lower() + def norm(value: str) -> str: + return strip_diacritics(value).lower() if fold_diacritics \ + else value.lower() + + needle = norm(query) body = body_only(text) for line in body.splitlines(): stripped = line.strip() if not stripped: continue - if needle in stripped.lower(): + if needle in norm(stripped): if len(stripped) > max_len: stripped = stripped[:max_len] + "…" return stripped @@ -1626,6 +1661,8 @@ def search_memory( tags: Optional[List[str]] = None, scopes: Optional[List[str]] = None, limit: int = 20, + fold_diacritics: bool = True, + search_frontmatter: bool = True, ) -> List[dict]: """Walk the vault, return result dicts sorted newest-first. @@ -1650,6 +1687,14 @@ def search_memory( `persons`, `tags`, `excerpt`. Sorted by frontmatter `date` descending; files without a date sort to the end. + `fold_diacritics` strips combining marks from both the query and + the text before matching, so "Kase" reaches "Käse" and the reverse. + `search_frontmatter` matches the title, tag and person *values* + alongside the body, so a page titled "Elternabend" is found by + searching for Elternabend. Both are on by default; passing False + restores the older behaviour, which is what the retrieval lab + measures against. + On a missing vault directory or invalid regex, returns `[]` rather than raising -- callers decide how to surface the failure. @@ -1658,7 +1703,9 @@ def search_memory( return [] try: - pattern = re.compile(query, re.IGNORECASE) + pattern = re.compile( + strip_diacritics(query) if fold_diacritics else query, + re.IGNORECASE) except re.error: return [] @@ -1694,14 +1741,37 @@ def search_memory( # Match against body only -- frontmatter field names (`date:`, # `tags:`, `persons:`, ...) would otherwise trivially match # generic keywords and drown real hits. - if not pattern.search(body_only(text)): - continue - + body = body_only(text) fm = _parse_frontmatter(text) doc_persons = _fm_list(fm, "persons") + doc_tags = _fm_list(fm, "tags") + + # The values, never the keys. Stripping frontmatter kept a + # query for "date" from hitting every page through its `date:` + # line, and threw away the title and tags in the process -- + # which are the most descriptive text a page has. Joining the + # values back in restores them without letting the field names + # back through. + # + # Persons stay out. A name says who a page concerns, not what + # it says, so matching it as prose makes any question + # mentioning Lisa match every page Lisa is on. Measured: it + # took one query from four hits to twenty and pushed the + # answer from rank two to rank nine. `--person` is the right + # way to ask that question and it already exists. + haystack = body + if search_frontmatter: + haystack = "\n".join(( + str(fm.get("title") or ""), + " ".join(doc_tags), + body, + )) + if not pattern.search( + strip_diacritics(haystack) if fold_diacritics else haystack): + continue + if persons and not any(p.lower() in want_persons for p in doc_persons): continue - doc_tags = _fm_list(fm, "tags") if tags: doc_norm = {_norm_tag(t) for t in doc_tags} if not (doc_norm & want_tags): @@ -1714,7 +1784,8 @@ def search_memory( "date": fm.get("date") or "", "persons": doc_persons, "tags": doc_tags, - "excerpt": _excerpt(text, query), + "excerpt": _excerpt(text, query, + fold_diacritics=fold_diacritics), # The `> [!summary]` callout, stripped of blockquote # prefixes. Drives the synthesis step: feeding summaries # to the LLM is cheaper than feeding bodies and usually diff --git a/tests/stacklets/test_memory_search.py b/tests/stacklets/test_memory_search.py index f75d74e8..d2ff579e 100644 --- a/tests/stacklets/test_memory_search.py +++ b/tests/stacklets/test_memory_search.py @@ -193,6 +193,166 @@ def test_title_word_in_body_still_matches(self, vault): assert len(results) == 1 +@pytest.fixture +def titled_vault(tmp_path): + """A page whose subject is named only in its title. + + The shared fixture repeats every title as a `# Heading`, so it + cannot tell "found via the title" from "found via the body". Real + archivist pages do the same most of the time, which is why this + gap went unnoticed: it only bites on pages where the title is the + only place the subject is named. + """ + v = tmp_path / "vault" + _write(v / "family/health/termin.md", """ + --- + title: Zahnarzttermin Lisa + date: 2026-09-04 + persons: + - Lisa + tags: + - Topic:Health + --- + + Dienstag um halb vier, Praxis am Marktplatz. Vorher noch anrufen. + """) + return v + + +class TestFrontmatterValues: + """A page titled "Elternabend" is found by searching for Elternabend. + + Frontmatter is stripped before matching so that field *names* do + not match every file in the vault: a query for "date" would + otherwise hit every page via its `date:` line. That rule threw the + *values* out with the keys, and the values are the most + descriptive text a page has. A title is what the archivist chose + to call the page; a tag is what it decided the page is about. + Neither is noise. + + Measured in the retrieval lab: of the questions whose answer sits + in a page title, the body-only engine found none of them. + """ + + def test_a_word_only_in_the_title_finds_the_page(self, titled_vault): + results = search_memory("Zahnarzttermin", titled_vault) + assert len(results) == 1 + assert results[0]["rel"].endswith("termin.md") + + def test_a_tag_value_finds_the_page(self, vault): + """The shared fixture already has a tag that no body repeats.""" + results = search_memory("Cooking", vault) + assert len(results) == 1 + assert results[0]["rel"].endswith("quick-bread.md") + + def test_field_names_still_do_not_match(self, vault): + """The rule this replaces is still enforced. + + Promoting the values must not promote the keys with them, or + every page comes back for "date" exactly as before. + """ + assert search_memory("date", vault) == [] + assert search_memory("tags", vault) == [] + assert search_memory("title", vault) == [] + assert search_memory("persons", vault) == [] + + def test_a_title_only_hit_carries_no_invented_excerpt(self, titled_vault): + """The excerpt quotes the body, so a title hit has none to show. + + Better an empty excerpt than a line lifted from somewhere the + query never matched. + """ + results = search_memory("Zahnarzttermin", titled_vault) + assert results[0]["excerpt"] == "" + + def test_the_body_only_behaviour_is_still_reachable(self, titled_vault): + assert search_memory( + "Zahnarzttermin", titled_vault, search_frontmatter=False) == [] + + +# ─── Diacritics ────────────────────────────────────────────────────────── + +@pytest.fixture +def umlaut_vault(tmp_path): + """A vault written the way a German family writes: with umlauts.""" + v = tmp_path / "vault" + _write(v / "family/groceries/einkauf.md", """ + --- + title: Einkaufsliste + date: 2026-09-10 + persons: + - Marge + --- + + # Einkaufsliste + + Käse und Öl nachkaufen, die Tür klemmt auch wieder. + """) + return v + + +class TestDiacritics: + """An umlaut typed one way still finds it written the other. + + The family writes "Käse"; somebody searching from a phone, an + English keyboard, or in a hurry types "Kase". Byte-literal matching + returns *nothing at all* for that, which is the worst kind of + search failure: not a bad result, an empty one that reads like the + vault has no such page. + + Folding runs on both sides, so it does not matter which side + carries the umlaut. What it deliberately does not do is fold "ue" + into "ü" -- that is transliteration, not a diacritic, and it needs + a German-specific rule rather than a Unicode one. + """ + + def test_a_query_without_the_umlaut_finds_the_page_with_it( + self, umlaut_vault): + results = search_memory("Kase", umlaut_vault) + assert len(results) == 1 + assert results[0]["rel"].endswith("einkauf.md") + + def test_a_query_with_the_umlaut_finds_it_too(self, umlaut_vault): + assert len(search_memory("Käse", umlaut_vault)) == 1 + + def test_it_works_for_every_umlaut_not_just_a(self, umlaut_vault): + assert len(search_memory("Ol", umlaut_vault)) == 1 + assert len(search_memory("Tur", umlaut_vault)) == 1 + + def test_the_excerpt_still_shows_the_line_that_matched( + self, umlaut_vault): + """A hit with no excerpt is a hit the reader cannot judge.""" + results = search_memory("Kase", umlaut_vault) + assert "Käse" in results[0]["excerpt"] + + def test_a_spelled_out_umlaut_is_not_folded(self, umlaut_vault): + """"Kaese" is a different word to Unicode, and stays one. + + Worth pinning rather than leaving implicit: this is the gap + measured in the retrieval lab, and closing it would take a + German transliteration table, not a normalisation form. + """ + assert search_memory("Kaese", umlaut_vault) == [] + + def test_regex_syntax_survives_folding(self, umlaut_vault): + """The query is a regex, so folding must not rewrite its operators. + + Case-folding the pattern would turn `\\W` into `\\w` and invert + what it means. Here `\\W` has to keep matching the space after + "Käse"; if it had become `\\w` this finds nothing. + """ + assert len(search_memory(r"Kase\Wund", umlaut_vault)) == 1 + assert len(search_memory(r"K.se", umlaut_vault)) == 1 + + def test_the_old_byte_literal_behaviour_is_still_reachable( + self, umlaut_vault): + """So a before-and-after stays measurable, per the handover.""" + assert search_memory( + "Kase", umlaut_vault, fold_diacritics=False) == [] + assert len(search_memory( + "Käse", umlaut_vault, fold_diacritics=False)) == 1 + + # ─── Filters ───────────────────────────────────────────────────────────── class TestFilters: diff --git a/tools/retrieval-lab/evaluate.py b/tools/retrieval-lab/evaluate.py index 8a2b4f99..61639f90 100644 --- a/tools/retrieval-lab/evaluate.py +++ b/tools/retrieval-lab/evaluate.py @@ -110,17 +110,24 @@ class Result: Backend = Callable[[Sequence[str]], Result] -def regex_backend(vault: Path) -> Backend: - """Today's engine: OR the keywords into a regex, sort by date. +def regex_backend(vault: Path, fold: bool = False, + frontmatter: bool = False) -> Backend: + """The regex walk: OR the keywords into a pattern, sort by date. This is `search_memory` unchanged, driven exactly as `stack memory search --nl` drives it, so the baseline is the shipping behaviour and not a reconstruction of it. + + `fold` is the one cheap change that needs no index: strip combining + marks from both sides before matching. Kept as a separate arm so + the gain from five lines can be read apart from the gain that costs + an index, a second table and a fusion step. """ def run(keywords: Sequence[str]) -> Result: started = time.perf_counter() hits = search_memory(keywords_to_regex(list(keywords)), vault, - limit=LIMIT) + limit=LIMIT, fold_diacritics=fold, + search_frontmatter=frontmatter) elapsed = time.perf_counter() - started return Result([h["rel"] for h in hits], [], elapsed) return run @@ -372,6 +379,7 @@ def main() -> None: backends: dict[str, Backend] = { "regex": regex_backend(vault), + "regex+cheap": regex_backend(vault, fold=True, frontmatter=True), "fts5": fts5_backend(db), "fts5+tri": fts5_backend(db, substrings=True), } diff --git a/tools/retrieval-lab/goldset.yaml b/tools/retrieval-lab/goldset.yaml index ea704591..f889928a 100644 --- a/tools/retrieval-lab/goldset.yaml +++ b/tools/retrieval-lab/goldset.yaml @@ -149,22 +149,51 @@ queries: class: compound_tail gold: [milk-brand] - # ── fold: the umlaut is dropped ────────────────────────────────────── + # ── frontmatter: the word is only in the title ─────────────────────── + # + # These were filed as `fold` until the regex+fold arm came back + # byte-identical to plain regex and the reason turned out to be that + # none of the three words appears in a page body at all. "TÜV", + # "Zählerstand" and "Reisepässe" are titles. The regex engine strips + # frontmatter before matching, so it cannot see them however they are + # spelled, and the win here belongs to indexing the title rather than + # to folding anything. - q: Warum ist das Auto durch den TUV gefallen lang: de - class: fold + class: frontmatter gold: [car-inspection] - q: Wie hoch war der Zahlerstand im Januar lang: de - class: fold + class: frontmatter gold: [electricity-meter] - q: Wann laufen die Reisepasse ab + lang: de + class: frontmatter + gold: [passport-expiry] + + # ── fold: the umlaut is dropped, and the word is in the body ───────── + - q: Was stand in der Nachprufung + lang: de + class: fold + gold: [car-inspection] + + - q: Wann kam die Uberweisung + lang: de + class: fold + gold: [rental-deposit] + + - q: Wie lange dauert ein Termin im Burgerburo lang: de class: fold gold: [passport-expiry] + - q: Wann kommt die Abrechnung im Marz + lang: de + class: fold + gold: [electricity-meter] + # ── translit: the umlaut is spelled out ────────────────────────────── - q: Was war mit der Spuelmaschine lang: de From 3856b1038d622cb4fe63d157385fad19680c757f Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 17:56:07 +0200 Subject: [PATCH 08/11] feat(memory): say which search words appear nowhere in the vault An empty search means one of two things and a caller has to act differently on each: the words were wrong, or the fact is not written down. "no results" cannot tell them apart, so a bad guess either gives up on something that is there or keeps rephrasing at something that is not. $ stack memory search "repair|expense" no results. These words appear nowhere in the vault: expense Measured in the agent rig: a quarter of searches came back empty, and those skew English against German pages. Telling the agent to search in the family's language did not move it, because when it writes its first query nothing has told it what that language is. This delivers the same fact where it can be used. Costs a second walk only on a search that already returned nothing. The rig gains --skills and --explain-misses so the pairing can be measured; that run has not been completed yet. --- docs/design/brain/retrieval-engine-poc.md | 59 ++++++++ .../workspace/skills/family-memory/SKILL.md | 9 +- stacklets/memory/cli/search.py | 8 + stacklets/memory/lib.py | 40 +++++ tests/stacklets/test_memory_search.py | 49 +++++- tools/agent-lab/rig/lab-api.py | 18 ++- tools/agent-lab/rig/rig.py | 8 +- tools/retrieval-lab/skill_ab.py | 143 ++++++++++++++++++ 8 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 tools/retrieval-lab/skill_ab.py diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index 878f864c..fd5fb340 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -418,6 +418,65 @@ and its answer from rank two to rank nine. A name says who a page concerns, not what it says. `--person` already asks that question properly. Persons stay out of the haystack. +## The query side + +The search log from the repeated run is the most useful artefact this +work produced, because it says what the agent actually does rather than +what the skill asks of it. 161 searches: + +- **The median query is 2-3 terms**, written as a regex alternation + (`subscription|license|domain`). An earlier claim in this document + that the agent "searches one keyword at a time" was an artefact of + counting whitespace-separated words in a query that has no spaces. +- **`--nl` was used 0 times in 161 searches.** The keyword-rewrite + path, a full LLM round trip and a documented feature, is one the + agent has never reached for. It writes its own regex instead. +- **27% of searches came back empty**, and the empty ones skew English: + German-ish queries dead-end around 16% of the time, non-German ones + around 35%. The agent is asked a German question, answers in German, + and searches a German vault in English. + +### Telling the agent to use the right language did not work + +Measured, three repeats per arm, dead-end rate straight from the search +log: + +| | searches | dead ends | rate | calls | +|---|---|---|---|---| +| before | 83 | 27 | 33% | 115 | +| after | 74 | 23 | 31% | 101 | + +German-ish queries moved 54% to 57%, and the same English dead ends +recurred. The instruction was unfollowable rather than ignored: it +asked the agent to search "in the language the family wrote the page +in" at the moment it writes its *first* query, when nothing has yet +told it what that language is. + +### Giving it the fact instead + +An empty result means one of two things and the caller must act +differently on each: the words were wrong, or the fact is not written +down. `no results` cannot tell them apart, so a bad guess either gives +up on a fact that is there or keeps rephrasing at one that is not. + +`unmatched_terms` names the query words that appear nowhere in the +vault, and the CLI prints them on the empty path only: + +``` +$ memory search "repair|expense" +no results. These words appear nowhere in the vault: expense +``` + +`repair` is absent from that list because it matches the `repairs` +tag, which is only visible thanks to the frontmatter change above. The +two compound. + +**Status: unverified at agent level.** The rig run was killed partway +through. The helper is unit-tested and checked by hand on both cases, +and it costs a second vault walk only on a search that already +returned nothing, so the successful path is unchanged. Whether it +actually cuts the dead-end rate is the run that still has to happen. + ## What has not been measured The agent has not run against this. Everything above is the engine in diff --git a/stacklets/agent/workspace/skills/family-memory/SKILL.md b/stacklets/agent/workspace/skills/family-memory/SKILL.md index 7da2d6e3..838f362a 100644 --- a/stacklets/agent/workspace/skills/family-memory/SKILL.md +++ b/stacklets/agent/workspace/skills/family-memory/SKILL.md @@ -10,10 +10,15 @@ Vault = all family knowledge. Look before answering. ``` LOOKUP: brief names the page -> read_file(page) # no search - else -> memory_search(2-4 literal keywords) + else -> memory_search(2-4 keywords joined by |) + never escape the | + compound word -> search the stem (Camping), not the whole (Campingfahrt) scope family/ first; miss -> widen independent lookups -> ONE call, queries=[..] # max 3 - miss -> retry once, new keywords; then say tried + ask + "these words appear nowhere" -> MY words are wrong, not the vault. + retry those words in the language the found pages are written in + empty with no such line -> the words are fine, the fact is absent. + say what I tried; do NOT keep rephrasing profile -> memory_person(name) "lately|since when|who did" -> memory_history # search ranks NOW, not change full source document -> paperless_id frontmatter + `stack docs show --content` diff --git a/stacklets/memory/cli/search.py b/stacklets/memory/cli/search.py index 62ef1a80..511b5b3e 100644 --- a/stacklets/memory/cli/search.py +++ b/stacklets/memory/cli/search.py @@ -105,6 +105,7 @@ keywords_to_regex, refresh_vault_if_stale, search_memory, + unmatched_terms, vault_path_for, ) @@ -320,6 +321,13 @@ def run(args, stacklet, config) -> dict | None: print(f"Searched for: {', '.join(keywords)}\n") if not results: + # Empty means either "not written down" or "wrong words", and + # the caller acts differently on each. Naming the words that + # matched nothing anywhere is the difference, and it costs a + # second walk only on a search that already failed. + if missing := unmatched_terms(query, vault, scopes=ns.scope or None): + print("no results. These words appear nowhere in the vault: " + + ", ".join(missing)) sys.exit(1) if ns.count: diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index 27431ad7..45900baa 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -1974,6 +1974,46 @@ async def rewrite_query( return keywords +def unmatched_terms( + query: str, + vault: Path, + scopes: Optional[List[str]] = None, + fold_diacritics: bool = True, + search_frontmatter: bool = True, +) -> List[str]: + """Which alternatives of an OR-query appear nowhere in the vault. + + An empty result means one of two things, and the caller has to act + differently on each: the fact is not written down, or the words + were wrong. "No results" cannot tell them apart, so a caller that + guessed badly either gives up on a fact that is there, or keeps + guessing at one that is not. + + Measured in the agent rig: a quarter of searches came back empty, + and most were English words against German pages. Telling the agent + to search in the family's language did not move it, because when it + writes the first query nothing has told it what that language is. + Naming the words that matched nothing delivers the same fact at the + moment it can be used. + + Splits on the unescaped `|` because that is the query the agent + actually writes -- it never passes `--nl`, it builds the alternation + itself. A query with no `|` is one term and is checked as one. + + Only worth calling when a search returned nothing: it re-walks the + vault once per term, which is the right trade on a path that has + already failed and is wasted on one that has not. + """ + terms = [t for t in re.split(r"(? str: """Render keywords as the alternation regex `search_memory` reads. diff --git a/tests/stacklets/test_memory_search.py b/tests/stacklets/test_memory_search.py index d2ff579e..443772fa 100644 --- a/tests/stacklets/test_memory_search.py +++ b/tests/stacklets/test_memory_search.py @@ -31,7 +31,7 @@ # archivist bot consumes directly. sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "stacklets" / "memory")) -from lib import search_memory # noqa: E402 +from lib import search_memory, unmatched_terms # noqa: E402 # Probe that matches every fixture doc via body content (one keyword @@ -270,6 +270,53 @@ def test_the_body_only_behaviour_is_still_reachable(self, titled_vault): "Zahnarzttermin", titled_vault, search_frontmatter=False) == [] +class TestUnmatchedTerms: + """"No results" that says *why* there were none. + + A search that comes back empty means one of two very different + things: the fact is not in the vault, or the words were wrong. The + caller has to act differently in each case -- say "I did not find + it" versus try different words -- and "no results" alone cannot + tell them apart. + + Measured in the rig: a quarter of the agent's searches dead-ended, + and most of those were English words against German pages. Telling + it to search in the family's language did not help, because at the + moment it writes the first query it has no way to know what that + language is. This is the same fact, delivered where it is usable. + """ + + def test_a_word_that_appears_nowhere_is_named(self, vault): + assert unmatched_terms("Brummen|spaceship", vault) == ["spaceship"] + + def test_words_that_do_appear_are_not_named(self, vault): + assert unmatched_terms("Brummen|Hoover", vault) == [] + + def test_every_missing_word_is_named(self, vault): + assert unmatched_terms("spaceship|submarine", vault) == [ + "spaceship", "submarine" + ] + + def test_it_reads_the_alternation_the_agent_actually_writes(self, vault): + """The agent passes `a|b|c`, not a keyword list. + + It never uses `--nl`; it writes the regex itself. Recovering + the terms means splitting that query, which is the only shape + this needs to handle. + """ + assert unmatched_terms("Brummen|spaceship|Hoover", vault) == [ + "spaceship" + ] + + def test_it_agrees_with_the_search_it_explains(self, vault): + """Folding is on, so "Kase" is not reported as a missing word + when the page says "Käse" and the search would have found it.""" + assert unmatched_terms("Tierarzt", vault) == [] + + def test_a_query_with_no_alternation_still_works(self, vault): + assert unmatched_terms("spaceship", vault) == ["spaceship"] + + # ─── Diacritics ────────────────────────────────────────────────────────── @pytest.fixture diff --git a/tools/agent-lab/rig/lab-api.py b/tools/agent-lab/rig/lab-api.py index 410934b7..b483731a 100644 --- a/tools/agent-lab/rig/lab-api.py +++ b/tools/agent-lab/rig/lab-api.py @@ -45,6 +45,7 @@ # stacklet rather than reimplemented, for the same reason the list-edit # transform is: an A/B against a copy of the engine measures the copy. import fts_index # noqa: E402 +from lib import unmatched_terms # noqa: E402 def _body_only(text: str) -> str: @@ -120,11 +121,23 @@ def _search_regex(ns, vault: Path, pattern) -> tuple[str, int]: m = re.search(r"^date:\s*(\S+)", text, re.MULTILINE) hits.append((m.group(1) if m else "", _block(vault, str(rel), pattern))) if not hits: - return "no results\n", 1 + return _no_results(ns, vault), 1 hits.sort(key=lambda h: h[0], reverse=True) return "\n".join(h[1] for h in hits[: ns.limit]) + "\n", 0 +def _no_results(ns, vault: Path) -> str: + """Empty, and why. Runs the production helper, never a copy of it.""" + if not ARGS.explain_misses: + return "no results\n" + missing = unmatched_terms(ns.query, vault, + scopes=[ns.scope] if ns.scope else None) + if not missing: + return "no results\n" + return ("no results. These words appear nowhere in the vault: " + + ", ".join(missing) + "\n") + + def _search_ranked(ns, vault: Path, keywords, pattern, gated: bool) -> tuple[str, int]: """The ranked engine: best match first, optionally behind a gate.""" @@ -416,6 +429,9 @@ def main(): parser.add_argument("--llm", default="http://localhost:8888/v1") parser.add_argument("--key", default="none") parser.add_argument("--model", default=None) + parser.add_argument("--explain-misses", action="store_true", + help="on an empty result, name the query words that " + "appear nowhere in the vault") parser.add_argument("--search-log", default=str( Path(__file__).parent / "state" / "search-log.jsonl"), help="append every search and its hits here, for run analysis") diff --git a/tools/agent-lab/rig/rig.py b/tools/agent-lab/rig/rig.py index b3152525..a8ec3f0b 100644 --- a/tools/agent-lab/rig/rig.py +++ b/tools/agent-lab/rig/rig.py @@ -56,7 +56,11 @@ def cmd_reset(_args): for name in ("SOUL.md", "AGENTS.md"): text = (seed / name).read_text().replace("__AGENT_NAME__", "Stacky") (ws / name).write_text(text) - shutil.copytree(seed / "skills", ws / "skills") + # `--skills` seeds from somewhere else, which is how two versions of + # a skill get compared: the prompt is the thing under test, so it + # has to be swappable without editing the tree between runs. + shutil.copytree(Path(_args.skills) if _args.skills else seed / "skills", + ws / "skills") # Same minimal USER.md the production entrypoint seeds. (ws / "USER.md").write_text( "# User Profile\n\n" @@ -169,6 +173,8 @@ def main() -> int: p_reset = sub.add_parser("reset") p_reset.add_argument("--corpus", default=None, metavar="DIR", help="seed the vault from DIR instead of demo-vault") + p_reset.add_argument("--skills", default=None, metavar="DIR", + help="seed skills from DIR instead of the repo's") p_turn = sub.add_parser("turn") p_turn.add_argument("message") p_turn.add_argument("--session", default="rig:main") diff --git a/tools/retrieval-lab/skill_ab.py b/tools/retrieval-lab/skill_ab.py new file mode 100644 index 00000000..be9dc844 --- /dev/null +++ b/tools/retrieval-lab/skill_ab.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Compare two versions of the family-memory skill, on one search engine. + + uv run --extra test python tools/retrieval-lab/skill_ab.py --repeat 3 + +Everything measured so far changed what answers a query. This changes +the query. The search log from the earlier runs says the agent asks in +the wrong language: it converses in German, searches in English, and +those searches come back empty about twice as often. That is a prompt +problem, and prompts are cheaper to change than engines. + +**The metric, fixed before running.** Dead-end rate: the share of +searches that returned nothing, read straight out of the lab-api search +log rather than out of a reply somebody had to interpret. The skill +change targets exactly that, so it is the number that can falsify it. +Iterations and answers are reported alongside, because a skill that +cuts dead ends by sending the agent round more loops has not helped. + +The baseline skill is materialised from git rather than kept as a +second copy in the tree, so it cannot drift out of step with what +shipped. + +Prerequisites: one lab-api on 42011 serving the engine under test, and +proxy.py. Both arms reset the rig, so the vault is reseeded per arm and +nothing carries over. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +RIG = REPO / "tools" / "agent-lab" / "rig" +SKILLS = "stacklets/agent/workspace/skills" + +sys.path.insert(0, str(HERE)) +from agent_ab import SCENARIOS, run_turn # noqa: E402 + + +def materialise_baseline(ref: str, out: Path) -> Path: + """Check the pre-change skills tree out of git, into a scratch dir.""" + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + listing = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", f"{ref}:{SKILLS}"], + cwd=REPO, capture_output=True, text=True, check=True).stdout.split() + for rel in listing: + blob = subprocess.run( + ["git", "show", f"{ref}:{SKILLS}/{rel}"], + cwd=REPO, capture_output=True, text=True, check=True).stdout + target = out / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(blob, encoding="utf-8") + return out + + +def reset(corpus: Path, skills: Path) -> None: + subprocess.run( + [sys.executable, str(RIG / "rig.py"), "reset", + "--corpus", str(corpus), "--skills", str(skills)], + check=True, capture_output=True) + + +def dead_ends(log: Path) -> dict: + """What the agent asked for, and how often it got nothing back.""" + if not log.exists(): + return {"searches": 0, "empty": 0, "rate": 0.0} + records = [json.loads(line) for line in + log.read_text(encoding="utf-8").splitlines() if line.strip()] + empty = sum(1 for r in records if r["exit"] != 0) + return { + "searches": len(records), + "empty": empty, + "rate": empty / len(records) if records else 0.0, + "empty_queries": [r["query"] for r in records if r["exit"] != 0], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repeat", type=int, default=3) + parser.add_argument("--baseline-ref", default="HEAD", + help="git ref holding the pre-change skill") + parser.add_argument("--corpus", + default=str(HERE / "out" / "rig-vault")) + # One port per arm, because the miss explanation is a server flag + # and the skill line that reacts to it is meaningless without it. + # The two ship together, so they are measured together. + parser.add_argument("--port-before", type=int, default=42013) + parser.add_argument("--port-after", type=int, default=42011) + parser.add_argument("--log", + default=str(RIG / "state" / "search-regex.jsonl")) + ns = parser.parse_args() + + arms = { + "before": (materialise_baseline(ns.baseline_ref, + HERE / "out" / "skills-baseline"), + ns.port_before), + "after": (REPO / SKILLS, ns.port_after), + } + + out: list[dict] = [] + summary: dict[str, dict] = {} + log = Path(ns.log) + for arm, (skills, port) in arms.items(): + reset(Path(ns.corpus), skills) + if log.exists(): + log.unlink() + for run in range(ns.repeat): + for scenario in SCENARIOS: + session = f"skill{run}:{scenario['id']}:{arm}" + print(f"[{arm:>6} run {run}] {scenario['id']}", flush=True) + turn = run_turn(scenario["q"], arm, port, session) + out.append({**scenario, **turn, "run": run, "arm": arm}) + print(f" {turn['llm_calls']} calls, " + f"{turn['wall_s']}s", flush=True) + summary[arm] = dead_ends(log) + shutil.copy(log, HERE / "out" / f"search-{arm}.jsonl") + + (HERE / "out" / "skill-ab.json").write_text( + json.dumps({"turns": out, "searches": summary}, indent=2, + ensure_ascii=False), encoding="utf-8") + + print("\n" + "=" * 64) + print(f"{'arm':>7} {'searches':>9} {'dead ends':>10} {'rate':>6} " + f"{'calls':>7}") + for arm in arms: + s = summary[arm] + calls = sum(r["llm_calls"] or 0 for r in out if r["arm"] == arm) + print(f"{arm:>7} {s['searches']:>9} {s['empty']:>10} " + f"{s['rate']:>5.0%} {calls:>7}") + print(f"\ndetail: {HERE / 'out' / 'skill-ab.json'}") + + +if __name__ == "__main__": + main() From 26940346b32e3d0b59761764708dde38a814714f Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 17:59:42 +0200 Subject: [PATCH 09/11] docs(brain): replay the agent's real queries, and correct the attribution Every other number here rests on a gold set somebody wrote. Replaying the 104 queries the agent actually sent says that of the two shipped search fixes, only one moves anything: queries returning nothing drop from 31% to 24%, and every rescue comes from matching titles and tags. Diacritic folding contributes zero on real agent queries. It is correct and tested, but the agent does not type German without umlauts, it types English. Credited earlier by the class it was designed for rather than by what moved. --- docs/design/brain/retrieval-engine-poc.md | 35 ++++++++++ tools/retrieval-lab/replay.py | 85 +++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 tools/retrieval-lab/replay.py diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index fd5fb340..4eef24fb 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -477,6 +477,41 @@ and it costs a second vault walk only on a search that already returned nothing, so the successful path is unchanged. Whether it actually cuts the dead-end rate is the run that still has to happen. +## Replaying the agent's own queries + +Every number above rests on a gold set somebody wrote. This one does +not: it replays the 104 distinct queries the agent actually sent, +recovered from the lab-api search logs, against each version of the +engine. Reproduce with `tools/retrieval-lab/replay.py`. + +Zero results is the right metric for these two fixes. Ranking decides +which page comes first; these decide whether there is a page at all, +and an agent can iterate past bad ordering but not past nothing. + +| engine | queries returning nothing | +|---|---| +| old (body only, byte literal) | 32 / 104 (31%) | +| folding only | 32 / 104 (31%) | +| title and tags only | **25 / 104 (24%)** | +| both (shipped) | 25 / 104 (24%) | + +**Folding contributes nothing on real agent queries.** All seven +rescues come from matching titles and tags. That is the fourth +attribution in this document that did not survive being measured, and +the pattern is always the same: a fix was credited by the class it was +*designed* for rather than by what moved. + +Folding is not wrong, it is unexercised. The agent does not type German +without umlauts; it types English. `Kase` for `Käse` is a phone +keyboard and a human in a hurry, and no evidence here says a family +does that often. It stays because it is correct, cheap and tested, not +because it was shown to help. + +The rescues also explain themselves once read: `repairs|expenses`, +`flight|travel|ticket` land on pages whose *tags* are English while +their prose is German. Matching tags accidentally bridges the language +gap that the skill instruction could not. + ## What has not been measured The agent has not run against this. Everything above is the engine in diff --git a/tools/retrieval-lab/replay.py b/tools/retrieval-lab/replay.py new file mode 100644 index 00000000..427049d9 --- /dev/null +++ b/tools/retrieval-lab/replay.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Replay the agent's own queries against each version of the engine. + + uv run --extra test python tools/retrieval-lab/replay.py + +Every other measurement here uses a gold set somebody wrote. This one +uses the queries the agent actually sent, recovered from the lab-api +search logs, and asks the one question a gold set cannot answer: of the +searches that really happened, how many came back empty before the +change and how many after. + +Zero results is the right metric for these two fixes. Ranking decides +which page is first; these decide whether there is a page at all, and +an agent can iterate its way past bad ordering but not past nothing. + +Each fix is scored on its own, because attributing a gain to the wrong +change has already happened twice in this work. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +sys.path.insert(0, str(REPO / "lib")) +sys.path.insert(0, str(REPO / "stacklets" / "memory")) + +from lib import search_memory # noqa: E402 + +# (fold diacritics, search frontmatter values) +ARMS: dict[str, tuple[bool, bool]] = { + "old (body only, byte literal)": (False, False), + "folding only": (True, False), + "title/tags only": (False, True), + "both (shipped)": (True, True), +} + + +def collect(paths: list[Path]) -> list[str]: + """Distinct queries, in the order the agent first sent them.""" + queries: list[str] = [] + for path in paths: + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + queries.append(json.loads(line)["query"]) + return list(dict.fromkeys(queries)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vault", default=str( + REPO / "tools" / "agent-lab" / "rig" / "state" / "vault")) + parser.add_argument("--logs", nargs="*", default=None) + ns = parser.parse_args() + + if ns.logs: + paths = [Path(p) for p in ns.logs] + else: + state = REPO / "tools" / "agent-lab" / "rig" / "state" + paths = sorted(state.glob("search-*.jsonl")) + \ + sorted((HERE / "out").glob("search-*.jsonl")) + paths = [p for p in paths if p.exists()] + if not paths: + sys.exit("no search logs found -- run the agent rig first") + + vault = Path(ns.vault) + queries = collect(paths) + print(f"{len(queries)} distinct real agent queries, " + f"{len(list(vault.rglob('*.md')))} pages\n") + + for name, (fold, frontmatter) in ARMS.items(): + dead = sum(1 for q in queries + if not search_memory(q, vault, limit=5, + fold_diacritics=fold, + search_frontmatter=frontmatter)) + print(f"{name:<30} zero results: {dead:>3}/{len(queries)} " + f"({dead / len(queries):.0%})") + + +if __name__ == "__main__": + main() From 423ce0e19a778ebe9f62ca0848681e21eb367f39 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 21:37:09 +0200 Subject: [PATCH 10/11] docs(brain): mark the retrieval plan superseded and hand over round two The upgrade handover reads as build-ready and would send the next reader straight into building the index that was just measured as not worth building. It now says so at the top. Adds a round-two handover: the one change that improved anything, the four that did not, which of the 4371 lines belong on main, the open decisions (--nl unused in 161 searches, the oMLX prefill ceiling, the unfinished rig run), and the method notes behind four findings that were reported and then retracted. --- docs/design/brain/retrieval-engine-poc.md | 17 ++- .../handover/memory-retrieval-upgrade.md | 18 +++ docs/design/handover/retrieval-round-2.md | 122 ++++++++++++++++++ 3 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 docs/design/handover/retrieval-round-2.md diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index 4eef24fb..3f4dff8c 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -1,10 +1,21 @@ # Retrieval engine PoC: what the numbers say -**Status:** probe complete, nothing wired into the CLI yet. -**Answers:** `docs/design/handover/memory-retrieval-upgrade.md` -**Bench:** `tools/retrieval-lab/` (177-page fabricated vault, 47 questions) +**Status:** measured and stopped. The index was built and is *not* +adopted; one small change to the regex walk was. +**Answers:** `docs/design/handover/memory-retrieval-upgrade.md` (now superseded) +**Next round:** `docs/design/handover/retrieval-round-2.md` +**Bench:** `tools/retrieval-lab/` (177-page fabricated vault, 51 questions) **Date:** 2026-09-16 +**The outcome, so nobody has to read 500 lines for it.** The FTS5 index +roughly triples recall@1 on the bench and changes nothing a family +would notice; the agent closes the gap by searching again. The only +measured improvement is matching title and tag values, worth seven +real queries rescued from returning nothing (31% to 24%). Four other +findings in this document were reported and then retracted when +measured a second time; each retraction is kept in place rather than +edited away, because the pattern is the lesson. + The handover proposed SQLite FTS5 + BM25 as tier 1 and named a trigram companion as the fallback if German compounds hurt. This measured both against the shipping regex engine before writing any of it into diff --git a/docs/design/handover/memory-retrieval-upgrade.md b/docs/design/handover/memory-retrieval-upgrade.md index 07d57a1a..7ffca6ca 100644 --- a/docs/design/handover/memory-retrieval-upgrade.md +++ b/docs/design/handover/memory-retrieval-upgrade.md @@ -1,5 +1,23 @@ # Handover: family-memory retrieval upgrade +> **SUPERSEDED, 2026-09-16. Do not build tier 1 from this document.** +> +> It was built and measured. The FTS5 index roughly triples recall@1 on +> a bench (23% to 60%) and makes no difference to what a family gets: +> seven questions through the real agent, three repeats, two arms, no +> change in answers, iterations or wall time beyond run-to-run spread. +> The agent closes the gap by searching again and reading a page. +> +> The tier-2 reasoning below is also wrong about *why*. "BM25 Wins at +> Scale" puts a file-system agent ahead of BM25 at the corpus size a +> family vault actually is; BM25 wins on token cost (39x), not accuracy. +> +> What did help is much smaller and is already in `search_memory`: +> matching title and tag values. Read +> `docs/design/brain/retrieval-engine-poc.md` before acting on anything +> here, and `docs/design/handover/retrieval-round-2.md` for what is +> worth doing next. + **Status:** research done, not built. Ready for a dev-rig implementation session. **Audience:** an agent session on a development rig, not production. **Seed context (read first):** diff --git a/docs/design/handover/retrieval-round-2.md b/docs/design/handover/retrieval-round-2.md new file mode 100644 index 00000000..65b54137 --- /dev/null +++ b/docs/design/handover/retrieval-round-2.md @@ -0,0 +1,122 @@ +# Handover: where retrieval stands, and what round two should do + +**Status:** round one measured and stopped. Nothing is blocked. +**Branch:** `feature/improve-brain-retrieval2`, PR #99 (pushed, unmerged) +**Numbers and method:** `docs/design/brain/retrieval-engine-poc.md` +**Supersedes the plan in:** `docs/design/handover/memory-retrieval-upgrade.md` +**Date:** 2026-09-16 + +## What round one was, in one paragraph + +The earlier handover asked for a SQLite FTS5 + BM25 index to replace the +regex walk behind `stack memory search`. It was built, benchmarked +against a fabricated corpus, then run through the real agent on a +420-page vault. The index wins the benchmark and loses the only test +that matters. One small change did earn its place. The rest of this +document is what to carry forward and what to leave alone. + +## The one thing that improved + +**Search matches title and tag values, not only the body.** Frontmatter +was stripped before matching so a query for "date" would not hit every +page through its `date:` line, and that threw the title out with the +field names. A page titled "Zahnarzttermin Lisa" was invisible unless +its body repeated the word. + +Measured by replaying the 104 queries the agent really sent +(`tools/retrieval-lab/replay.py`): queries returning nothing fall from +**31% to 24%**, seven real searches rescued. That is the whole of the +measured gain from this work. + +## What did not work, so nobody repeats it + +| Change | Result | +|---|---| +| FTS5 + trigram + RRF index | 23% to 60% recall@1 on the bench, **no agent-level difference** | +| Confidence gate on keyword coverage | answers-to-absent-facts 92% to 17% on the bench, **bought nothing**; the model already declines | +| Skill instruction "search in the family's language" | dead-end rate 33% to 31%, **noise** | +| Diacritic folding ("Kase" finds "Käse") | **zero** on real agent queries; the agent types English, not umlaut-less German | + +The pattern behind all four: the reasoning layer was already absorbing +the weakness being fixed. An agent that can search twice and read a page +does not need better ranking, and does not invent facts just because +search handed it a near-miss. + +## What should land on main + +Round one produced 4371 lines. Most of it should not be carried. + +**Land (about 700 lines, all measured or explanatory):** + +- title and tag matching in `search_memory`, plus its tests +- `docs/design/brain/retrieval-engine-poc.md`, whose job is to stop the + index being built a second time +- the rig's search log (`lab-api.py --search-log`) and + `tools/retrieval-lab/replay.py`. Every real insight in round one came + from the search log, and replay is what finally answered "did anything + improve" honestly + +**Leave on PR #99:** + +- `stacklets/memory/fts_index.py` and its tests, 1013 lines with no + callers. Dead code in main is code somebody eventually wires up + because it is there. Check the branch out if round two needs it. +- the rest of `tools/retrieval-lab/`, about 2330 lines. Genuinely useful + if there is another retrieval question, but its gold set needed + correcting twice in one session, so it is not yet an artefact to + depend on. +- the `SKILL.md` hunk: a prompt change with no evidence behind it. + +**Marginal, decide rather than drift:** diacritic folding is 8 lines and +costs about 15% of search latency (10.5 to 12.0 ms p50) for zero +measured benefit. But it was measured against the *agent*, and +`stack memory search` also serves family members typing German on phone +keyboards through chat. That is the case it fixes and it is unmeasured, +not disproven. + +## Open, and worth deciding before round two + +- **`--nl` was used 0 times in 161 agent searches.** A documented flag, a + full LLM round trip, and a chunk of `search.py` that the agent has + never reached for. It writes its own regex alternation instead. Either + the agent contract should point at it or it should go. +- **The oMLX prefill ceiling.** Five of six runs of the multi-search + question died on `predicted peak would exceed prefill safety cap + 46.8GB, kv_len=8192`. That is a harder limit on complex questions than + retrieval quality is, and it has nothing to do with search. +- **The unverified half of round one.** `unmatched_terms` makes an empty + search say which words appear nowhere, so "wrong words" and "not + written down" stop looking identical. Unit-tested and hand-checked; + the rig run pairing it with its skill lines was killed partway. About + 20 minutes to finish. + +## What round two should actually be + +**Embeddings, and nothing else in retrieval.** Paraphrase recall is 29% +and every remaining failure is semantic: no lexical engine reaches "läuft +ab" from a page that says "Kündigung muss drei Monate vorher raus". That +is the only lever left on answer quality, and the bench already has the +number it has to beat. + +Before building it, settle whether it clears the bar this round did not: +a measurable improvement **through the agent**, not on a benchmark. + +## Method notes, learned the hard way + +Round one reported four findings that did not survive being measured +again. Each correction came from running one more arm, never from +thinking harder. If round two runs agent experiments: + +- **One run per cell is a dice roll.** A claimed correctness win here was + the old engine's single unlucky draw; it was right 2 of 3 times on + repeat. Always repeat, always alternate which arm goes first. +- **Log what each arm was actually asked.** The two arms in one A/B + shared only 19% of their keyword vocabulary, because the agent writes + its own query. Without that log, a difference between arms cannot be + told apart from the agent having asked one of them better questions. +- **Attribute by what moved, not by what a fix was designed for.** Three + of the four bad claims came from crediting a change with a gain that + belonged to a different change. +- **Replay real queries, not only a gold set.** The gold set said folding + was valuable. The agent's own queries said it does nothing. A gold set + written by the same person who wrote the fix will agree with the fix. From d245ae5bacfe96711170772f6acd56b3bb61eef6 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 16 Sep 2026 21:54:55 +0200 Subject: [PATCH 11/11] chore: remove machine-identifying details from the public repo Four spots in a public repo named a specific machine rather than the fabricated household everything else uses. - a WiFi network in a knowledge-doc example carried a real network name, while every other entry around it is invented (Duff Insurance, Dr Hibbert, Firma Huber). Now invented too. - a probe artifact recorded the endpoint's full list of 28 served models, which fingerprints one machine. The count is kept, since that is all the probe used it for. - two retrieval notes quoted a prefill error verbatim, including the memory cap in gigabytes. They now describe the limit without the numbers, which is the part that mattered anyway. --- docs/design/brain/knowledge-architecture.md | 2 +- docs/design/brain/retrieval-engine-poc.md | 7 ++-- docs/design/handover/retrieval-round-2.md | 7 ++-- .../results/20260915T120024Z-probe.json | 32 ++----------------- 4 files changed, 12 insertions(+), 36 deletions(-) diff --git a/docs/design/brain/knowledge-architecture.md b/docs/design/brain/knowledge-architecture.md index 66e100f8..c4bc8df4 100644 --- a/docs/design/brain/knowledge-architecture.md +++ b/docs/design/brain/knowledge-architecture.md @@ -329,7 +329,7 @@ personal (12 entries): school, dance class, preferences ## Home - [ctx] Bathroom renovation ongoing, Firma Bauer [g7h8i9j:household/home.md] -- [fact] WiFi: network "merles", password in contacts.md [d4e5f6g:household/home.md] +- [fact] WiFi: network "evergreen", password in contacts.md [d4e5f6g:household/home.md] ``` **Full documents** (variable size, retrieved only when Kit needs details): diff --git a/docs/design/brain/retrieval-engine-poc.md b/docs/design/brain/retrieval-engine-poc.md index 3f4dff8c..b94643ec 100644 --- a/docs/design/brain/retrieval-engine-poc.md +++ b/docs/design/brain/retrieval-engine-poc.md @@ -335,9 +335,10 @@ convert into fewer iterations or better answers. ### An infrastructure limit, not a retrieval one `repair-total` hit an oMLX prefill guard rejection in **5 of 6 runs**, -on both arms: `predicted peak would exceed prefill safety cap 46.8GB -... kv_len=8192`. Questions that need several searches grow the -context past what the endpoint will prefill. The earlier claim that +on both arms: the guard refused the prompt because its predicted peak +would exceed the endpoint's prefill safety cap, at a context of 8192 +tokens. Questions that need several searches grow the context past what +the endpoint will prefill. The earlier claim that regex "saw three of four repair bills" was reading whichever arm got further before erroring. That question measures the endpoint, not the engine, until the guard is raised or the context trimmed. diff --git a/docs/design/handover/retrieval-round-2.md b/docs/design/handover/retrieval-round-2.md index 65b54137..cf3e8f51 100644 --- a/docs/design/handover/retrieval-round-2.md +++ b/docs/design/handover/retrieval-round-2.md @@ -81,9 +81,10 @@ not disproven. never reached for. It writes its own regex alternation instead. Either the agent contract should point at it or it should go. - **The oMLX prefill ceiling.** Five of six runs of the multi-search - question died on `predicted peak would exceed prefill safety cap - 46.8GB, kv_len=8192`. That is a harder limit on complex questions than - retrieval quality is, and it has nothing to do with search. + question died on the prefill memory guard, which refused the prompt at + a context of 8192 tokens because its predicted peak would exceed the + endpoint's safety cap. That is a harder limit on complex questions + than retrieval quality is, and it has nothing to do with search. - **The unverified half of round one.** `unmatched_terms` makes an empty search say which words appear nowhere, so "wrong words" and "not written down" stop looking identical. Unit-tested and hand-checked; diff --git a/tools/agent-lab/results/20260915T120024Z-probe.json b/tools/agent-lab/results/20260915T120024Z-probe.json index e8457060..0d9f8322 100644 --- a/tools/agent-lab/results/20260915T120024Z-probe.json +++ b/tools/agent-lab/results/20260915T120024Z-probe.json @@ -14,34 +14,7 @@ }, "payload": { "served_models": [ - "Gemma-4-31B-JANG_4M-CRACK", - "LFM2-24B-A2B-MLX-4bit", - "LFM2.5-Embedding-350M-4bit", - "Meta-Llama-3.1-8B-Instruct-4bit", - "Qwen3-30B-A3B-Instruct-2507-MLX-4bit", - "Qwen3-30B-A3B-Instruct-2507-MLX-fp16", - "Qwen3-30B-A3B-MLX-4bit", - "Qwen3-Embedding-8B-4bit-DWQ", - "Qwen3-VL-8B-Instruct-MLX-4bit", - "Qwen3.5-27B-4bit", - "Qwen3.5-27B-Claude-4.6-Opus-Reasoning-Distilled-qx64-hi-mlx", - "Qwen3.5-27B-heretic-8bit", - "Qwen3.5-2B-bf16", - "Qwen3.5-35B-A3B-4bit", - "Qwen3.5-35B-A3B-4bit-fp16", - "Qwen3.5-9B-MLX-4bit", - "Qwen3.5-9B-MLX-8bit", - "Qwen3.6-35B-A3B-4bit", - "Qwen3.6-35B-A3B-4bit-fp16", - "Qwen3.6-35B-A3B-UD-MLX-4bit", - "Qwen3.6-35B-A3B-UD-MLX-4bit-fp16", - "bge-m3-mlx-8bit", - "gemma-3-12b-it-qat-4bit", - "gemma-3-12b-it-qat-fp16", - "gemma-4-26b-a4b-it-4bit", - "gemma-4-31b-it-MLX-8bit", - "mlx-community--Qwen3.5-35B-A3B-4bit", - "mlx-community--gemma-3-12b-it-qat-4bit" + "" ], "first": { "phase": "first", @@ -86,6 +59,7 @@ "generation_tokens_per_second": 7650.61 }, "error": null - } + }, + "served_model_count": 28 } }