From a84aaa19658610050da0a31d6c1eb15f9fe51e1c Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 09:42:32 +0200 Subject: [PATCH 01/10] fix(agent): keep prior answers verbatim in lean state Replacing prior tool-turn answers with "[earlier answer from ...]" taught the model that format. In a long room it sent the placeholder as its reply and repeated the same grep. - lean_state: only prior tool results become pointers - test: no assistant message holds a placeholder after 10 tool turns - log entry in agent-improvement-log.md --- docs/design/agent/agent-improvement-log.md | 20 ++++++++++ stacklets/agent/runtime/lean_state.py | 44 +++++++--------------- stacklets/agent/runtime/sitecustomize.py | 5 +-- tests/stacklets/test_lean_state.py | 28 +++++++++++--- 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/docs/design/agent/agent-improvement-log.md b/docs/design/agent/agent-improvement-log.md index 06229c6b..ee660d18 100644 --- a/docs/design/agent/agent-improvement-log.md +++ b/docs/design/agent/agent-improvement-log.md @@ -1276,3 +1276,23 @@ list_edit(op=add, items=["butter","eggs","flour"]) -> "added 3" in one commit (was 3 calls, 3 commits). Full lifecycle correct: every item kept, [x] preserved, clean commit trail. Rig now runs the production transform, so this exercised the real store code. + +## 2026-09-19 - Answer decay removed from lean_state + +Failure: in a room with a long history, a vault question ran 11 +iterations, sent the same grep 4 times, then replied with the placeholder +text `[earlier answer from grep({...}); re-run for the current value]` +instead of an answer. + +Cause: commit 6b80a08 replaced each prior answer of a tool-using turn +with that placeholder. In a long room most assistant turns in the +context were placeholders, and the model copied the format. + +| Rewrite | Role | Result | +|---|---|---| +| prior tool result -> `[prior result of ...]` | tool | kept, not copied | +| prior tool-turn answer -> `[earlier answer from ...]` | assistant | removed, copied into replies | + +Trade-off: a prior answer that recites a list is in the context again. +The tool result behind it is still a pointer. Regression test: +`test_no_assistant_message_holds_a_placeholder`. diff --git a/stacklets/agent/runtime/lean_state.py b/stacklets/agent/runtime/lean_state.py index d50ab89b..780266f5 100644 --- a/stacklets/agent/runtime/lean_state.py +++ b/stacklets/agent/runtime/lean_state.py @@ -62,43 +62,27 @@ def _call_labels(messages: list[dict[str, Any]]) -> dict[str, str]: def lean_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Collapse previous-turn *derived data* to pointers, keeping conversation. - - Two kinds of stale projection are replaced by a pointer that names the call - that produced them, so the model re-fetches instead of reciting: - - a tool **result** from a previous turn, and - - the assistant's synthesized **answer** in a turn that used a tool (a list - it read out, a count it reported). - Assistant messages in turns that used no tool are pure conversation and are - left verbatim, as is everything from the current turn (anchored on the last + """Collapse previous-turn tool results to pointers, keeping conversation. + + A tool **result** from a previous turn is replaced by a pointer that names + the call that produced it, so the model re-fetches instead of reciting. + Everything else stays verbatim, including the assistant's answers from + previous turns and everything from the current turn (anchored on the last user message). Only ``content`` is rewritten; the ``tool_call_id`` pairing with the assistant's ``tool_calls`` is preserved, so the list stays valid. + + Assistant answers are never rewritten. A placeholder in the assistant role + is an example of what an answer looks like: in a long room most prior + answers were placeholders, and the model copied the format into its + reply. Placeholders in the tool role are not copied. """ boundary = _last_user_index(messages) labels = _call_labels(messages) out: list[dict[str, Any]] = [] - turn_calls: list[str] = [] # tool calls issued in the turn being scanned - since_tool = False # have we passed a tool result in this turn? for i, m in enumerate(messages): - role = m.get("role") - if role == "user": - turn_calls, since_tool = [], False - for tc in m.get("tool_calls") or []: - if label := labels.get(tc.get("id")): - turn_calls.append(label) - prior = i < boundary - if role == "tool": - since_tool = True - if prior: - call = labels.get(m.get("tool_call_id"), "the tool") - m = {**m, "content": f"[prior result of {call}; re-run for the current value]"} - elif prior and role == "assistant" and since_tool and not m.get("tool_calls"): - # The synthesized answer of a tool-using turn is itself a stale - # projection -- decay it too, naming the call, so the model - # re-derives instead of reciting an old conclusion. Answers from - # turns that used no tool (pure conversation) are left untouched. - calls = "; ".join(dict.fromkeys(turn_calls)) or "a tool call" - m = {**m, "content": f"[earlier answer from {calls}; re-run for the current value]"} + if i < boundary and m.get("role") == "tool": + call = labels.get(m.get("tool_call_id"), "the tool") + m = {**m, "content": f"[prior result of {call}; re-run for the current value]"} out.append(m) return out diff --git a/stacklets/agent/runtime/sitecustomize.py b/stacklets/agent/runtime/sitecustomize.py index 1dd234d1..2f5bd02c 100644 --- a/stacklets/agent/runtime/sitecustomize.py +++ b/stacklets/agent/runtime/sitecustomize.py @@ -153,9 +153,8 @@ def _runtime_lines(state, msg, workspace, *, skip=False): _LEAN_ENABLED = _os.environ.get("AGENT_LEAN_STATE", "1") != "0" def _build_messages_lean(self, *args, **kwargs): - # Post-process the assembled message list: stale prior-turn derived data - # (tool results and tool-synthesized answers) become pointers; the - # current turn stays intact. + # Post-process the assembled message list: stale prior-turn tool results + # become pointers; answers and the current turn stay intact. messages = _orig_build_messages(self, *args, **kwargs) if _LEAN_ENABLED: messages = _lean_messages(messages) diff --git a/tests/stacklets/test_lean_state.py b/tests/stacklets/test_lean_state.py index 27684c80..b8517c42 100644 --- a/tests/stacklets/test_lean_state.py +++ b/tests/stacklets/test_lean_state.py @@ -68,8 +68,9 @@ def test_pairing_preserved_and_nothing_dropped(self): assert out[1]["tool_calls"][0]["id"] == "c1" # assistant call kept assert len(out) == len(msgs) # nothing dropped - def test_tool_turn_answer_is_decayed(self): - # a previous turn that called a tool: its synthesized answer decays too + def test_tool_turn_answer_is_kept(self): + # a previous turn that called a tool: only the tool result is pointered, + # the answer stays verbatim msgs = [ {"role": "user", "content": "was ist offen?"}, _asst_call("c1", "exec", '{"command": "stack memory topic x todo"}'), @@ -78,9 +79,26 @@ def test_tool_turn_answer_is_decayed(self): {"role": "user", "content": "und jetzt?"}, # current turn ] out = lean_messages(msgs) - assert "Noch 8 offen" not in out[3]["content"] - assert "earlier answer from" in out[3]["content"] - assert "exec(" in out[3]["content"] + assert out[3]["content"] == "Noch 8 offen: a, b, c" + assert "8 open" not in out[2]["content"] + + def test_no_assistant_message_holds_a_placeholder(self): + # Regression: with every prior answer replaced by + # "[earlier answer from grep(...)]" the model sent that format as + # its reply. No assistant content may carry a placeholder. + msgs = [] + for n in range(10): + msgs += [ + {"role": "user", "content": f"frage {n}"}, + _asst_call(f"c{n}", "grep", '{"pattern": "Lauf"}'), + _tool_result(f"c{n}", f"treffer {n}"), + {"role": "assistant", "content": f"antwort {n}"}, + ] + msgs.append({"role": "user", "content": "neue frage"}) + out = lean_messages(msgs) + answers = [m["content"] for m in out + if m["role"] == "assistant" and not m.get("tool_calls")] + assert answers == [f"antwort {n}" for n in range(10)] def test_conversational_answer_is_kept(self): # a previous turn with NO tool: its answer is conversation, kept verbatim From 3570c2b10344361d110339ca083b95f9c32c826e Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 09:59:59 +0200 Subject: [PATCH 02/10] refactor(agent): leave context size to nanobot, drop lean_state and grep routing Our context shims worked against nanobot's own mechanisms. lean_state duplicated microcompact, broke the prefix cache, and its "re-run" placeholders made the model repeat tool calls. grep_tool ran regex patterns as semantic queries, so the model retried the same grep. - lean_state removed; state_log keeps the debug log, opt-in via AGENT_STATE_LOG=1 - grep_tool removed; vault grep is literal again - compact_tools: vault read tools join nanobot's microcompact set - config.json: context_window_tokens 32768, so token consolidation and history snip act (budget 23.5k) and prefill stays short - agent-lab: replay subcommand removed with lean_messages - ADR-012 update, improvement log entry --- docs/adr/adr-012-nanobot-fork.md | 13 ++ docs/design/agent/agent-improvement-log.md | 17 +++ stacklets/agent/config.json | 1 + stacklets/agent/runtime/README.md | 17 ++- stacklets/agent/runtime/compact_tools.py | 28 ++++ stacklets/agent/runtime/grep_tool.py | 85 ------------ stacklets/agent/runtime/lean_state.py | 115 ---------------- stacklets/agent/runtime/sitecustomize.py | 100 ++++++-------- stacklets/agent/runtime/state_log.py | 39 ++++++ tests/stacklets/conftest.py | 9 +- tests/stacklets/test_agent_grep_tool.py | 21 --- tests/stacklets/test_agent_runtime_shims.py | 42 +++--- tests/stacklets/test_agent_vault_tools.py | 2 +- tests/stacklets/test_lean_state.py | 145 -------------------- tests/stacklets/test_state_log.py | 35 +++++ tools/agent-lab/README.md | 3 +- tools/agent-lab/lab.py | 70 +--------- tools/agent-lab/rig/README.md | 2 +- 18 files changed, 228 insertions(+), 516 deletions(-) create mode 100644 stacklets/agent/runtime/compact_tools.py delete mode 100644 stacklets/agent/runtime/grep_tool.py delete mode 100644 stacklets/agent/runtime/lean_state.py create mode 100644 stacklets/agent/runtime/state_log.py delete mode 100644 tests/stacklets/test_agent_grep_tool.py delete mode 100644 tests/stacklets/test_lean_state.py create mode 100644 tests/stacklets/test_state_log.py diff --git a/docs/adr/adr-012-nanobot-fork.md b/docs/adr/adr-012-nanobot-fork.md index ed6ed455..f6e1c804 100644 --- a/docs/adr/adr-012-nanobot-fork.md +++ b/docs/adr/adr-012-nanobot-fork.md @@ -199,3 +199,16 @@ diff keeps growing. * Where the fork lives. Arthur refers to reactivating an existing one; it is not visible under `famstack-dev` or `arthware-dev` from this machine. * Whether the vault tools move in or stay on the discovery seam. + +## Update 2026-09-19 + +Two shims removed; history size is left to nanobot's own settings. + +| Shim | Change | Reason | +|---|---|---| +| `lean_state` | removed; `state_log` keeps the debug log, opt-in (`AGENT_STATE_LOG=1`) | duplicated nanobot's microcompact, broke the prefix cache, and its "re-run" placeholders made the model repeat tool calls | +| `grep_tool` | removed; grep is literal again | ran a regex as a semantic query (lesson 7); the model retried the same grep | +| `compact_tools` | added; patches `agent.runner._COMPACTABLE_TOOLS` | nanobot's microcompact now also shortens old vault tool results | + +`config.json` sets `context_window_tokens` to 32768, so nanobot's token +consolidation and history snip act at a size that keeps prefill short. diff --git a/docs/design/agent/agent-improvement-log.md b/docs/design/agent/agent-improvement-log.md index ee660d18..3dc17786 100644 --- a/docs/design/agent/agent-improvement-log.md +++ b/docs/design/agent/agent-improvement-log.md @@ -1296,3 +1296,20 @@ context were placeholders, and the model copied the format. Trade-off: a prior answer that recites a list is in the context again. The tool result behind it is still a pointer. Regression test: `test_no_assistant_message_holds_a_placeholder`. + +## 2026-09-19 - Context size left to nanobot + +The shims that rewrote context worked against nanobot's own mechanisms. + +| Item | Before | After | +|---|---|---| +| Prior tool results | `lean_state` placeholder, all of them, every turn | nanobot microcompact: 10 newest kept, older >= 500 chars omitted | +| Vault tool results | not compactable (names missing from `_COMPACTABLE_TOOLS`) | compactable (`compact_tools.py`) | +| Vault grep | semantic query via `grep_tool` | literal grep | +| `context_window_tokens` | 200000 (default; consolidation never ran) | 32768 (budget 23.5k, consolidation target 11.7k) | +| `llm-state.log` | every `build_messages` call | only with `AGENT_STATE_LOG=1` | + +Why autocompact did not shrink the room: it keeps the last 8 messages, +extended back to the start of that turn. A turn with 11 tool calls has +24 messages, so the whole turn stayed. + diff --git a/stacklets/agent/config.json b/stacklets/agent/config.json index e6af6e6f..51c4b090 100644 --- a/stacklets/agent/config.json +++ b/stacklets/agent/config.json @@ -16,6 +16,7 @@ "model_preset": "primary", "max_tool_iterations": 12, "max_messages": 40, + "context_window_tokens": 32768, "disabled_skills": ["memory", "my"] } }, diff --git a/stacklets/agent/runtime/README.md b/stacklets/agent/runtime/README.md index 679f24e5..d9e7f2c7 100644 --- a/stacklets/agent/runtime/README.md +++ b/stacklets/agent/runtime/README.md @@ -1,13 +1,22 @@ -# Agent runtime shims — per-turn briefing + lean state +# Agent runtime shims — per-turn briefing This directory is a **contained modification of nanobot**, loaded into the agent container. It exists because nanobot has no plugin seam for shaping per-turn context, and we did not want to fork nanobot for a couple of hooks. Everything here lives in the stacklet; upstream `nanobot-ai` is installed unchanged. -Two independent shims live here, each a thin monkeypatch over a pure module: -**brief** (what the agent knows going in) and **lean_state** (keeping what it -carries forward small and fresh). +The main context shim is **brief** (what the agent knows going in). History +size is left to nanobot's own settings in `config.json`: + +| Mechanism | Setting | Value | Effect | +|---|---|---|---| +| Replay window | `max_messages` | 40 | Older messages move to `history.jsonl` and the `# Recent History` section | +| Token budget | `context_window_tokens` | 32768 | Budget = window - 8192 output - 1024 = 23.5k; consolidation trims to 50% of it, history snip cuts at it | +| Idle autocompact | `idleCompactAfterMinutes` | 15 (default) | Idle session: summary plus the last turn | +| Microcompact | `_COMPACTABLE_TOOLS` | nanobot's read tools + vault tools (`compact_tools.py`) | Keeps the 10 newest tool results, older ones (>= 500 chars) become `[ result omitted from context]` | + +A small window keeps prefill short on the local model. `AGENT_STATE_LOG=1` +writes the message list of each turn to `~/.nanobot/llm-state.log`. ## What it does diff --git a/stacklets/agent/runtime/compact_tools.py b/stacklets/agent/runtime/compact_tools.py new file mode 100644 index 00000000..5866f987 --- /dev/null +++ b/stacklets/agent/runtime/compact_tools.py @@ -0,0 +1,28 @@ +"""Let nanobot's microcompact shorten the results of our vault tools. + +Before each model call, nanobot's `AgentRunner._microcompact` keeps the +10 most recent results of the tools in `_COMPACTABLE_TOOLS` and replaces +older ones (500 characters or more) with `[ result omitted from +context]`. The set lists nanobot's own read tools only. Our vault tools +return results of about 1k tokens each, so without this they stay in +the context in full for as long as the session replays them. + +PIN: `nanobot.agent.runner._COMPACTABLE_TOOLS` (a frozenset of tool +names, read at call time). Re-verify on a nanobot bump. +""" + +from __future__ import annotations + +# Tool names as the model sees them. list_edit is not here: its results +# are one line ("added 3") and stay under the 500-character floor. +VAULT_READ_TOOLS = frozenset({ + "memory_search", + "memory_person", + "memory_history", +}) + + +def install() -> None: + import nanobot.agent.runner as runner + + runner._COMPACTABLE_TOOLS = frozenset(runner._COMPACTABLE_TOOLS) | VAULT_READ_TOOLS diff --git a/stacklets/agent/runtime/grep_tool.py b/stacklets/agent/runtime/grep_tool.py deleted file mode 100644 index 822b0007..00000000 --- a/stacklets/agent/runtime/grep_tool.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Route vault grep calls through semantic family memory search.""" - -from __future__ import annotations - -import re -from typing import Any - -_PATH_RE = re.compile(r"^\s*#\d+\s+.*?\s([^\s]+\.md)\s+score=", re.MULTILINE) - - -def _is_vault_path(path: str | None) -> bool: - path = (path or ".").strip().replace("\\", "/") - return path in {"vault", "./vault"} or path.startswith(("vault/", "./vault/")) - - -def install() -> None: - """Patch nanobot's grep tool so vault searches use memory_search.""" - from nanobot.agent.tools.search import GrepTool - - original = GrepTool.execute - - async def execute_with_memory( - self: GrepTool, - pattern: str, - path: str = ".", - glob: str | None = None, - type: str | None = None, - case_insensitive: bool = False, - fixed_strings: bool = False, - output_mode: str = "files_with_matches", - context_before: int = 0, - context_after: int = 0, - max_matches: int | None = None, - max_results: int | None = None, - head_limit: int | None = None, - offset: int = 0, - **kwargs: Any, - ) -> str: - if not _is_vault_path(path): - return await original( - self, - pattern=pattern, - path=path, - glob=glob, - type=type, - case_insensitive=case_insensitive, - fixed_strings=fixed_strings, - output_mode=output_mode, - context_before=context_before, - context_after=context_after, - max_matches=max_matches, - max_results=max_results, - head_limit=head_limit, - offset=offset, - **kwargs, - ) - - limit = head_limit or max_results or max_matches or 10 - if limit == 0: - limit = 20 - - scope = None - normalized = path.strip().replace("\\", "/").removeprefix("./") - if normalized.startswith("vault/"): - scope = normalized.removeprefix("vault/").strip("/") or None - - from memory_tool import MemorySearchTool - - result = await MemorySearchTool().execute( - query=pattern, - limit=min(max(int(limit), 1), 20), - scope=scope, - ) - paths = [f"vault/{path}" for path in _PATH_RE.findall(result)] - path_block = "" - if paths: - path_block = "Paths to read:\n" + "\n".join(f"- {path}" for path in paths) + "\n\n" - return ( - "Semantic vault search via memory_search. " - "Use returned vault paths with read_file for source verification.\n\n" - + path_block - + result - ) - - GrepTool.execute = execute_with_memory diff --git a/stacklets/agent/runtime/lean_state.py b/stacklets/agent/runtime/lean_state.py deleted file mode 100644 index 780266f5..00000000 --- a/stacklets/agent/runtime/lean_state.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Keep the agent's *state* lean, as distinct from the chat *transcript*. - -The transcript is the Matrix room: complete, immutable, always replayable. The -state is the far smaller working memory we feed the model each turn. By default -nanobot's context is a near-verbatim replay of the transcript, so a stale tool -result rides along and the model answers "what's still open?" by reciting an old -list instead of re-fetching. - -This module trims exactly one thing, well: tool results from *previous* turns. -It replaces the result body with the very call that produced it -- `name(args)` -- -so the state carries a cheap, self-documenting pointer (the model can re-issue -that call to refresh) instead of a fat, stale payload. The *current* turn's own -tool results are left whole, so the model still reasons over what it just -fetched. - -Portability: `lean_messages` is a pure transform over the OpenAI-style message -list. When we fork nanobot, call it as the last step of -`ContextBuilder.build_messages` and delete the monkeypatch in sitecustomize.py. -Nothing else moves. -""" - -from __future__ import annotations - -from typing import Any - -# Keep the re-fetch pointer short even when the original call carried big args -# (a write_file, a long command). We name the call, not its payload. -_ARGS_CAP = 200 - - -def _last_user_index(messages: list[dict[str, Any]]) -> int: - """Index of the last user message: the boundary of the current turn. - - At or after it is this turn -- its fresh tool results must stay. Before it is - a previous turn -- those tool results get pointered. Anchoring on the current - user message is correct whether ``build_messages`` runs once per turn or once - per tool-loop iteration, so we never trim results the model is mid-reasoning - over. - """ - for i in range(len(messages) - 1, -1, -1): - if messages[i].get("role") == "user": - return i - return len(messages) # no user message -> treat everything as prior - - -def _call_labels(messages: list[dict[str, Any]]) -> dict[str, str]: - """Map each ``tool_call_id`` to a short ``name(arguments)`` label, read from - the assistant messages that requested the calls.""" - labels: dict[str, str] = {} - for m in messages: - for tc in m.get("tool_calls") or []: - tcid = tc.get("id") - if not tcid: - continue - fn = tc.get("function") or {} - name = fn.get("name") or "tool" - args = (fn.get("arguments") or "").strip() - if len(args) > _ARGS_CAP: - args = args[:_ARGS_CAP] + "..." - labels[tcid] = f"{name}({args})" if args else f"{name}()" - return labels - - -def lean_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Collapse previous-turn tool results to pointers, keeping conversation. - - A tool **result** from a previous turn is replaced by a pointer that names - the call that produced it, so the model re-fetches instead of reciting. - Everything else stays verbatim, including the assistant's answers from - previous turns and everything from the current turn (anchored on the last - user message). Only ``content`` is rewritten; the ``tool_call_id`` pairing - with the assistant's ``tool_calls`` is preserved, so the list stays valid. - - Assistant answers are never rewritten. A placeholder in the assistant role - is an example of what an answer looks like: in a long room most prior - answers were placeholders, and the model copied the format into its - reply. Placeholders in the tool role are not copied. - """ - boundary = _last_user_index(messages) - labels = _call_labels(messages) - out: list[dict[str, Any]] = [] - for i, m in enumerate(messages): - if i < boundary and m.get("role") == "tool": - call = labels.get(m.get("tool_call_id"), "the tool") - m = {**m, "content": f"[prior result of {call}; re-run for the current value]"} - out.append(m) - return out - - -_PREVIEW_CAP = 320 - - -def format_state_for_log(messages: list[dict[str, Any]]) -> str: - """Render the message list as a compact transcript for the log. - - One line per message -- ROLE + a clipped, single-line preview -- so the state - we actually send the model can be eyeballed against the Matrix chat: is the - system prompt what we think, did the stale results collapse to pointers, is - the current turn intact. Debug aid only; never in the model's context. - """ - lines = [] - for m in messages: - role = str(m.get("role", "?")).upper() - content = m.get("content") - if isinstance(content, list): # multimodal parts -> just the text - content = " ".join(p.get("text", "") for p in content - if isinstance(p, dict)) - content = (content or "").replace("\n", " / ") - if len(content) > _PREVIEW_CAP: - content = content[:_PREVIEW_CAP] + "..." - if calls := m.get("tool_calls"): - names = ", ".join((c.get("function") or {}).get("name", "?") for c in calls) - content = f"->calls {names} {content}".rstrip() - lines.append(f" {role:9} {content}") - return "\n".join(lines) diff --git a/stacklets/agent/runtime/sitecustomize.py b/stacklets/agent/runtime/sitecustomize.py index 2f5bd02c..252ffe8f 100644 --- a/stacklets/agent/runtime/sitecustomize.py +++ b/stacklets/agent/runtime/sitecustomize.py @@ -7,7 +7,9 @@ Two kinds of patch live here, each a thin monkeypatch over a pure module. -First, two context shims that reshape what the model sees per turn: +First, the context shims. History length and old tool results are left to +nanobot's own mechanisms (replay window, idle autocompact, microcompact, token +consolidation); see config.json and compact_tools.py. 1. brief (brief.py) — prepends a per-turn family briefing (who is speaking, the topic) to nanobot's runtime lines. Injected late (after the stable prompt and @@ -15,12 +17,11 @@ tokens cached with late injection vs 0 injecting the same content early via USER.md. -2. lean_state (lean_state.py) — replaces previous-turn tool results with the - call that produced them (`name(args)`), so the agent re-fetches instead of - reciting stale data. The transcript (Matrix) keeps the full result; the state - we feed the model keeps only a cheap pointer. +2. state_log (state_log.py) — with AGENT_STATE_LOG=1, writes the message list + of each turn to ~/.nanobot/llm-state.log. Off by default. It does not change + the message list. -Second, four vault tools, which add capability rather than reshaping context. +Second, the vault tools, which add capability rather than reshaping context. They are *tools* and not lines in a skill because that is the difference between a capability the model chooses and one it has to remember: told in prose to run `stack memory history`, it called `memory_search` four times @@ -31,9 +32,8 @@ 5. history_tool (history_tool.py) — a `memory_history` tool for questions with time in them. Search ranks pages by what they say now, so "lately", "since when" and "who changed this" are unanswerable by it, silently. -6. grep_tool (grep_tool.py) — routes greps under `vault/` into memory_search, so - the agent gets semantic hits instead of literal matches on a corpus where the - words it greps for are rarely the words on disk. +6. compact_tools (compact_tools.py) — adds the vault read tools to nanobot's + microcompact set, so their old results are shortened like nanobot's own. Third, one shim that widens when the agent is allowed to answer at all: @@ -63,13 +63,13 @@ PIN / RECHECK ON UPGRADE (re-verify after any `nanobot-ai` version bump) brief: `nanobot.agent.context.runtime_lines(state, msg, workspace, *, skip=False) -> list[str]` - lean_state: `nanobot.agent.context.ContextBuilder.build_messages(...) -> list[dict]` + state_log: `nanobot.agent.context.ContextBuilder.build_messages(...) -> list[dict]` memory_tool: `nanobot.agent.tools.loader.ToolLoader.discover(self) -> list[type[Tool]]` `nanobot.agent.tools.base.Tool`, `nanobot.agent.tools.base.tool_parameters` `nanobot.agent.tools.schema.{StringSchema, IntegerSchema, tool_parameters_schema}` person_tool: same symbols as memory_tool history_tool: same symbols as memory_tool - grep_tool: `nanobot.agent.tools.search.GrepTool.execute(...) -> str` + compact_tools: `nanobot.agent.runner._COMPACTABLE_TOOLS` (frozenset of tool names) vault_write: `nanobot.agent.tools.filesystem.WriteFileTool.execute(self, path, content) -> str` `nanobot.agent.tools.filesystem.EditFileTool.execute(self, path, ...) -> str` `nanobot.agent.tools.apply_patch.ApplyPatchTool.execute(self, edits, ...) -> str` @@ -129,64 +129,54 @@ def _runtime_lines(state, msg, workspace, *, skip=False): _log.exception("brief shim could not attach (nanobot internals changed?)") -# ── lean_state: previous-turn tool results -> a pointer naming the call ────── -# Also the single place to see the *state* (what the model receives) next to the -# *transcript* (the Matrix room): every turn logs the leaned message list, one -# line per message, greppable by "[llm-state]" in `docker logs stack-agent`. +# ── state_log: the message list per turn, for debugging (opt-in) ───────────── +# AGENT_STATE_LOG=1 writes what nanobot built for each call to llm-state.log, +# one line per message, to compare against the Matrix chat. Off by default: +# nanobot also calls build_messages to estimate tokens, so the file grows by +# several blocks per turn. try: - import datetime as _dt import os as _os - import nanobot.agent.context as _ctx_ls - from lean_state import format_state_for_log as _format_state - from lean_state import lean_messages as _lean_messages - - _orig_build_messages = _ctx_ls.ContextBuilder.build_messages - # Bind-mounted home (~/.nanobot -> famstack-data/agent), so this file is - # readable on the host for analysis, one appended block per turn. - _STATE_LOG = _os.path.expanduser("~/.nanobot/llm-state.log") - - # Experiment switch: AGENT_LEAN_STATE=0 disables the history rewrite and - # keeps the message list append-only. The rewrite invalidates the oMLX - # prefix cache from the first rewritten message on (measured 2026-09-15, - # see docs/design/agent/agent-improvement-log.md). Default stays on. - _LEAN_ENABLED = _os.environ.get("AGENT_LEAN_STATE", "1") != "0" - - def _build_messages_lean(self, *args, **kwargs): - # Post-process the assembled message list: stale prior-turn tool results - # become pointers; answers and the current turn stay intact. - messages = _orig_build_messages(self, *args, **kwargs) - if _LEAN_ENABLED: - messages = _lean_messages(messages) - try: # a debug view; never worth breaking a turn over - stamp = _dt.datetime.now().isoformat(timespec="seconds") - with open(_STATE_LOG, "a", encoding="utf-8") as fh: - fh.write(f"\n===== {stamp} {len(messages)} messages =====\n" - + _format_state(messages) + "\n") - print(f"[llm-state] {len(messages)} msgs -> llm-state.log", flush=True) - except Exception: - pass - return messages - - _ctx_ls.ContextBuilder.build_messages = _build_messages_lean - _log.info("lean-state message shim active") + if _os.environ.get("AGENT_STATE_LOG", "0") == "1": + import datetime as _dt + + import nanobot.agent.context as _ctx_ls + from state_log import format_state_for_log as _format_state + + _orig_build_messages = _ctx_ls.ContextBuilder.build_messages + # Bind-mounted home (~/.nanobot -> famstack-data/agent), so this file + # is readable on the host for analysis. + _STATE_LOG = _os.path.expanduser("~/.nanobot/llm-state.log") + + def _build_messages_logged(self, *args, **kwargs): + messages = _orig_build_messages(self, *args, **kwargs) + try: # a debug view; never worth breaking a turn over + stamp = _dt.datetime.now().isoformat(timespec="seconds") + with open(_STATE_LOG, "a", encoding="utf-8") as fh: + fh.write(f"\n===== {stamp} {len(messages)} messages =====\n" + + _format_state(messages) + "\n") + except Exception: + pass + return messages + + _ctx_ls.ContextBuilder.build_messages = _build_messages_logged + _log.info("state-log shim active") except Exception: - _log.exception("lean-state shim could not attach (nanobot internals changed?)") + _log.exception("state-log shim could not attach (nanobot internals changed?)") -# ── vault tools: memory_search, memory_person, and grep routed through them ── +# ── vault tools: memory_search, memory_person, memory_history, writes ──────── # These add capability rather than reshaping context, but attach the same way. # Each is installed in its own try so one tool failing costs only itself; a -# single shared block would let a moved GrepTool symbol take memory_search down -# with it. memory_tool goes first because grep_tool routes into it. +# single shared block would let one moved nanobot symbol take the others down. for _module_name, _what in ( ("memory_tool", "memory_search tool"), ("person_tool", "memory_person tool"), ("history_tool", "memory_history tool"), - ("grep_tool", "vault grep -> memory_search routing"), ("vault_write", "write_file on a vault page -> stack memory write"), ("list_tool", "list_edit item tool -> stack memory list-edit"), ("tool_trim", "unused-tool trim (AGENT_TOOL_TRIM=0 to disable)"), + ("compact_tools", "vault tools in nanobot's microcompact set"), ("thread_session", "thread-scoped sessions (AGENT_THREAD_SESSIONS=0 to disable)"), ): try: @@ -292,7 +282,7 @@ def _is_bot_mentioned_or_our_thread(self, event): from brief import topic_for_room_label as _topic_for_room_label from join_greeting import greeting_prompt as _greeting_prompt - # Same workspace nanobot mounts the projection into; `lean_state` + # Same workspace nanobot mounts the projection into; `state_log` # above resolves its log the same way. _WORKSPACE = _Path(_ospath.expanduser("~/.nanobot/workspace")) diff --git a/stacklets/agent/runtime/state_log.py b/stacklets/agent/runtime/state_log.py new file mode 100644 index 00000000..2944cd58 --- /dev/null +++ b/stacklets/agent/runtime/state_log.py @@ -0,0 +1,39 @@ +"""Render the message list nanobot builds for a turn, for the debug log. + +The transcript is the Matrix room. The state is the message list that +`ContextBuilder.build_messages` assembles for the model. This module +formats that list as one line per message, so the two can be compared. + +The log shows the list before nanobot's per-call steps in +`AgentRunner` (microcompact, tool-result budget, history snip). Those +steps change only tool results and the oldest messages. +""" + +from __future__ import annotations + +from typing import Any + +_PREVIEW_CAP = 320 + + +def format_state_for_log(messages: list[dict[str, Any]]) -> str: + """Render the message list as a compact transcript for the log. + + One line per message: ROLE and a clipped, single-line preview. Debug + aid only; never in the model's context. + """ + lines = [] + for m in messages: + role = str(m.get("role", "?")).upper() + content = m.get("content") + if isinstance(content, list): # multimodal parts -> just the text + content = " ".join(p.get("text", "") for p in content + if isinstance(p, dict)) + content = (content or "").replace("\n", " / ") + if len(content) > _PREVIEW_CAP: + content = content[:_PREVIEW_CAP] + "..." + if calls := m.get("tool_calls"): + names = ", ".join((c.get("function") or {}).get("name", "?") for c in calls) + content = f"->calls {names} {content}".rstrip() + lines.append(f" {role:9} {content}") + return "\n".join(lines) diff --git a/tests/stacklets/conftest.py b/tests/stacklets/conftest.py index f69061fd..311d6707 100644 --- a/tests/stacklets/conftest.py +++ b/tests/stacklets/conftest.py @@ -79,10 +79,6 @@ class ToolRegistry: def register(self, tool): pass - class GrepTool: - async def execute(self, *args, **kwargs): - return "stock grep" - # The three write tools, `async def` exactly as upstream declares them. # That detail is the contract, not decoration: nanobot's tool loop # awaits the result, so a shim that replaces one with a sync function @@ -156,6 +152,10 @@ def mod(name, **attrs): mod("nanobot.agent") mod("nanobot.agent.context", runtime_lines=runtime_lines, ContextBuilder=ContextBuilder) + # AgentRunner's microcompact reads this set at call time; + # compact_tools extends it. + mod("nanobot.agent.runner", + _COMPACTABLE_TOOLS=frozenset({"read_file", "exec", "grep"})) mod("nanobot.agent.tools") mod("nanobot.agent.tools.base", Tool=Tool, tool_parameters=tool_parameters) mod("nanobot.agent.tools.schema", @@ -163,7 +163,6 @@ def mod(name, **attrs): tool_parameters_schema=tool_parameters_schema) mod("nanobot.agent.tools.loader", ToolLoader=ToolLoader) mod("nanobot.agent.tools.registry", ToolRegistry=ToolRegistry) - mod("nanobot.agent.tools.search", GrepTool=GrepTool) mod("nanobot.agent.tools.filesystem", WriteFileTool=WriteFileTool, EditFileTool=EditFileTool) mod("nanobot.agent.tools.apply_patch", ApplyPatchTool=ApplyPatchTool) diff --git a/tests/stacklets/test_agent_grep_tool.py b/tests/stacklets/test_agent_grep_tool.py deleted file mode 100644 index 0e668045..00000000 --- a/tests/stacklets/test_agent_grep_tool.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -from grep_tool import _PATH_RE, _is_vault_path - - -def test_detects_vault_paths(): - assert _is_vault_path("vault") - assert _is_vault_path("./vault") - assert _is_vault_path("vault/family") - assert _is_vault_path("./vault/family") - - -def test_non_vault_paths_are_not_routed(): - assert not _is_vault_path(".") - assert not _is_vault_path("memory/history.jsonl") - assert not _is_vault_path("not-vault/family") - - -def test_extracts_memory_result_paths(): - block = "#1 2026-06-23 [] family/emails/example.md score=0.7\n Title\n" - assert _PATH_RE.findall(block) == ["family/emails/example.md"] diff --git a/tests/stacklets/test_agent_runtime_shims.py b/tests/stacklets/test_agent_runtime_shims.py index f9a31d47..9002df8d 100644 --- a/tests/stacklets/test_agent_runtime_shims.py +++ b/tests/stacklets/test_agent_runtime_shims.py @@ -23,10 +23,10 @@ import pytest -SHIMMED_MODULES = ("sitecustomize", "brief", "lean_state", - "memory_tool", "person_tool", "history_tool", "grep_tool", +SHIMMED_MODULES = ("sitecustomize", "brief", "state_log", + "memory_tool", "person_tool", "history_tool", "name_trigger", "thread_trigger", "join_greeting", "vault_write", - "list_tool", "tool_trim", "thread_session") + "list_tool", "tool_trim", "thread_session", "compact_tools") # The stub nanobot itself lives in conftest as `nanobot_stub`, shared with @@ -90,23 +90,31 @@ def test_asking_the_vault_when_something_happened_is_a_tool(nanobot): assert "MemoryHistoryTool" in _discovered(mods) -def test_vault_greps_are_routed_through_memory_search(nanobot): - """A grep under `vault/` must no longer hit the stock literal matcher. +def test_vault_tool_results_are_microcompacted(nanobot): + """nanobot shortens old results of the tools in this set only. - The vault is prose. Literal grep over it answers almost nothing, which - is why this routing exists. + Without our names in it, every memory_search result stays in the + context in full for as long as the session replays it. """ mods = nanobot() - grep = mods["nanobot.agent.tools.search"].GrepTool - assert grep.execute.__name__ == "execute_with_memory" + tools = mods["nanobot.agent.runner"]._COMPACTABLE_TOOLS + assert {"memory_search", "memory_person", "memory_history"} <= tools + assert {"read_file", "exec", "grep"} <= tools, "nanobot's own set must stay" def test_context_shims_are_attached(nanobot): - """The two older shims, pinned the same way as the new tools.""" + """The briefing attaches; the message list is nanobot's own.""" mods = nanobot() ctx = mods["nanobot.agent.context"] assert ctx.runtime_lines.__name__ == "_runtime_lines" - assert ctx.ContextBuilder.build_messages.__name__ == "_build_messages_lean" + assert ctx.ContextBuilder.build_messages.__name__ == "build_messages" + + +def test_state_log_is_opt_in(nanobot, monkeypatch): + monkeypatch.setenv("AGENT_STATE_LOG", "1") + mods = nanobot() + ctx = mods["nanobot.agent.context"] + assert ctx.ContextBuilder.build_messages.__name__ == "_build_messages_logged" def test_being_named_counts_as_a_mention(nanobot, monkeypatch): @@ -267,10 +275,10 @@ def test_a_moved_symbol_does_not_take_the_others_down(nanobot): """One missing nanobot symbol must cost only its own tool. This is why each install runs in its own try. Sharing one block would - mean a renamed GrepTool silently removed memory_search too, and the - agent would lose vault access over an unrelated upgrade. + mean a renamed runner constant silently removed memory_search too, and + the agent would lose vault access over an unrelated upgrade. """ - mods = nanobot(drop="nanobot.agent.tools.search.GrepTool") + mods = nanobot(drop="nanobot.agent.runner._COMPACTABLE_TOOLS") assert _discovered(mods) == {"MemorySearchTool", "MemoryPersonTool", "MemoryHistoryTool", "ListEditTool"} @@ -289,9 +297,9 @@ def test_the_stub_can_actually_express_a_detached_shim(nanobot): assert not hasattr(tools, "ToolLoader"), "the drop hook must really remove it" # Nothing to append to, so neither tool can have registered anywhere. - grep = mods["nanobot.agent.tools.search"].GrepTool - assert grep.execute.__name__ == "execute_with_memory", ( - "grep routing is independent of the loader and should still attach" + tools = mods["nanobot.agent.runner"]._COMPACTABLE_TOOLS + assert "memory_search" in tools, ( + "compact_tools is independent of the loader and should still attach" ) diff --git a/tests/stacklets/test_agent_vault_tools.py b/tests/stacklets/test_agent_vault_tools.py index 0cee29f5..04ec666b 100644 --- a/tests/stacklets/test_agent_vault_tools.py +++ b/tests/stacklets/test_agent_vault_tools.py @@ -36,7 +36,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent MEMORY_DIR = REPO_ROOT / "stacklets" / "memory" -TOOL_MODULES = ("memory_tool", "person_tool", "grep_tool", "sitecustomize") +TOOL_MODULES = ("memory_tool", "person_tool", "sitecustomize") # ── loading the real components under test ─────────────────────────── diff --git a/tests/stacklets/test_lean_state.py b/tests/stacklets/test_lean_state.py deleted file mode 100644 index b8517c42..00000000 --- a/tests/stacklets/test_lean_state.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Lean agent state: previous-turn tool results become a pointer naming the call. - -The transcript (Matrix) keeps the full result; the state we feed the model keeps -only the call that produced it, so the model re-fetches instead of reciting a -stale payload. See stacklets/agent/runtime/lean_state.py. -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent - / "stacklets" / "agent" / "runtime")) - -from lean_state import format_state_for_log, lean_messages # noqa: E402 - - -def _asst_call(tcid, name, args): - return {"role": "assistant", "content": "", - "tool_calls": [{"id": tcid, "type": "function", - "function": {"name": name, "arguments": args}}]} - - -def _tool_result(tcid, content): - return {"role": "tool", "tool_call_id": tcid, "content": content} - - -class TestLeanMessages: - def test_prior_tool_result_becomes_a_named_pointer(self): - msgs = [ - {"role": "system", "content": "sys"}, - {"role": "user", "content": "was ist offen?"}, - _asst_call("c1", "exec", '{"command": "stack memory topic x todo"}'), - _tool_result("c1", "8 open\n- a\n- b"), - {"role": "assistant", "content": "8 offen"}, - {"role": "user", "content": "und jetzt?"}, # current turn boundary - ] - out = lean_messages(msgs) - tool_msg = out[3] - assert tool_msg["role"] == "tool" - assert tool_msg["tool_call_id"] == "c1" # pairing preserved - assert "8 open" not in tool_msg["content"] # stale payload gone - assert 'exec({"command": "stack memory topic x todo"})' in tool_msg["content"] - assert "re-run" in tool_msg["content"] - - def test_current_turn_tool_result_is_kept(self): - msgs = [ - {"role": "user", "content": "erste frage"}, - _tool_result("old", "STALE"), # a previous turn - {"role": "user", "content": "zweite frage"}, # current turn boundary - _asst_call("c2", "exec", '{"command": "grep foo"}'), - _tool_result("c2", "FRESH RESULT"), # current turn -> kept - ] - out = lean_messages(msgs) - assert out[1]["content"] != "STALE" - assert out[4]["content"] == "FRESH RESULT" - - def test_pairing_preserved_and_nothing_dropped(self): - msgs = [ - {"role": "user", "content": "q"}, - _asst_call("c1", "exec", '{"command": "x"}'), - _tool_result("c1", "big result"), - {"role": "assistant", "content": "an answer"}, - {"role": "user", "content": "next"}, - ] - out = lean_messages(msgs) - assert out[1]["tool_calls"][0]["id"] == "c1" # assistant call kept - assert len(out) == len(msgs) # nothing dropped - - def test_tool_turn_answer_is_kept(self): - # a previous turn that called a tool: only the tool result is pointered, - # the answer stays verbatim - msgs = [ - {"role": "user", "content": "was ist offen?"}, - _asst_call("c1", "exec", '{"command": "stack memory topic x todo"}'), - _tool_result("c1", "8 open ..."), - {"role": "assistant", "content": "Noch 8 offen: a, b, c"}, - {"role": "user", "content": "und jetzt?"}, # current turn - ] - out = lean_messages(msgs) - assert out[3]["content"] == "Noch 8 offen: a, b, c" - assert "8 open" not in out[2]["content"] - - def test_no_assistant_message_holds_a_placeholder(self): - # Regression: with every prior answer replaced by - # "[earlier answer from grep(...)]" the model sent that format as - # its reply. No assistant content may carry a placeholder. - msgs = [] - for n in range(10): - msgs += [ - {"role": "user", "content": f"frage {n}"}, - _asst_call(f"c{n}", "grep", '{"pattern": "Lauf"}'), - _tool_result(f"c{n}", f"treffer {n}"), - {"role": "assistant", "content": f"antwort {n}"}, - ] - msgs.append({"role": "user", "content": "neue frage"}) - out = lean_messages(msgs) - answers = [m["content"] for m in out - if m["role"] == "assistant" and not m.get("tool_calls")] - assert answers == [f"antwort {n}" for n in range(10)] - - def test_conversational_answer_is_kept(self): - # a previous turn with NO tool: its answer is conversation, kept verbatim - msgs = [ - {"role": "user", "content": "danke!"}, - {"role": "assistant", "content": "Gern geschehen."}, - {"role": "user", "content": "was ist offen?"}, # current turn - ] - out = lean_messages(msgs) - assert out[1]["content"] == "Gern geschehen." - - def test_orphan_tool_result_falls_back(self): - msgs = [_tool_result("orphan", "data"), - {"role": "user", "content": "now"}] - out = lean_messages(msgs) - assert "the tool" in out[0]["content"] - assert "data" not in out[0]["content"] - - def test_long_args_are_capped(self): - big = '{"content": "' + "x" * 500 + '"}' - msgs = [ - {"role": "user", "content": "q"}, - _asst_call("c1", "write_file", big), - _tool_result("c1", "ok"), - {"role": "user", "content": "next"}, - ] - out = lean_messages(msgs) - assert "..." in out[2]["content"] # args truncated - assert len(out[2]["content"]) < len(big) - - -class TestFormatStateForLog: - def test_compact_one_line_per_message(self): - msgs = [ - {"role": "system", "content": "long " * 200}, - {"role": "user", "content": "hallo\nwelt"}, - _asst_call("c1", "exec", '{"command":"x"}'), - ] - out = format_state_for_log(msgs) - assert "SYSTEM" in out and "USER" in out - assert "..." in out # long content clipped - assert " / " in out # newline flattened to one line - assert "->calls exec" in out # the tool call is named - assert out.count("\n") == len(msgs) - 1 # exactly one line per message diff --git a/tests/stacklets/test_state_log.py b/tests/stacklets/test_state_log.py new file mode 100644 index 00000000..80584410 --- /dev/null +++ b/tests/stacklets/test_state_log.py @@ -0,0 +1,35 @@ +"""The debug state log renders one line per message. + +See stacklets/agent/runtime/state_log.py. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent + / "stacklets" / "agent" / "runtime")) + +from state_log import format_state_for_log # noqa: E402 + + +def _asst_call(tcid, name, args): + return {"role": "assistant", "content": "", + "tool_calls": [{"id": tcid, "type": "function", + "function": {"name": name, "arguments": args}}]} + + +class TestFormatStateForLog: + def test_compact_one_line_per_message(self): + msgs = [ + {"role": "system", "content": "long " * 200}, + {"role": "user", "content": "hallo\nwelt"}, + _asst_call("c1", "exec", '{"command":"x"}'), + ] + out = format_state_for_log(msgs) + assert "SYSTEM" in out and "USER" in out + assert "..." in out # long content clipped + assert " / " in out # newline flattened to one line + assert "->calls exec" in out # the tool call is named + assert out.count("\n") == len(msgs) - 1 # exactly one line per message diff --git a/tools/agent-lab/README.md b/tools/agent-lab/README.md index 04fcab2c..18ad158b 100644 --- a/tools/agent-lab/README.md +++ b/tools/agent-lab/README.md @@ -18,7 +18,8 @@ python3 tools/agent-lab/lab.py cache --prefix-words 3000 --history-turns 6 object. Use it to find the cache fields that oMLX reports. - `cache`: measures prefix-cache behavior in four phases: cold, warm, warm2, mutate. The mutate phase rewrites one early message in place. - This simulates the `lean_state` rewrite in the agent runtime. + This simulates a rewrite of the history, for example an old tool + result replaced by a placeholder. Expected result: warm turns have a low TTFT. The mutate turn has a TTFT near the cold value. diff --git a/tools/agent-lab/lab.py b/tools/agent-lab/lab.py index 76268936..45e255f2 100644 --- a/tools/agent-lab/lab.py +++ b/tools/agent-lab/lab.py @@ -212,7 +212,7 @@ def cmd_cache(ep: Endpoint, args) -> dict: warm: the same conversation plus the real reply and one new turn. warm2: one more append-only turn. mutate: the warm2 conversation with one early message rewritten, - plus one new turn. This simulates the lean_state rewrite. + plus one new turn. This simulates any mid-history rewrite. Expected result: warm and warm2 have a low TTFT. mutate has a TTFT near the cold value, because the prefix diverges early. """ @@ -232,8 +232,8 @@ def turn(question: str) -> dict: print("phase warm2 ...") rows.append(_phase_row("warm2", turn("Give a final short sentence."))) - # Rewrite one early assistant message in place. This is the same - # operation lean_state applies to prior tool results. + # Rewrite one early assistant message in place, as a history + # rewrite (placeholder for an old tool result) would. msgs[2]["content"] = "[prior result of tool(args); re-run for the current value]" print("phase mutate ...") rows.append(_phase_row("mutate", turn("Give one more short sentence."))) @@ -243,62 +243,6 @@ def turn(question: str) -> dict: return {"phases": rows, "prefix_words": args.prefix_words, "history_turns": args.history_turns} -def build_scripted_history(seed: int, prefix_words: int, turns: int) -> list[list[dict]]: - """Build a scripted tool-using conversation, one block list per turn. - - Each turn has the shape nanobot produces: user question, assistant - tool call, tool result, assistant answer. All content is synthetic. - The script is deterministic, so both replay arms see identical bytes. - """ - blocks = [[{"role": "system", "content": filler_text(prefix_words, seed)}]] - for t in range(turns): - call_id = f"call_{t}" - args = json.dumps({"query": f"topic {t} " + " ".join(filler_text(4, seed + t).split())}) - blocks.append([ - {"role": "user", "content": f"Question {t}: " + filler_text(30, seed + 10 + t)}, - {"role": "assistant", "content": "", "tool_calls": [{ - "id": call_id, "type": "function", - "function": {"name": "memory_search", "arguments": args}, - }]}, - {"role": "tool", "tool_call_id": call_id, - "content": f"Result {t}: " + filler_text(150, seed + 100 + t)}, - {"role": "assistant", "content": f"Answer {t}: " + filler_text(60, seed + 200 + t)}, - ]) - return blocks - - -def cmd_replay(ep: Endpoint, args) -> dict: - """Replay the real lean_state transform against an append-only arm. - - Arm A builds each turn's payload with lean_messages() from the agent - runtime, as production does. Arm B sends the same history verbatim. - The two arms use different seeds, so they do not share cache blocks. - Expected result: arm B hits the prefix cache from turn 2 on. Arm A - misses it on every turn after the first tool result. - """ - sys.path.insert(0, str(REPO_ROOT / "stacklets" / "agent" / "runtime")) - from lean_state import lean_messages - - arms = {} - for arm, (transform, seed) in { - "lean_state (production)": (lean_messages, args.seed), - "append-only": (lambda m: m, args.seed + 1000), - }.items(): - blocks = build_scripted_history(seed, args.prefix_words, args.turns) - rows = [] - print(f"arm: {arm}") - for t in range(1, args.turns + 1): - # The payload at turn t: system + t-1 full turns + this turn's - # user message. This is the state at the start of a turn. - history = [m for block in blocks[:t] for m in block] - payload = transform(history + [blocks[t][0]]) - r = ep.chat(payload, max_tokens=args.max_tokens, stream=args.stream) - rows.append(_phase_row(f"turn {t}", r)) - print(json.dumps(rows[-1])) - arms[arm] = rows - return {"arms": arms, "prefix_words": args.prefix_words, "turns": args.turns} - - def append_log(entry_title: str, ep: Endpoint, payload: dict, note: str, result_file: Path): """Append one STE summary entry to the improvement log.""" stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") @@ -351,19 +295,13 @@ def main() -> int: p_cache.add_argument("--max-tokens", type=int, default=48) p_cache.add_argument("--seed", type=int, default=42) - p_replay = sub.add_parser("replay", help="replay real lean_state vs append-only") - p_replay.add_argument("--prefix-words", type=int, default=3000) - p_replay.add_argument("--turns", type=int, default=4) - p_replay.add_argument("--max-tokens", type=int, default=48) - p_replay.add_argument("--seed", type=int, default=7) - args = parser.parse_args() if not args.url or not args.model: print("error: no endpoint or model. Set --url/--model or stack.toml [ai].", file=sys.stderr) return 2 ep = Endpoint(args.url, args.key, args.model, args.timeout) - payload = {"probe": cmd_probe, "cache": cmd_cache, "replay": cmd_replay}[args.cmd](ep, args) + payload = {"probe": cmd_probe, "cache": cmd_cache}[args.cmd](ep, args) RESULTS_DIR.mkdir(exist_ok=True) stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") diff --git a/tools/agent-lab/rig/README.md b/tools/agent-lab/rig/README.md index d01fd533..5f2d6e7a 100644 --- a/tools/agent-lab/rig/README.md +++ b/tools/agent-lab/rig/README.md @@ -112,7 +112,7 @@ Full detail: `docs/design/agent/agent-improvement-log.md`. Change one thing, keep the same question and session name, compare the log tables. For an isolated behavior toggle, pass it as env, for example -`--env AGENT_LEAN_STATE=0` or `--env AGENT_TOOL_TRIM=0`. Always keep a +`--env AGENT_TOOL_TRIM=0` or `--env AGENT_THREAD_SESSIONS=0`. Always keep a baseline run so a result is a comparison, not a vibe check. For search-engine work, keep a `--backend regex` baseline (see the retrieval handover). From 8aa3179ffcd8b08a1de807aea22374b9a99bfa8c Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 11:06:24 +0200 Subject: [PATCH 03/10] chore: add CLAUDE.md that loads AGENTS.md and both role guides Claude Code reads CLAUDE.md, not AGENTS.md, so the repo rules were never in its context. - CLAUDE.md imports AGENTS.md, docs/agent/ops.md, docs/agent/dev.md - .gitignore: keep the root CLAUDE.md, still ignore nested ones --- .gitignore | 4 +++- CLAUDE.md | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 CLAUDE.md diff --git a/.gitignore b/.gitignore index f031c933..2c637ded 100644 --- a/.gitignore +++ b/.gitignore @@ -64,8 +64,10 @@ impl/ **CLAUDE.md **AGENT.md -# …except the repo's canonical agent guide at the root. +# …except the repo's canonical agent guide at the root, and the +# CLAUDE.md that loads it into Claude Code. !/AGENT.md +!/CLAUDE.md # Local planning workspace (Nimbalyst) — not product source nimbalyst-local/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..f6145af0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,3 @@ +@AGENTS.md +@docs/agent/ops.md +@docs/agent/dev.md From c6bb2657d6c9dfea4383b6eab5ec389950305f60 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 13:41:29 +0200 Subject: [PATCH 04/10] feat(memory): search the brain, rank by rare keywords Search read the source vault and sorted by date. Generated pages (the diary) exist only in the brain, and the newest pages naming a person filled the limit before the page with the rare word. - search reads memory/brain (memory/vault before the first curator run); no Forgejo pull on the brain, the curator writes it - rank: each top-level alternative scores its IDF, then date; one keyword keeps date order - excerpt: the line with the rarest matching keyword - "matches:" line for multi-keyword queries - title/tag values and diacritic folding from 2ca1116 - ADR-011 update: search reads the brain --- docs/adr/adr-011-vault-brain-projection.md | 18 + stacklets/memory/cli/search.py | 88 +++-- stacklets/memory/lib.py | 199 ++++++++++-- tests/stacklets/test_memory_search.py | 361 ++++++++++++++++++++- 4 files changed, 618 insertions(+), 48 deletions(-) diff --git a/docs/adr/adr-011-vault-brain-projection.md b/docs/adr/adr-011-vault-brain-projection.md index 016cf880..608cf23c 100644 --- a/docs/adr/adr-011-vault-brain-projection.md +++ b/docs/adr/adr-011-vault-brain-projection.md @@ -85,3 +85,21 @@ Consequences of the rule: trailer on brain commits), and a rebase of the prototype branch onto the todo work. Implementation is tracked in `docs/todos/brain-projection-plan.md`. + +## Update 2026-09-19: search reads the brain + +`stack memory search` reads the brain, not the vault. The rule above +("search and todos read memory") left search blind to generated pages. +The diary is compiled from the diary room, so its text exists only in +the brain, and a diary question could not be answered by search. The +agent fell back to grep on its `vault/` mount, which is the brain. + +| Reader | Tree | Reason | +|---|---|---| +| `stack memory search` | brain (vault before the first curator run) | brain holds every vault page plus the compiled pages | +| todos, writes | vault | read-your-writes stays a vault promise | + +Cost: a page filed seconds ago is findable after the next mirror tick, +not at once. Read-your-writes is tested for todos only +(`tests/integration/test_demo_rig_e2e.py`), so no stated invariant changes. + diff --git a/stacklets/memory/cli/search.py b/stacklets/memory/cli/search.py index 62ef1a80..d2af4026 100644 --- a/stacklets/memory/cli/search.py +++ b/stacklets/memory/cli/search.py @@ -1,10 +1,13 @@ -"""stack memory search — full-text query over the curated memory vault. +"""stack memory search — full-text query over the family brain. -The memory vault is the local checkout the memory stacklet maintains -at `/memory/vault/`. It holds the *derived intelligence* layer: -doc briefings, entity notes, bookmarks, correspondent profiles, and -(soon) periodic summaries. Search is the agent-facing read surface -over that layer. +Without `--vault`, search reads the brain working copy at +`/memory/brain/`: every page of the memory vault, mirrored, +plus the pages compiled from it (diary, profiles, topic pages). The +diary is compiled from the diary room, so its text exists only in the +brain. It is also the tree the agent reads as `vault/`. Before the +curator's first run there is no brain, and search reads the memory +vault at `/memory/vault/` instead. A page filed seconds ago +is findable after the curator mirrors it (ADR-011, updated 2026-09-19). This file is a thin argparse + formatter wrapper around `memory.lib.search_memory` — the engine lives in the lib so the @@ -82,9 +85,10 @@ `--count` prints just the integer total. Agents read the default text output directly — JSON would cost tokens for no readable gain. -Before walking the vault, the command compares the local `HEAD` to -the remote `HEAD` via `git ls-remote` and pulls only when they -differ. The fast path (no upstream changes) costs one round-trip; +When it reads the memory vault (a `--vault` override, or no brain +yet), the command first compares the local `HEAD` to the remote `HEAD` +via `git ls-remote` and pulls only when they differ. The brain is +never pulled: the curator commits into that working copy. The fast path (no upstream changes) costs one round-trip; the slow path adds a full fast-forward pull. Pass `--no-refresh` to skip the check entirely — useful for scripting, offline use, or when running against a `--vault` override that isn't a clone. @@ -102,9 +106,11 @@ # from `lib` rather than relying on a package install. sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from lib import ( # noqa: E402 + brain_path_for, keywords_to_regex, refresh_vault_if_stale, search_memory, + split_alternatives, vault_path_for, ) @@ -229,7 +235,7 @@ def _looks_like_a_sentence(query: str) -> bool: # ── output ────────────────────────────────────────────────────────────── -def _format_block(r: dict, link_base: str = "") -> str: +def _format_block(r: dict, link_base: str = "", show_matches: bool = False) -> str: """Render one result as the default human/agent block. `date` is shown as a 10-char placeholder when missing so the @@ -243,6 +249,10 @@ def _format_block(r: dict, link_base: str = "") -> str: corrected title moves it. A page with no capture id (a hand-written wiki entry) gets no line rather than a link that would break quietly. + + `show_matches` adds which of several keywords the page contains, + rarest first. A page that matched only a person's name is rarely + the answer, and the reader should see that without opening it. """ persons = ( "[" + ",".join(r["persons"]) + "]" if r["persons"] else "[]" @@ -254,12 +264,46 @@ def _format_block(r: dict, link_base: str = "") -> str: ] if r["excerpt"]: lines.append(f" …{r['excerpt']}…") + if show_matches and r.get("matched_terms"): + lines.append(f" matches: {', '.join(r['matched_terms'])}") if capture_id := (r.get("capture_id") or "").strip(): if url := public(go_capture(capture_id), link_base): lines.append(f" {url}") return "\n".join(lines) +def resolve_vault(vault_arg: str | None, config) -> "tuple[Path, bool] | dict": + """The tree to search, and whether it is the brain. + + Returns an `{"error": ...}` dict when there is nothing to search. + Shared with `stack memory ask`, which reads the same tree. + """ + searching_brain = False + if vault_arg: + vault = Path(vault_arg).expanduser().resolve() + else: + data_dir = (config or {}).get("data_dir") + if not data_dir: + return { + "error": ( + "no vault available — pass --vault or run inside a " + "configured stack" + ), + } + # The brain is the compiled view: every source page mirrored, + # plus the pages generated from it (diary, profiles, topic + # pages). The diary's text exists nowhere in source, so a + # search of source cannot answer a question the diary answers. + # It is also the tree the agent reads as `vault/`, so a printed + # path is one it can open. Before the curator's first run there + # is no brain yet, and source is all there is. + brain = brain_path_for(Path(data_dir)) + searching_brain = brain.exists() + vault = brain if searching_brain else vault_path_for(Path(data_dir)) + + return vault, searching_brain + + # ── entry point ───────────────────────────────────────────────────────── def run(args, stacklet, config) -> dict | None: @@ -273,24 +317,19 @@ def run(args, stacklet, config) -> dict | None: parser = _parser() ns = parser.parse_args(args) - if ns.vault: - vault = Path(ns.vault).expanduser().resolve() - else: - data_dir = (config or {}).get("data_dir") - if not data_dir: - return { - "error": ( - "no vault available — pass --vault or run inside a " - "configured stack" - ), - } - vault = vault_path_for(Path(data_dir)) + resolved = resolve_vault(ns.vault, config) + if isinstance(resolved, dict): + return resolved + vault, searching_brain = resolved if not vault.exists(): print(f"error: vault not found at {vault}", file=sys.stderr) sys.exit(3) - if not ns.no_refresh: + # The refresh pulls the source clone from Forgejo. The brain working + # copy belongs to the curator, which commits into it; a pull here + # would race its writes. + if not ns.no_refresh and not searching_brain: status = refresh_vault_if_stale(vault) if status == "pulled": print("[memory] vault updated from Forgejo", file=sys.stderr) @@ -336,5 +375,6 @@ def run(args, stacklet, config) -> dict | None: # the same way rather than reading a container's env. home_url = (config or {}).get("home_url", "") link_base = f"{home_url}/go" if home_url else "" - print("\n\n".join(_format_block(r, link_base) for r in results)) + show_matches = len(split_alternatives(query)) > 1 + print("\n\n".join(_format_block(r, link_base, show_matches) for r in results)) return None diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index a31b3e0a..e941ffea 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -30,9 +30,11 @@ from __future__ import annotations import json +import math import re import subprocess import time +import unicodedata from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Callable, List, Optional @@ -1599,26 +1601,97 @@ def extract_summary_callout(text: str) -> str: return "\n".join(captured).strip() -def _excerpt(text: str, query: str, max_len: int = 200) -> str: - """First non-empty body line that mentions `query` (case-insensitive). +def strip_diacritics(text: str) -> str: + """Drop combining marks, so "Käse" and "Kase" are the same word. + + The family writes German; somebody searching from a phone keyboard + or in a hurry types the bare vowel. Byte-literal matching answers + that with nothing at all, which reads like the vault has no such + page rather than like the query was spelled differently. + + Case is deliberately left alone. The query is a regex, and + lower-casing one rewrites its operators: `\\W` becomes `\\w` and + stops meaning the opposite of what it said. Callers match + case-insensitively anyway. + + Decomposing first is what makes both spellings of an umlaut agree: + a precomposed "ü" and a "u" followed by a combining diaeresis are + the same letter to a reader and have to be the same here. + + What this is not is transliteration. "Kaese" stays a different word + from "Käse"; folding those together needs a German-specific table, + not a Unicode normalisation form. + """ + decomposed = unicodedata.normalize("NFD", text) + return "".join(c for c in decomposed if not unicodedata.combining(c)) + + +def _excerpt(text: str, patterns: List["re.Pattern[str]"], max_len: int = 200, + fold_diacritics: bool = True) -> str: + """First non-empty body line that matches a pattern, tried in order. + + `patterns` are the query's alternatives, rarest first, compiled + against folded text. A page found through "Bart|Seepferdchen" shows + the line that says "Seepferdchen": the rare word is the one that tells the + reader why this page came back. The query itself is a regex, so + matching it as a substring (the old way) found nothing for any + query with a `|` in it. Body starts after the *closing* `---` of the frontmatter block -- otherwise hits on `title:` or `persons:` lines would surface as excerpts, which is noisy and misleading. + + Folds the same way the match did, so a page found through "Kase" + still shows the line saying "Käse". A hit whose excerpt came back + empty is a hit the reader cannot judge. """ - needle = query.lower() - body = body_only(text) - for line in body.splitlines(): - stripped = line.strip() - if not stripped: - continue - if needle in stripped.lower(): - if len(stripped) > max_len: - stripped = stripped[:max_len] + "…" - return stripped + lines = [line.strip() for line in body_only(text).splitlines()] + lines = [line for line in lines if line] + for pattern in patterns: + for line in lines: + if pattern.search(strip_diacritics(line) if fold_diacritics else line): + if len(line) > max_len: + line = line[:max_len] + "…" + return line return "" +def split_alternatives(query: str) -> List[str]: + """The top-level alternatives of a regex, e.g. `a|b(c|d)` -> `a`, `b(c|d)`. + + Splits on `|` outside groups and character classes, and never on an + escaped one. A query without a top-level `|` is one alternative. + """ + parts: List[str] = [] + current: List[str] = [] + depth, in_class, escaped = 0, False, False + for ch in query: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif in_class: + in_class = ch != "]" + elif ch == "[": + in_class = True + elif ch == "(": + depth += 1 + elif ch == ")": + depth = max(0, depth - 1) + elif ch == "|" and depth == 0: + parts.append("".join(current)) + current = [] + continue + current.append(ch) + parts.append("".join(current)) + return [part for part in parts if part.strip()] + + +def _idf(doc_count: int, doc_freq: int) -> float: + """BM25's inverse document frequency: rare words weigh more.""" + return math.log(1 + (doc_count - doc_freq + 0.5) / (doc_freq + 0.5)) + + def search_memory( query: str, vault: Path, @@ -1626,8 +1699,10 @@ def search_memory( tags: Optional[List[str]] = None, scopes: Optional[List[str]] = None, limit: int = 20, + fold_diacritics: bool = True, + search_frontmatter: bool = True, ) -> List[dict]: - """Walk the vault, return result dicts sorted newest-first. + """Walk the vault, return result dicts, most relevant first. `query` is a Python regex matched case-insensitively against the *body* of each file -- frontmatter is stripped before matching so @@ -1647,8 +1722,26 @@ def search_memory( can't read personal notes by accident. Returns dicts with keys `path`, `rel`, `title`, `date`, - `persons`, `tags`, `excerpt`. Sorted by frontmatter `date` - descending; files without a date sort to the end. + `persons`, `tags`, `excerpt`, `matched_terms`. Sorted by relevance, + then by frontmatter `date` descending; files without a date sort to + the end. + + Relevance: each top-level alternative of the query (`Bart|Seepferdchen` has + two) scores its inverse document frequency on every page it + matches, summed. A word that is on every page about a child adds + almost nothing; a word on one page decides the order. Before this, + results were sorted by date alone, so the newest pages naming the + child filled the limit and the one page with the rare word never + came back. A query with one alternative scores every hit the same + and keeps the date order. + + `fold_diacritics` strips combining marks from both the query and + the text before matching, so "Kase" reaches "Käse" and the reverse. + `search_frontmatter` matches the title, tag and person *values* + alongside the body, so a page titled "Elternabend" is found by + searching for Elternabend. Both are on by default; passing False + restores the older behaviour, which is what the retrieval lab + measures against. On a missing vault directory or invalid regex, returns `[]` rather than raising -- callers decide how to surface the @@ -1657,10 +1750,19 @@ def search_memory( if not vault.exists(): return [] + folded_query = strip_diacritics(query) if fold_diacritics else query try: - pattern = re.compile(query, re.IGNORECASE) + pattern = re.compile(folded_query, re.IGNORECASE) except re.error: return [] + try: + term_patterns = [re.compile(term, re.IGNORECASE) + for term in split_alternatives(folded_query)] + except re.error: + # An alternative that is not a regex on its own, e.g. a split + # the splitter got wrong. Rank the whole query as one term. + term_patterns = [] + term_patterns = term_patterns or [pattern] persons = persons or [] tags = tags or [] @@ -1676,6 +1778,11 @@ def search_memory( scope_prefixes = [s if s.endswith("/") else f"{s}/" for s in scopes] results: List[dict] = [] + # Corpus statistics for the ranking: pages in scope, and how many + # of them contain each alternative. Counted before the person and + # tag filters, because rarity is a property of the vault. + doc_count = 0 + doc_freq = [0] * len(term_patterns) for md_path in vault.rglob("*.md"): if not md_path.is_file(): continue @@ -1691,17 +1798,46 @@ def search_memory( text = md_path.read_text(encoding="utf-8", errors="ignore") except OSError: continue + doc_count += 1 # Match against body only -- frontmatter field names (`date:`, # `tags:`, `persons:`, ...) would otherwise trivially match # generic keywords and drown real hits. - if not pattern.search(body_only(text)): - continue - + body = body_only(text) fm = _parse_frontmatter(text) doc_persons = _fm_list(fm, "persons") + doc_tags = _fm_list(fm, "tags") + + # The values, never the keys. Stripping frontmatter kept a + # query for "date" from hitting every page through its `date:` + # line, and threw away the title and tags in the process -- + # which are the most descriptive text a page has. Joining the + # values back in restores them without letting the field names + # back through. + # + # Persons stay out. A name says who a page concerns, not what + # it says, so matching it as prose makes any question + # mentioning Lisa match every page Lisa is on. Measured: it + # took one query from four hits to twenty and pushed the + # answer from rank two to rank nine. `--person` is the right + # way to ask that question and it already exists. + haystack = body + if search_frontmatter: + haystack = "\n".join(( + str(fm.get("title") or ""), + " ".join(doc_tags), + body, + )) + if fold_diacritics: + haystack = strip_diacritics(haystack) + if not pattern.search(haystack): + continue + matched = [i for i, term in enumerate(term_patterns) + if term.search(haystack)] + for i in matched: + doc_freq[i] += 1 + if persons and not any(p.lower() in want_persons for p in doc_persons): continue - doc_tags = _fm_list(fm, "tags") if tags: doc_norm = {_norm_tag(t) for t in doc_tags} if not (doc_norm & want_tags): @@ -1714,7 +1850,10 @@ def search_memory( "date": fm.get("date") or "", "persons": doc_persons, "tags": doc_tags, - "excerpt": _excerpt(text, query), + # Filled in after ranking, for the returned hits only. + "excerpt": "", + "_text": text, + "_matched": matched, # The `> [!summary]` callout, stripped of blockquote # prefixes. Drives the synthesis step: feeding summaries # to the LLM is cheaper than feeding bodies and usually @@ -1737,11 +1876,25 @@ def search_memory( "capture_id": fm.get("capture_id") or "", }) + idf = [_idf(doc_count, df) for df in doc_freq] + for r in results: + r["_score"] = round(sum(idf[i] for i in r["_matched"]), 6) results.sort( - key=lambda r: (str(r.get("date") or ""), r["rel"]), + key=lambda r: (r["_score"], str(r.get("date") or ""), r["rel"]), reverse=True, ) - return results[:limit] + results = results[:limit] + + terms = split_alternatives(query) if len(term_patterns) > 1 else [query] + for r in results: + rarest_first = sorted(r.pop("_matched"), key=lambda i: -idf[i]) + r["matched_terms"] = [terms[i] for i in rarest_first + if i < len(terms)] + r["excerpt"] = _excerpt(r.pop("_text"), + [term_patterns[i] for i in rarest_first], + fold_diacritics=fold_diacritics) + del r["_score"] + return results # ─── Natural-language query rewrite ────────────────────────────────────── diff --git a/tests/stacklets/test_memory_search.py b/tests/stacklets/test_memory_search.py index f75d74e8..e6888a81 100644 --- a/tests/stacklets/test_memory_search.py +++ b/tests/stacklets/test_memory_search.py @@ -193,6 +193,166 @@ def test_title_word_in_body_still_matches(self, vault): assert len(results) == 1 +@pytest.fixture +def titled_vault(tmp_path): + """A page whose subject is named only in its title. + + The shared fixture repeats every title as a `# Heading`, so it + cannot tell "found via the title" from "found via the body". Real + archivist pages do the same most of the time, which is why this + gap went unnoticed: it only bites on pages where the title is the + only place the subject is named. + """ + v = tmp_path / "vault" + _write(v / "family/health/termin.md", """ + --- + title: Zahnarzttermin Lisa + date: 2026-09-04 + persons: + - Lisa + tags: + - Topic:Health + --- + + Dienstag um halb vier, Praxis am Marktplatz. Vorher noch anrufen. + """) + return v + + +class TestFrontmatterValues: + """A page titled "Elternabend" is found by searching for Elternabend. + + Frontmatter is stripped before matching so that field *names* do + not match every file in the vault: a query for "date" would + otherwise hit every page via its `date:` line. That rule threw the + *values* out with the keys, and the values are the most + descriptive text a page has. A title is what the archivist chose + to call the page; a tag is what it decided the page is about. + Neither is noise. + + Measured in the retrieval lab: of the questions whose answer sits + in a page title, the body-only engine found none of them. + """ + + def test_a_word_only_in_the_title_finds_the_page(self, titled_vault): + results = search_memory("Zahnarzttermin", titled_vault) + assert len(results) == 1 + assert results[0]["rel"].endswith("termin.md") + + def test_a_tag_value_finds_the_page(self, vault): + """The shared fixture already has a tag that no body repeats.""" + results = search_memory("Cooking", vault) + assert len(results) == 1 + assert results[0]["rel"].endswith("quick-bread.md") + + def test_field_names_still_do_not_match(self, vault): + """The rule this replaces is still enforced. + + Promoting the values must not promote the keys with them, or + every page comes back for "date" exactly as before. + """ + assert search_memory("date", vault) == [] + assert search_memory("tags", vault) == [] + assert search_memory("title", vault) == [] + assert search_memory("persons", vault) == [] + + def test_a_title_only_hit_carries_no_invented_excerpt(self, titled_vault): + """The excerpt quotes the body, so a title hit has none to show. + + Better an empty excerpt than a line lifted from somewhere the + query never matched. + """ + results = search_memory("Zahnarzttermin", titled_vault) + assert results[0]["excerpt"] == "" + + def test_the_body_only_behaviour_is_still_reachable(self, titled_vault): + assert search_memory( + "Zahnarzttermin", titled_vault, search_frontmatter=False) == [] + + +# ─── Diacritics ────────────────────────────────────────────────────────── + +@pytest.fixture +def umlaut_vault(tmp_path): + """A vault written the way a German family writes: with umlauts.""" + v = tmp_path / "vault" + _write(v / "family/groceries/einkauf.md", """ + --- + title: Einkaufsliste + date: 2026-09-10 + persons: + - Marge + --- + + # Einkaufsliste + + Käse und Öl nachkaufen, die Tür klemmt auch wieder. + """) + return v + + +class TestDiacritics: + """An umlaut typed one way still finds it written the other. + + The family writes "Käse"; somebody searching from a phone, an + English keyboard, or in a hurry types "Kase". Byte-literal matching + returns *nothing at all* for that, which is the worst kind of + search failure: not a bad result, an empty one that reads like the + vault has no such page. + + Folding runs on both sides, so it does not matter which side + carries the umlaut. What it deliberately does not do is fold "ue" + into "ü" -- that is transliteration, not a diacritic, and it needs + a German-specific rule rather than a Unicode one. + """ + + def test_a_query_without_the_umlaut_finds_the_page_with_it( + self, umlaut_vault): + results = search_memory("Kase", umlaut_vault) + assert len(results) == 1 + assert results[0]["rel"].endswith("einkauf.md") + + def test_a_query_with_the_umlaut_finds_it_too(self, umlaut_vault): + assert len(search_memory("Käse", umlaut_vault)) == 1 + + def test_it_works_for_every_umlaut_not_just_a(self, umlaut_vault): + assert len(search_memory("Ol", umlaut_vault)) == 1 + assert len(search_memory("Tur", umlaut_vault)) == 1 + + def test_the_excerpt_still_shows_the_line_that_matched( + self, umlaut_vault): + """A hit with no excerpt is a hit the reader cannot judge.""" + results = search_memory("Kase", umlaut_vault) + assert "Käse" in results[0]["excerpt"] + + def test_a_spelled_out_umlaut_is_not_folded(self, umlaut_vault): + """"Kaese" is a different word to Unicode, and stays one. + + Worth pinning rather than leaving implicit: this is the gap + measured in the retrieval lab, and closing it would take a + German transliteration table, not a normalisation form. + """ + assert search_memory("Kaese", umlaut_vault) == [] + + def test_regex_syntax_survives_folding(self, umlaut_vault): + """The query is a regex, so folding must not rewrite its operators. + + Case-folding the pattern would turn `\\W` into `\\w` and invert + what it means. Here `\\W` has to keep matching the space after + "Käse"; if it had become `\\w` this finds nothing. + """ + assert len(search_memory(r"Kase\Wund", umlaut_vault)) == 1 + assert len(search_memory(r"K.se", umlaut_vault)) == 1 + + def test_the_old_byte_literal_behaviour_is_still_reachable( + self, umlaut_vault): + """So a before-and-after stays measurable, per the handover.""" + assert search_memory( + "Kase", umlaut_vault, fold_diacritics=False) == [] + assert len(search_memory( + "Käse", umlaut_vault, fold_diacritics=False)) == 1 + + # ─── Filters ───────────────────────────────────────────────────────────── class TestFilters: @@ -363,6 +523,106 @@ def test_results_ordered_by_date_desc(self, stack_cli, vault): assert dates == sorted(dates, reverse=True) +# ─── Ranking ───────────────────────────────────────────────────────────── + +@pytest.fixture +def child_vault(tmp_path): + """One old diary entry with a rare word, many newer pages naming the child. + + The shape of a real family vault: a child's name is on dozens of + pages, and the one fact asked about is on one of them, months ago. + """ + v = tmp_path / "vault" + _write(v / "family/diary/2026/06.md", """ + --- + title: Tagebuch Juni + date: 2026-06-30 + persons: + - Homer + --- + + Bart war heute im Zoo. + Bart hat heute sein Seepferdchen geschafft. + """) + for day in range(10, 18): + _write(v / f"family/notes/2026-09-{day}-garten.md", f""" + --- + title: Garten {day} + date: 2026-09-{day} + persons: + - Homer + --- + + Bart war heute im Garten. + """) + return v + + +class TestRanking: + """Rare words decide the order, then the date. + + Sorted by date alone, "Bart|Seepferdchen" returned the five newest + pages naming Bart, and the one page with "Seepferdchen" never came + back. + """ + + def test_the_page_with_the_rare_word_comes_first(self, child_vault): + results = search_memory("Bart|Seepferdchen", child_vault, limit=5) + assert results[0]["rel"] == "family/diary/2026/06.md" + + def test_rare_word_outranks_a_newer_page_with_only_the_common_one( + self, child_vault): + results = search_memory("Bart|geschafft|Seepferdchen", child_vault, limit=3) + assert results[0]["rel"] == "family/diary/2026/06.md" + + def test_the_excerpt_shows_the_line_with_the_rare_word(self, child_vault): + """Not the first line naming the child, which says nothing.""" + results = search_memory("Bart|Seepferdchen", child_vault, limit=1) + assert "Seepferdchen" in results[0]["excerpt"] + + def test_one_word_keeps_the_date_order(self, child_vault): + results = search_memory("Bart", child_vault, limit=3) + dates = [r["date"] for r in results] + assert dates == ["2026-09-17", "2026-09-16", "2026-09-15"] + + def test_a_group_is_one_alternative(self, child_vault): + """`(Seepferdchen|Zoo)` is one term, not two, so it ranks as one.""" + results = search_memory("Bart|(Seepferdchen|Zoo)", child_vault, limit=1) + assert results[0]["rel"] == "family/diary/2026/06.md" + + def test_an_escaped_bar_is_a_literal_not_a_split(self, tmp_path): + """`a\\|b` asks for the text "a|b", so it must stay one term.""" + v = tmp_path / "vault" + _write(v / "family/notes/pipe.md", """ + --- + title: Pipe + date: 2026-01-01 + --- + + Der Befehl lautet ls|wc und sonst nichts. + """) + assert len(search_memory(r"ls\|wc", v)) == 1 + assert search_memory(r"ls\|xx", v) == [] + + +class TestMatchesLine: + def test_multi_keyword_output_names_the_matches(self, stack_cli, child_vault): + code, out, _ = stack_cli( + "memory", "search", "Bart|Seepferdchen", "--limit", "1", + "--vault", str(child_vault), "--no-refresh", + ) + assert code == 0 + assert "matches: Seepferdchen, Bart" in out + + def test_one_keyword_has_no_matches_line(self, stack_cli, child_vault): + code, out, _ = stack_cli( + "memory", "search", "Seepferdchen", + "--vault", str(child_vault), "--no-refresh", + ) + assert code == 0 + assert "matches:" not in out + + # ─── In-process engine (archivist call path) ───────────────────────────── class TestSearchMemoryLib: @@ -381,7 +641,7 @@ def test_returns_result_dicts_with_expected_keys(self, vault): assert set(r.keys()) == { "path", "rel", "title", "date", "persons", "tags", "excerpt", "summary", - "paperless_id", "capture_id", + "paperless_id", "capture_id", "matched_terms", } assert r["rel"].endswith("radlager.md") assert r["persons"] == ["Homer"] @@ -573,3 +833,102 @@ def test_scope_flag_repeats_or_within_axis(self, stack_cli, vault): lines = [ln for ln in out.strip().splitlines() if ln] assert not any(ln.startswith("homer/") for ln in lines) assert len(lines) == 3 + + +# ─── Which tree a configured search reads ──────────────────────────────── + +def _search_cli(): + """The real `stack memory search` module, loaded by path like the CLI + loader does, so `run()` can be driven with a configured data dir.""" + import importlib.machinery + import importlib.util + path = Path(__file__).resolve().parents[2] / "stacklets/memory/cli/search.py" + loader = importlib.machinery.SourceFileLoader("memory_cli_search_tree", str(path)) + spec = importlib.util.spec_from_file_location(loader.name, path, loader=loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +class TestSearchReadsTheBrain: + """A configured search reads the brain, the compiled view of the vault. + + Generated pages exist only there: the diary is compiled from the + diary room, and nothing in the source vault holds its text. Search + read the source vault, so a question answered on a diary page came + back with an UNO manual and school emails, and the agent fell back + to grep. The brain is also the tree the agent reads as `vault/`, so + a printed path is one it can open. + """ + + def _data_dir(self, tmp_path): + source = tmp_path / "memory" / "vault" + brain = tmp_path / "memory" / "brain" + _write(source / "family/notes/garten.md", """ + --- + title: Garten + date: 2026-09-17 + --- + + Bart war heute im Garten. + """) + _write(brain / "family/notes/garten.md", """ + --- + title: Garten + date: 2026-09-17 + --- + + Bart war heute im Garten. + """) + _write(brain / "family/diary/2026/06.md", """ + --- + title: Juni + generated: true + --- + + Bart hat heute sein Seepferdchen geschafft. + """) + return tmp_path + + def test_a_page_that_exists_only_in_the_brain_is_found(self, tmp_path, capsys): + data_dir = self._data_dir(tmp_path) + + _search_cli().run(["Seepferdchen", "--paths", "--no-refresh"], None, + {"data_dir": str(data_dir)}) + + assert capsys.readouterr().out.split() == ["family/diary/2026/06.md"] + + def test_a_page_in_both_trees_is_listed_once(self, tmp_path, capsys): + data_dir = self._data_dir(tmp_path) + + _search_cli().run(["Garten", "--paths", "--no-refresh"], None, + {"data_dir": str(data_dir)}) + + assert capsys.readouterr().out.split() == ["family/notes/garten.md"] + + def test_an_explicit_vault_still_wins(self, tmp_path, capsys): + data_dir = self._data_dir(tmp_path) + source = data_dir / "memory" / "vault" + + with pytest.raises(SystemExit) as exit_: + _search_cli().run(["Seepferdchen", "--paths", "--vault", str(source)], None, + {"data_dir": str(data_dir)}) + + assert exit_.value.code == 1, "the source vault has no diary page" + + def test_before_the_first_curator_run_the_source_vault_is_searched( + self, tmp_path, capsys): + """A fresh install has no brain yet. Search still answers.""" + _write(tmp_path / "memory/vault/family/notes/garten.md", """ + --- + title: Garten + date: 2026-09-17 + --- + + Bart war heute im Garten. + """) + + _search_cli().run(["Garten", "--paths", "--no-refresh"], None, + {"data_dir": str(tmp_path)}) + + assert capsys.readouterr().out.split() == ["family/notes/garten.md"] From fed043a056dd778818dc6f7ab2b3bfd41f6a0c71 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 13:41:29 +0200 Subject: [PATCH 05/10] feat(agent): memory_search ranks by person and logs its results - person becomes a ranking keyword, not a --person filter; a parent's diary entry about a child does not list the child - stderr block per search: query, regex, count, path and matched keywords per hit; no page text - description matches what the output contains --- stacklets/agent/runtime/memory_tool.py | 52 +++++++++++++-- tests/stacklets/test_agent_vault_tools.py | 81 +++++++++++++++++++++++ 2 files changed, 127 insertions(+), 6 deletions(-) diff --git a/stacklets/agent/runtime/memory_tool.py b/stacklets/agent/runtime/memory_tool.py index 8f5d5d2b..05594cef 100644 --- a/stacklets/agent/runtime/memory_tool.py +++ b/stacklets/agent/runtime/memory_tool.py @@ -4,6 +4,7 @@ import asyncio import re +import sys from nanobot.agent.tools.base import Tool, tool_parameters from nanobot.agent.tools.schema import ( @@ -40,7 +41,9 @@ nullable=True, ), person=StringSchema( - "Optional person filter, such as lisa or homer.", + "Optional person the question is about, such as lisa. Added as " + "a keyword: it ranks pages naming them higher, it does not " + "hide pages that do not list them.", nullable=True, ), tag=StringSchema( @@ -61,9 +64,12 @@ def name(self) -> str: @property def description(self) -> str: return ( - "Search the family memory vault. Results include rank, score, vault path, " - "snippet, and source links when available. Use before answering factual " - "questions about family people, plans, documents, notes, bookmarks, or topics." + "Search the family memory vault by keywords. Most relevant first: a page " + "with a rare keyword ranks above pages with only common ones. Each result " + "has date, persons, vault path, title, the matching line, which keywords " + "matched, and a source link for captured notes. Use before answering " + "factual questions about family people, plans, documents, notes, " + "bookmarks, or topics." ) @property @@ -108,7 +114,15 @@ async def _search_one( # which made a second LLM call inside every multi-word search # (measured 2026-09-15: one full model call per search, on the # same GPU as the turn). The model now supplies the keywords. - words = [re.escape(w) for w in query.split()] + words = query.split() + # The person is a keyword, not a filter. `--person` drops every + # page whose frontmatter does not list them, and a diary entry + # written by a parent about a child usually lists the parent. + # As a keyword, the name ranks their pages up, and the ranking + # weighs it low because it is on many pages. + if person and person.lower() not in {w.lower() for w in words}: + words.append(person) + words = [re.escape(w) for w in words] pattern = "|".join(words) if len(words) > 1 else query args = [ "stack", @@ -118,7 +132,7 @@ async def _search_one( "--limit", str(limit or 5), ] - for flag, value in (("--scope", scope), ("--person", person), ("--tag", tag)): + for flag, value in (("--scope", scope), ("--tag", tag)): if value: args.extend([flag, value]) @@ -134,6 +148,7 @@ async def _search_one( # answer. Only 2 and up (bad arguments, unreadable vault) are # failures. Reporting an empty result as a failure tells the model # to try again when the honest reply is that there is nothing there. + _log_search(query, pattern, proc.returncode, out) if proc.returncode not in (0, 1): return f"Error: memory search failed with exit {proc.returncode}: {err or out}" # The status decides, not the text. A search that matched nothing @@ -145,6 +160,31 @@ async def _search_one( return out or "(no memory results)" +def _log_search(query: str, pattern: str, returncode: int, out: str) -> None: + """Write the query and the ranked result paths to the container log. + + nanobot logs the tool call but not its result, so a search that + missed the page looked the same as a model that ignored it. Paths + and matched keywords only: the excerpt is page text and stays out. + """ + if returncode not in (0, 1): + outcome, hits = f"error exit {returncode}", [] + else: + # One block per result; its first line ends with the vault path. + blocks = [b for b in out.split("\n\n") if b.strip()] if returncode == 0 else [] + hits = [] + for block in blocks: + lines = block.strip().splitlines() + path = lines[0].split()[-1] + matches = next((ln.strip() for ln in lines[1:] + if ln.strip().startswith("matches:")), "") + hits.append(f"{path} {matches}".rstrip()) + outcome = f"{len(hits)} results" + lines = [f"[memory_search] query={query!r} pattern={pattern!r} -> {outcome}"] + lines += [f" {i}. {hit}" for i, hit in enumerate(hits, 1)] + print("\n".join(lines), file=sys.stderr, flush=True) + + def install() -> None: """Append MemorySearchTool to nanobot discovery without forking nanobot.""" from nanobot.agent.tools.loader import ToolLoader diff --git a/tests/stacklets/test_agent_vault_tools.py b/tests/stacklets/test_agent_vault_tools.py index 04ec666b..2840b776 100644 --- a/tests/stacklets/test_agent_vault_tools.py +++ b/tests/stacklets/test_agent_vault_tools.py @@ -288,6 +288,87 @@ def test_results_are_passed_through_verbatim(self, vault_tools): assert answer == block.decode().strip() +class TestEverySearchIsLogged: + """Each search leaves its query and ranked result paths in the log. + + The agent log records the tool call but not what came back, so a + search that missed the page could not be told apart from a model + that ignored it. The log names paths and matched keywords only, + never the excerpt, so page text stays out of it. + """ + + RESULTS = ( + b"2026-06-30 [Homer] family/diary/2026/06.md\n" + b" Tagebuch Juni\n" + b" \xe2\x80\xa6Bart hat heute sein Seepferdchen geschafft.\xe2\x80\xa6\n" + b" matches: Seepferdchen, Bart\n" + b"\n" + b"2026-09-17 [Homer] family/notes/garten.md\n" + b" Garten\n" + b" \xe2\x80\xa6Bart war heute im Garten.\xe2\x80\xa6\n" + b" matches: Bart\n" + ) + + def test_the_query_and_ranked_paths_are_logged(self, vault_tools, capsys): + result_of(vault_tools["memory_search"], returncode=0, + stdout=self.RESULTS, query="Bart Seepferdchen") + log = capsys.readouterr().err + + assert "query='Bart Seepferdchen'" in log + assert "pattern='Bart|Seepferdchen'" in log + assert "-> 2 results" in log + first = log.index("1. family/diary/2026/06.md") + second = log.index("2. family/notes/garten.md") + assert first < second + assert "matches: Seepferdchen, Bart" in log + + def test_page_text_stays_out_of_the_log(self, vault_tools, capsys): + result_of(vault_tools["memory_search"], returncode=0, + stdout=self.RESULTS, query="Bart Seepferdchen") + log = capsys.readouterr().err + + assert "Seepferdchen geschafft" not in log + assert "Tagebuch Juni" not in log + + def test_no_results_are_logged_as_such(self, vault_tools, capsys): + result_of(vault_tools["memory_search"], returncode=1, + stdout=b"", query="school run") + + assert "-> 0 results" in capsys.readouterr().err + + def test_a_failure_is_logged_with_its_exit_code(self, vault_tools, capsys): + result_of(vault_tools["memory_search"], returncode=2, + stderr=b"unrecognized arguments", query="school run") + + assert "-> error exit 2" in capsys.readouterr().err + + +def test_naming_a_person_does_not_hide_pages_that_do_not_list_them( + memory_cli, vault_tools, tmp_path, capsys): + """The model passes the person a question is about; that must not + filter out the answer. + + A diary entry about a child is written by a parent, so its + `persons:` lists the parent. Sent as `--person`, the child's name + hides that entry from a question about the child. + """ + vault = tmp_path / "vault" + page = vault / "family" / "diary" / "2026" / "06.md" + page.parent.mkdir(parents=True) + page.write_text( + "---\ntitle: Tagebuch Juni\ndate: 2026-06-30\npersons:\n - Homer\n---\n\n" + "Bart hat heute sein Seepferdchen geschafft.\n", encoding="utf-8") + + argv = argv_of(vault_tools["memory_search"], query="Seepferdchen", person="bart") + try: + memory_cli["search"].run( + argv[3:] + ["--vault", str(vault), "--no-refresh"], None, None) + except SystemExit as exit_: + pytest.fail(f"search found nothing (exit {exit_.code}) for `{' '.join(argv)}`") + + assert "family/diary/2026/06.md" in capsys.readouterr().out + + # ── gate 3: the transport between the tool and the CLI ─────────────── # # The two gates above both read argv straight out of the tool. Nothing From 64495825bfc86476e5098e3de5e3072e1193947c Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 13:41:29 +0200 Subject: [PATCH 06/10] feat(memory): add stack memory ask Same shape as stack web ask: question in, short answer with [N] citations and sources out. One model call; the search uses the question's own words, because the --nl rewrite adds ontology words that outrank the answering page. - host: search the brain, send top hits as JSON to the bot-runner - bot-runner: `answer` command, 600-token cap - --json, --sources N; timing line on stderr - sources print when the model is unreachable (exit 1) --- stacklets/memory/bot/cli/answer.py | 98 ++++++++++ stacklets/memory/bot/cli_entrypoint.py | 3 +- stacklets/memory/cli/_common.py | 7 +- stacklets/memory/cli/ask.py | 172 +++++++++++++++++ tests/stacklets/test_memory_ask.py | 255 +++++++++++++++++++++++++ 5 files changed, 533 insertions(+), 2 deletions(-) create mode 100644 stacklets/memory/bot/cli/answer.py create mode 100644 stacklets/memory/cli/ask.py create mode 100644 tests/stacklets/test_memory_ask.py diff --git a/stacklets/memory/bot/cli/answer.py b/stacklets/memory/bot/cli/answer.py new file mode 100644 index 00000000..03751989 --- /dev/null +++ b/stacklets/memory/bot/cli/answer.py @@ -0,0 +1,98 @@ +"""memory answer — answer a question from search hits, run where the model lives. + +`stack memory ask` does the keyword rewrite and the search on the +host, then sends the question and the top hits here as JSON on stdin: + + {"question": "...", "evidence": [{"path", "title", "date", + "excerpt", "summary"}, ...]} + +The answer goes to stdout with `[N]` citations that number the +evidence in the order given. Exit 0 with an answer, exit 1 without one +(bad payload, model unavailable, empty reply); the host then shows the +search results instead. + +The prompt follows the archivist's recall synthesis +(`stacklets/docs/bot/pipeline.py`, `_build_synthesize_prompt`). It is +kept here rather than imported, because that module belongs to the docs +stacklet and pulls in its whole pipeline. +""" + +from __future__ import annotations + +import json +import os +import sys +from datetime import date +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from stack.ai.client import LLM + +HELP = "Answer a question from search hits (JSON on stdin)" + + +def build_prompt(question: str, evidence: list[dict], language: str, today: str) -> str: + """The question, the numbered hits, and the rules for citing them.""" + items = [] + for n, ev in enumerate(evidence, 1): + head = f"[{n}] {ev.get('path', '')}" + if ev.get("title"): + head += f" | {ev['title']}" + if ev.get("date"): + head += f" | {ev['date']}" + lines = [head] + if ev.get("summary"): + lines.append(f" Summary: {ev['summary']}") + if ev.get("excerpt"): + lines.append(f" Matching line: {ev['excerpt']}") + items.append("\n".join(lines)) + evidence_block = "\n\n".join(items) + return f"""You are answering a family member's question from their own notes. Answer using ONLY the evidence below. Never invent facts. + +Today's date is {today}. Use it to resolve relative-time phrases. + +Question: {question} + +Evidence (numbered; cite the ones you used as [N]): + +{evidence_block} + +Rules: +- Answer in one or two sentences. +- Cite every fact: "[1]", "[2, 3]". +- If the evidence does not answer the question, say so in one sentence and name the closest hit. +- Respond in the family's language: {language}. +- No preamble. Answer directly. + +Answer:""" + + +async def run(llm: "LLM", argv: list[str]) -> int: + """Entry point the dispatcher calls with the shared LLM client.""" + try: + payload = json.loads(sys.stdin.read()) + question = str(payload["question"]) + evidence = list(payload["evidence"]) + except (ValueError, KeyError, TypeError) as e: + print(f"answer: bad payload on stdin: {e}", file=sys.stderr) + return 1 + + prompt = build_prompt(question, evidence, + language=os.environ.get("LANGUAGE", "en"), + today=date.today().isoformat()) + try: + # Capped like `stack web ask`: a valid answer is a few sentences, + # and a model caught in a repetition loop would otherwise run + # until the client timeout. + raw = await llm.complete("recall", prompt, json_mode=False, + temperature=0.0, max_tokens=600) + except Exception as e: # transport errors: the host falls back to results + print(f"answer: model unavailable: {e}", file=sys.stderr) + return 1 + + answer = (raw or "").strip() + if not answer: + print("answer: the model returned nothing", file=sys.stderr) + return 1 + print(answer) + return 0 diff --git a/stacklets/memory/bot/cli_entrypoint.py b/stacklets/memory/bot/cli_entrypoint.py index e76a4bc1..ff5af7a0 100644 --- a/stacklets/memory/bot/cli_entrypoint.py +++ b/stacklets/memory/bot/cli_entrypoint.py @@ -50,10 +50,11 @@ from stack.ai.client import LLM, LLMUnavailableError -from cli import diary, rewrite, wiki +from cli import answer, diary, rewrite, wiki _HANDLERS = { + "answer": answer.run, "diary": diary.run, "rewrite": rewrite.run, "wiki": wiki.run, diff --git a/stacklets/memory/cli/_common.py b/stacklets/memory/cli/_common.py index 1e227484..c103df85 100644 --- a/stacklets/memory/cli/_common.py +++ b/stacklets/memory/cli/_common.py @@ -67,7 +67,8 @@ def dispatch(command: str, *argv: str) -> dict: def dispatch_capture(command: str, *argv: str, - timeout: int = 60) -> tuple[int, str, str]: + timeout: int = 60, + input_text: str | None = None) -> tuple[int, str, str]: """The same hop, for a caller that wants the output as a value. `dispatch` is right when the container's output *is* the result: @@ -88,6 +89,9 @@ def dispatch_capture(command: str, *argv: str, version-skewed host (updated code, bot-runner not restarted yet) would dump that over a family's search results. One line stays diagnostic without ever becoming a wall. + + `input_text` goes to the command's stdin, for a payload too large or + too structured for argv (the evidence `answer` reads). """ if not _bot_runner_running(): return 1, "", f"{BOT_RUNNER_CONTAINER} is not running" @@ -100,6 +104,7 @@ def dispatch_capture(command: str, *argv: str, try: result = subprocess.run( cmd, capture_output=True, text=True, timeout=timeout, + input=input_text, ) except FileNotFoundError: return 1, "", "docker CLI not found on this host" diff --git a/stacklets/memory/cli/ask.py b/stacklets/memory/cli/ask.py new file mode 100644 index 00000000..2f77d31c --- /dev/null +++ b/stacklets/memory/cli/ask.py @@ -0,0 +1,172 @@ +"""stack memory ask — a question answered from the family brain. + +The same shape as `stack web ask`: a plain question, a short answer, +its sources. One model call: the search uses the question's own words +(question words and fillers dropped), and the model answers from the +top hits. No tool loop deciding what to read next. + +No model rewrites the question first. The `--nl` rewrite adds +category words from the family's ontology, and rare words weigh most in +the ranking, so a document that only shares those categories can beat +the page that answers. The question's own words avoid that. + + stack memory ask "Wann hat Bart sein Seepferdchen geschafft?" + + Am 12. Juni 2026 [1]. + + Sources + [1] April + family/diary/2026/04.md + +The sources print whether or not the model was reached, so the family +can read the pages themselves. A timing line on stderr says where the +time went, which is what this command is for: comparing a fixed +route with the agent's loop. + + stack memory ask "..." --json for an agent + stack memory ask "..." --sources 3 fewer hits for the model + +Search reads the brain (see `stack memory search`). Both model calls +run in the bot-runner, because the host CLI is stdlib-only. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from lib import refresh_vault_if_stale, search_memory # noqa: E402 +from _common import dispatch_capture # noqa: E402 +from search import resolve_vault # noqa: E402 + +HELP = "Ask a question and get an answer with sources" + +DEFAULT_SOURCES = 5 + +# Words that carry the grammar of a question, not its subject. Dropped +# so they neither match every page nor crowd the excerpt. Common words +# that survive this list ("bekommen") weigh little in the ranking. +_FILLERS = frozenset(""" +wann wer wen wem wessen was wie wo wohin woher wieso warum weshalb welche +welcher welches welchen welchem hat haben hatte hatten ist sind war waren +wird werden wurde wurden der die das den dem des ein eine einen einem einer +eines und oder aber mit von vom zu zum zur im in am an auf aus für bei nach +sein seine seinen seinem seiner seines ihr ihre ihren ihrem ihrer ihres mein +meine meinen unser unsere unseren es er sie wir ich du uns euch noch schon +denn doch mal bitte gibt gab kann können konnte soll sollen muss müssen +when who whom whose what how where why which did does do has have had is +are was were be been the a an and or of to in on at for from with by his +her hers their my our its it he she we i you us me get got please can could +should would will +""".split()) + + +def question_words(question: str) -> list[str]: + """The words of a question that say what it is about, in order, once each.""" + seen: dict[str, str] = {} + for word in re.findall(r"\w+", question): + key = word.lower() + if len(word) > 1 and key not in _FILLERS and key not in seen: + seen[key] = word + return list(seen.values()) + + +def _parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="stack memory ask", + description="Answer a question from the family brain, with sources.", + ) + p.add_argument("question", help="the question, in words") + p.add_argument("--json", action="store_true", + help="print the answer and sources as JSON") + p.add_argument("--sources", type=int, default=DEFAULT_SOURCES, + help=f"hits handed to the model (default {DEFAULT_SOURCES})") + p.add_argument("--vault", default=None, + help="search this directory instead of the brain") + p.add_argument("--no-refresh", action="store_true", + help="skip the upstream check when reading the memory vault") + return p + + +def _render_sources(results: list[dict]) -> str: + return "\n".join( + f" [{n}] {r['title']}\n {r['rel']}" for n, r in enumerate(results, 1) + ) + + +def run(args, stacklet, config) -> dict | None: + ns = _parser().parse_args(args) + + resolved = resolve_vault(ns.vault, config) + if isinstance(resolved, dict): + return resolved + vault, searching_brain = resolved + if not vault.exists(): + print(f"error: vault not found at {vault}", file=sys.stderr) + sys.exit(3) + if not ns.no_refresh and not searching_brain: + refresh_vault_if_stale(vault) + + started = time.monotonic() + keywords = question_words(ns.question) + query = "|".join(re.escape(w) for w in keywords) or re.escape(ns.question) + results = search_memory(query, vault, limit=max(ns.sources, 1)) + searched = time.monotonic() + + if not results: + print(f'Search returned nothing for "{ns.question}".', file=sys.stderr) + sys.exit(1) + + evidence = [ + { + "path": r["rel"], + "title": r["title"], + "date": str(r.get("date") or ""), + "excerpt": r.get("excerpt") or "", + "summary": r.get("summary") or "", + } + for r in results + ] + payload = json.dumps({"question": ns.question, "evidence": evidence}, + ensure_ascii=False) + rc, out, reason = dispatch_capture("answer", timeout=180, input_text=payload) + answered = time.monotonic() + answer = out.strip() if rc == 0 else "" + + print( + f"[memory] searched for: {', '.join(keywords) or ns.question}; " + f"search {searched - started:.1f}s, answer {answered - searched:.1f}s", + file=sys.stderr, + ) + + if not answer: + # The search worked; only the answering failed. The pages are + # still worth printing -- the family can read them itself. + detail = f" ({reason})" if reason else "" + print(f"\n Couldn't reach the model{detail}. What the search found:\n") + print(_render_sources(results)) + print() + sys.exit(1) + + if ns.json: + print(json.dumps({ + "question": ns.question, + "answer": answer, + "sources": [{"n": n, "title": r["title"], "path": r["rel"]} + for n, r in enumerate(results, 1)], + }, indent=2, ensure_ascii=False)) + return None + + print() + print(answer) + print() + print(" Sources") + print(_render_sources(results)) + print() + return None diff --git a/tests/stacklets/test_memory_ask.py b/tests/stacklets/test_memory_ask.py new file mode 100644 index 00000000..52071034 --- /dev/null +++ b/tests/stacklets/test_memory_ask.py @@ -0,0 +1,255 @@ +"""`stack memory ask` — a question answered from the brain, like `stack web ask`. + +The agent answers a vault question in a tool loop: search, read, +check, read again. Each step is a model call with a prefill. This +route is fixed at two model calls: one for the keywords (the `--nl` +rewrite), one for the answer from the top hits. It reports its own +timing so it can be compared with the agent. + +Its surface matches `stack web ask`: the answer, then the sources, +and the sources print even when the model could not be reached. + +The model lives in the bot-runner container. The host tests stand in +for that hop with a fake `dispatch_capture`; everything on the host +side is the real code. The container side is tested with a stub LLM, +the only external boundary it has. +""" + +from __future__ import annotations + +import asyncio +import io +import json +import sys +import textwrap +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory" / "cli")) + +import ask # noqa: E402 + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(text).lstrip("\n")) + + +@pytest.fixture +def data_dir(tmp_path): + """A brain with the diary page that holds the answer, and noise.""" + brain = tmp_path / "memory" / "brain" + _write(brain / "family/diary/2026/06.md", """ + --- + title: Juni + generated: true + --- + + ## 12. Juni + Bart hat heute sein Seepferdchen geschafft. + """) + _write(brain / "family/notes/garten.md", """ + --- + title: Garten + date: 2026-09-17 + --- + + Bart war heute im Garten. + """) + return tmp_path + + +@pytest.fixture +def container(monkeypatch): + """Fake bot-runner: keywords for `rewrite`, an answer for `answer`. + + Records every hop with the stdin it was given, so a test can check + what the model would have been shown. + """ + state = { + "keywords": "Bart\nSeepferdchen\n", + "answer": (0, "Am 12. Juni 2026 [1].\n", ""), + "calls": [], + } + + def fake_dispatch(command, *argv, timeout=60, input_text=None): + state["calls"].append({"command": command, "argv": argv, "input": input_text}) + if command == "rewrite": + return 0, state["keywords"], "" + if command == "answer": + return state["answer"] + return 2, "", f"unknown command {command}" + + monkeypatch.setattr(ask, "dispatch_capture", fake_dispatch) + return state + + +def _run(data_dir, *args): + return ask.run( + ["Wann hat Bart sein Seepferdchen geschafft?", "--no-refresh", *args], + None, {"data_dir": str(data_dir)}, + ) + + +class TestAsk: + def test_prints_the_answer_then_its_sources(self, data_dir, container, capsys): + _run(data_dir) + out = capsys.readouterr().out + + assert out.lstrip().startswith("Am 12. Juni 2026 [1].") + assert " Sources" in out + assert " [1] Juni\n family/diary/2026/06.md" in out + + def test_json_for_an_agent(self, data_dir, container, capsys): + _run(data_dir, "--json") + payload = json.loads(capsys.readouterr().out) + + assert payload["answer"] == "Am 12. Juni 2026 [1]." + assert payload["sources"][0] == { + "n": 1, "title": "Juni", "path": "family/diary/2026/06.md"} + + def test_sources_limits_the_hits(self, data_dir, container, capsys): + _run(data_dir, "--sources", "1", "--json") + + assert len(json.loads(capsys.readouterr().out)["sources"]) == 1 + + def test_the_model_is_shown_the_question_and_the_line_that_answers_it( + self, data_dir, container): + _run(data_dir) + hop = next(c for c in container["calls"] if c["command"] == "answer") + payload = json.loads(hop["input"]) + + assert payload["question"] == "Wann hat Bart sein Seepferdchen geschafft?" + first = payload["evidence"][0] + assert first["path"] == "family/diary/2026/06.md" + assert "Seepferdchen" in first["excerpt"] + + def test_reports_where_the_time_went(self, data_dir, container, capsys): + _run(data_dir) + err = capsys.readouterr().err + + assert "search" in err and "answer" in err + + def test_nothing_found_means_no_answer_call(self, data_dir, container, capsys): + with pytest.raises(SystemExit) as exit_: + ask.run(["Wann war der Einhornausflug?", "--no-refresh"], None, + {"data_dir": str(data_dir)}) + + assert exit_.value.code == 1 + assert "Search returned nothing" in capsys.readouterr().err + assert all(c["command"] != "answer" for c in container["calls"]) + + def test_without_a_model_the_sources_are_still_shown( + self, data_dir, container, capsys): + container["answer"] = (1, "", "stack-core-bot-runner is not running") + + with pytest.raises(SystemExit) as exit_: + _run(data_dir) + out = capsys.readouterr().out + + assert exit_.value.code == 1 + assert "Couldn't reach the model (stack-core-bot-runner is not running)" in out + assert "family/diary/2026/06.md" in out + + +class TestAskSearchesTheQuestionsOwnWords: + """The question's words, not a model's rewrite of them. + + A rewrite that adds category words ("Rechnung, Schwimmkurs") to a + question about a swimming badge lets a receipt beat the diary + entry: rare words weigh most in the ranking. The question's own + words find the entry. + """ + + def test_no_rewrite_call_is_made(self, data_dir, container): + _run(data_dir) + + assert all(c["command"] != "rewrite" for c in container["calls"]) + + def test_a_page_with_added_category_words_does_not_win( + self, data_dir, container): + _write(data_dir / "memory/brain/family/documents/rechnung-schwimmkurs.md", """ + --- + title: Rechnung Schwimmkurs Bart + date: 2026-03-02 + --- + + Rechnung für den Schwimmkurs von Bart, Hallenbad Springfield. + """) + container["keywords"] = "Bart\nSeepferdchen\nRechnung\nSchwimmkurs\n" + + _run(data_dir) + hop = next(c for c in container["calls"] if c["command"] == "answer") + + assert json.loads(hop["input"])["evidence"][0]["path"] == "family/diary/2026/06.md" + + def test_question_words_and_fillers_are_not_searched( + self, data_dir, container, capsys): + _run(data_dir) + err = capsys.readouterr().err + + assert "searched for: Bart, Seepferdchen, geschafft;" in err + + +# ── container side ────────────────────────────────────────────────────── + +class _StubLLM: + def __init__(self, reply): + self.reply = reply + self.prompts = [] + + async def complete(self, namespace, prompt, json_mode=False, **kwargs): + self.prompts.append(prompt) + return self.reply + + +def _answer_command(): + """Load `answer.py` by path: several stacklets have a `cli` package, + and whichever a test imported first would shadow this one.""" + import importlib.util + path = _REPO_ROOT / "stacklets" / "memory" / "bot" / "cli" / "answer.py" + spec = importlib.util.spec_from_file_location("memory_bot_cli_answer", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestAnswerCommand: + PAYLOAD = { + "question": "Wann hat Bart sein Seepferdchen geschafft?", + "evidence": [ + {"path": "family/diary/2026/06.md", "title": "Juni", "date": "", + "excerpt": "Bart hat heute sein Seepferdchen geschafft.", "summary": ""}, + {"path": "family/notes/garten.md", "title": "Garten", "date": "2026-09-17", + "excerpt": "Bart war heute im Garten.", "summary": ""}, + ], + } + + def test_prints_what_the_model_answered(self, monkeypatch, capsys): + llm = _StubLLM("Am 12. Juni 2026 [1].") + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(self.PAYLOAD))) + + code = asyncio.run(_answer_command().run(llm, [])) + + assert code == 0 + assert capsys.readouterr().out.strip() == "Am 12. Juni 2026 [1]." + + def test_the_prompt_numbers_the_evidence_for_citations(self, monkeypatch): + llm = _StubLLM("x") + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(self.PAYLOAD))) + + asyncio.run(_answer_command().run(llm, [])) + prompt = llm.prompts[0] + + assert self.PAYLOAD["question"] in prompt + assert "[1]" in prompt and "Seepferdchen geschafft" in prompt + assert "[2]" in prompt and "family/notes/garten.md" in prompt + + def test_an_empty_answer_is_a_failure(self, monkeypatch): + llm = _StubLLM(" ") + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(self.PAYLOAD))) + + assert asyncio.run(_answer_command().run(llm, [])) == 1 From 77b4dce30d530fab0923f05096e80d1ed627a71a Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 14:03:37 +0200 Subject: [PATCH 07/10] fix(agent): set the context window on the active model preset nanobot reads context_window_tokens from the active preset (default 200000); the value under agents.defaults was ignored, so token consolidation still budgeted against 200000. - config.json: context_window_tokens 32768 on model_presets.primary - test: the window sits on the active preset, not under defaults --- docs/adr/adr-012-nanobot-fork.md | 2 +- stacklets/agent/config.json | 4 ++-- stacklets/agent/runtime/README.md | 2 +- tests/stacklets/test_agent_config.py | 31 ++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 tests/stacklets/test_agent_config.py diff --git a/docs/adr/adr-012-nanobot-fork.md b/docs/adr/adr-012-nanobot-fork.md index f6e1c804..edef51a6 100644 --- a/docs/adr/adr-012-nanobot-fork.md +++ b/docs/adr/adr-012-nanobot-fork.md @@ -210,5 +210,5 @@ Two shims removed; history size is left to nanobot's own settings. | `grep_tool` | removed; grep is literal again | ran a regex as a semantic query (lesson 7); the model retried the same grep | | `compact_tools` | added; patches `agent.runner._COMPACTABLE_TOOLS` | nanobot's microcompact now also shortens old vault tool results | -`config.json` sets `context_window_tokens` to 32768, so nanobot's token +`config.json` sets the primary preset's `context_window_tokens` to 32768, so nanobot's token consolidation and history snip act at a size that keeps prefill short. diff --git a/stacklets/agent/config.json b/stacklets/agent/config.json index 51c4b090..66376d13 100644 --- a/stacklets/agent/config.json +++ b/stacklets/agent/config.json @@ -8,7 +8,8 @@ "model_presets": { "primary": { "provider": "custom", - "model": "${AGENT_MODEL}" + "model": "${AGENT_MODEL}", + "context_window_tokens": 32768 } }, "agents": { @@ -16,7 +17,6 @@ "model_preset": "primary", "max_tool_iterations": 12, "max_messages": 40, - "context_window_tokens": 32768, "disabled_skills": ["memory", "my"] } }, diff --git a/stacklets/agent/runtime/README.md b/stacklets/agent/runtime/README.md index d9e7f2c7..ec5865a4 100644 --- a/stacklets/agent/runtime/README.md +++ b/stacklets/agent/runtime/README.md @@ -11,7 +11,7 @@ size is left to nanobot's own settings in `config.json`: | Mechanism | Setting | Value | Effect | |---|---|---|---| | Replay window | `max_messages` | 40 | Older messages move to `history.jsonl` and the `# Recent History` section | -| Token budget | `context_window_tokens` | 32768 | Budget = window - 8192 output - 1024 = 23.5k; consolidation trims to 50% of it, history snip cuts at it | +| Token budget | `model_presets.primary.context_window_tokens` | 32768 | Budget = window - 8192 output - 1024 = 23.5k; consolidation trims to 50% of it, history snip cuts at it | | Idle autocompact | `idleCompactAfterMinutes` | 15 (default) | Idle session: summary plus the last turn | | Microcompact | `_COMPACTABLE_TOOLS` | nanobot's read tools + vault tools (`compact_tools.py`) | Keeps the 10 newest tool results, older ones (>= 500 chars) become `[ result omitted from context]` | diff --git a/tests/stacklets/test_agent_config.py b/tests/stacklets/test_agent_config.py new file mode 100644 index 00000000..c42c667d --- /dev/null +++ b/tests/stacklets/test_agent_config.py @@ -0,0 +1,31 @@ +"""The agent's nanobot config sets its limits where nanobot reads them. + +nanobot resolves generation limits from the active model preset when +`agents.defaults.model_preset` names one; the preset's own +`context_window_tokens` (default 200000) then wins over the same field +under `agents.defaults`. A window set only under defaults is silently +ignored, and token consolidation keeps budgeting against 200000. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +CONFIG = Path(__file__).resolve().parents[2] / "stacklets" / "agent" / "config.json" + + +def _config() -> dict: + return json.loads(CONFIG.read_text()) + + +def test_the_context_window_is_set_on_the_active_preset(): + config = _config() + active = config["agents"]["defaults"]["model_preset"] + + assert config["model_presets"][active]["context_window_tokens"] == 32768 + + +def test_no_context_window_where_nanobot_ignores_it(): + """A second value under defaults would read as the one in effect.""" + assert "context_window_tokens" not in _config()["agents"]["defaults"] From b0d795af7bc31bcda5c86f57c13db7355f9afb9a Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 14:06:10 +0200 Subject: [PATCH 08/10] feat(memory): show the section heading in a search excerpt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A diary month is one page and the day is its heading, so the matching line alone did not say when; the agent read the page again to find out. - excerpt: " › "; the heading alone when it matched --- stacklets/memory/lib.py | 37 ++++++++++++++---- tests/stacklets/test_memory_search.py | 54 +++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index e941ffea..500fb931 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -1626,9 +1626,16 @@ def strip_diacritics(text: str) -> str: return "".join(c for c in decomposed if not unicodedata.combining(c)) +_HEADING = re.compile(r"^#{1,6}\s+(.*)$") + + def _excerpt(text: str, patterns: List["re.Pattern[str]"], max_len: int = 200, fold_diacritics: bool = True) -> str: - """First non-empty body line that matches a pattern, tried in order. + """First body line that matches a pattern, with its section heading. + + The result is ` › `, or the line alone when + no heading comes before it, or the heading alone when the heading + itself matched. `patterns` are the query's alternatives, rarest first, compiled against folded text. A page found through "Bart|Seepferdchen" shows @@ -1645,14 +1652,28 @@ def _excerpt(text: str, patterns: List["re.Pattern[str]"], max_len: int = 200, still shows the line saying "Käse". A hit whose excerpt came back empty is a hit the reader cannot judge. """ - lines = [line.strip() for line in body_only(text).splitlines()] - lines = [line for line in lines if line] + # Each line with the nearest Markdown heading above it. A diary + # month is one page and the day is its heading, so a line without + # its heading does not say when it happened. + sections: List[tuple] = [] + heading = "" + for raw in body_only(text).splitlines(): + line = raw.strip() + if not line: + continue + if m := _HEADING.match(line): + heading = m.group(1).strip() + sections.append((heading, "")) + else: + sections.append((heading, line)) + for pattern in patterns: - for line in lines: - if pattern.search(strip_diacritics(line) if fold_diacritics else line): - if len(line) > max_len: - line = line[:max_len] + "…" - return line + for heading, line in sections: + candidate = line or heading + if pattern.search(strip_diacritics(candidate) if fold_diacritics else candidate): + if len(candidate) > max_len: + candidate = candidate[:max_len] + "…" + return f"{heading} › {candidate}" if line and heading else candidate return "" diff --git a/tests/stacklets/test_memory_search.py b/tests/stacklets/test_memory_search.py index e6888a81..d4148c2a 100644 --- a/tests/stacklets/test_memory_search.py +++ b/tests/stacklets/test_memory_search.py @@ -605,6 +605,60 @@ def test_an_escaped_bar_is_a_literal_not_a_split(self, tmp_path): assert search_memory(r"ls\|xx", v) == [] +class TestExcerptCarriesItsHeading: + """A diary month is one page; the day is the heading above the entry. + + The excerpt showed the matching line alone, so the model read the + page again to learn which day it was. With the nearest heading in + front, the result answers "when" by itself. + """ + + @pytest.fixture + def diary(self, tmp_path): + v = tmp_path / "vault" + _write(v / "family/diary/2026/06.md", """ + --- + title: Juni + --- + + # Juni 2026 + + ## 3. Juni + Bart war heute im Zoo. + + ## 12. Juni + Bart hat heute sein Seepferdchen geschafft. + """) + return v + + def test_the_nearest_heading_comes_before_the_line(self, diary): + results = search_memory("Seepferdchen", diary) + + assert results[0]["excerpt"] == "12. Juni › Bart hat heute sein Seepferdchen geschafft." + + def test_an_earlier_section_gets_its_own_heading(self, diary): + results = search_memory("Zoo", diary) + + assert results[0]["excerpt"] == "3. Juni › Bart war heute im Zoo." + + def test_a_match_in_a_heading_is_the_heading_alone(self, diary): + results = search_memory("12\\. Juni", diary) + + assert results[0]["excerpt"] == "12. Juni" + + def test_a_line_with_no_heading_above_stays_plain(self, tmp_path): + v = tmp_path / "vault" + _write(v / "family/notes/zettel.md", """ + --- + title: Zettel + --- + + Bart braucht eine neue Badehose. + """) + + assert search_memory("Badehose", v)[0]["excerpt"] == "Bart braucht eine neue Badehose." + + class TestMatchesLine: def test_multi_keyword_output_names_the_matches(self, stack_cli, child_vault): code, out, _ = stack_cli( From f018cdea65c9edaf24ec95eb7ac6a73582619851 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 14:06:10 +0200 Subject: [PATCH 09/10] feat(agent): cap memory_search at 8 hits and memory_history at 10 Tool output is prompt the model reads before its next step; on the local model each 1k tokens costs 5-7 s of prefill. The model asked for 20 hits and 30 changes, most of them noise. --- stacklets/agent/runtime/history_tool.py | 9 +++++++- stacklets/agent/runtime/memory_tool.py | 11 +++++++-- tests/stacklets/test_agent_vault_tools.py | 27 ++++++++++++++++++++++- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/stacklets/agent/runtime/history_tool.py b/stacklets/agent/runtime/history_tool.py index 32912bd2..6b2d370f 100644 --- a/stacklets/agent/runtime/history_tool.py +++ b/stacklets/agent/runtime/history_tool.py @@ -32,6 +32,9 @@ ) +MAX_CHANGES = 10 + + @tool_parameters( tool_parameters_schema( scope=StringSchema( @@ -54,7 +57,7 @@ nullable=True, ), limit=IntegerSchema( - "Optional. How many changes to return (default 10).", + "Optional. How many changes to return (default and at most 10).", nullable=True, ), ) @@ -87,6 +90,10 @@ async def execute(self, scope: str | None = None, by: str | None = None, since: str | None = None, item: str | None = None, limit: int | None = None) -> str: argv = ["stack", "memory", "history"] + # Each entry is prompt the model reads; more than ten rarely + # changes an answer and costs seconds of local prefill. + if limit: + limit = min(int(limit), MAX_CHANGES) if scope: argv.append(str(scope)) for flag, value in (("--item", item), ("--by", by), diff --git a/stacklets/agent/runtime/memory_tool.py b/stacklets/agent/runtime/memory_tool.py index 05594cef..6c08d675 100644 --- a/stacklets/agent/runtime/memory_tool.py +++ b/stacklets/agent/runtime/memory_tool.py @@ -15,6 +15,10 @@ ) +# Hits past the first handful rarely hold the answer and cost prefill. +MAX_HITS = 8 + + @tool_parameters( tool_parameters_schema( query=StringSchema( @@ -31,9 +35,9 @@ ), limit=IntegerSchema( 5, - description="Maximum number of results to return.", + description="Maximum number of results to return (at most 8).", minimum=1, - maximum=20, + maximum=8, nullable=True, ), scope=StringSchema( @@ -91,6 +95,9 @@ async def execute( # themselves are cheap and run concurrently. batch = [q for q in (queries or []) if q and q.strip()] or [query] batch = batch[:3] + # Every hit is prompt the model reads before its next step: about + # 150 tokens each, and 5-7 s of local prefill per 1k tokens. + limit = min(limit or 5, MAX_HITS) results = await asyncio.gather( *(self._search_one(q, limit, scope, person, tag) for q in batch) ) diff --git a/tests/stacklets/test_agent_vault_tools.py b/tests/stacklets/test_agent_vault_tools.py index 2840b776..e7761e67 100644 --- a/tests/stacklets/test_agent_vault_tools.py +++ b/tests/stacklets/test_agent_vault_tools.py @@ -36,7 +36,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent MEMORY_DIR = REPO_ROOT / "stacklets" / "memory" -TOOL_MODULES = ("memory_tool", "person_tool", "sitecustomize") +TOOL_MODULES = ("memory_tool", "person_tool", "history_tool", "sitecustomize") # ── loading the real components under test ─────────────────────────── @@ -87,6 +87,7 @@ def vault_tools(monkeypatch, nanobot_stub): tools = { "memory_search": importlib.import_module("memory_tool").MemorySearchTool, "memory_person": importlib.import_module("person_tool").MemoryPersonTool, + "memory_history": importlib.import_module("history_tool").MemoryHistoryTool, } yield tools for name in TOOL_MODULES: @@ -343,6 +344,30 @@ def test_a_failure_is_logged_with_its_exit_code(self, vault_tools, capsys): assert "-> error exit 2" in capsys.readouterr().err +class TestResultsAreCapped: + """A tool result is prompt the model has to read before its next step. + + On the local model each 1k tokens of tool output costs 5-7 s of + prefill. The model asked for 20 search hits and 30 history entries; + the hits past the first handful matched only a year and were noise. + """ + + def test_search_asks_for_at_most_eight_hits(self, vault_tools): + argv = argv_of(vault_tools["memory_search"], query="Bart Seepferdchen", limit=20) + + assert argv[argv.index("--limit") + 1] == "8" + + def test_a_smaller_search_limit_is_kept(self, vault_tools): + argv = argv_of(vault_tools["memory_search"], query="Bart Seepferdchen", limit=3) + + assert argv[argv.index("--limit") + 1] == "3" + + def test_history_asks_for_at_most_ten_changes(self, vault_tools): + argv = argv_of(vault_tools["memory_history"], since="2026-04-01", limit=30) + + assert argv[argv.index("--limit") + 1] == "10" + + def test_naming_a_person_does_not_hide_pages_that_do_not_list_them( memory_cli, vault_tools, tmp_path, capsys): """The model passes the person a question is about; that must not From effab34351a6a0eaf155ba39404a1bf9dbb502ab Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 19 Sep 2026 14:37:08 +0200 Subject: [PATCH 10/10] docs(brain): one content pattern; diary cards in the vault Design note, nothing built. Every brain page gets the document pattern (frontmatter, summary and facts, ontology tags, OKF fields). The diary gets one card per entry in the vault, posted as a thread reply on the entry's last main-timeline message and corrected by replying in that thread, like documents; the warm diary pages are compiled from the cards. - entry unit open: message group, day or occasion - built as a second compiler and compared against the current one --- docs/design/brain/content-pattern.md | 305 +++++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 docs/design/brain/content-pattern.md diff --git a/docs/design/brain/content-pattern.md b/docs/design/brain/content-pattern.md new file mode 100644 index 00000000..54472a4c --- /dev/null +++ b/docs/design/brain/content-pattern.md @@ -0,0 +1,305 @@ +# One content pattern for the brain + +Status: proposed, 2026-09-19. Nothing is built yet. +Related: [open-knowledge-format.md](open-knowledge-format.md), +[vault-format.md](vault-format.md), [diary-journal.md](diary-journal.md), +ADR-011 (vault as database, brain as projection). + +## Goal + +Every page type in the brain carries the same parts: the same +frontmatter fields, the same summary and facts block, the same tag +vocabulary. Search, the agent, `stack memory ask`, Obsidian and OKF +consumers then read all content the same way. + +The pattern already exists for documents and captures. This note +applies it to the diary and to all content added later. + +## Current state + +| Content | Written by | Frontmatter | Summary and facts | Tags from the ontology | +|---|---|---|---|---| +| Document mirror | archivist, `vault_entry.render_document` | `type: document`, `title`, `persons`, `tags`, `document_type`, `paperless_id` | `> [!summary]` callout: summary, facts, action items | yes | +| Capture (note, bookmark) | archivist, `vault_entry.render_capture` | `type: `, `title`, `persons`, `tags`, `source_uri`, `added` | `> [!summary]` callout | yes | +| Email message | mail bot, `vault_entry.render_email_message_section` | per message section | per-message callout | yes | +| Diary month | diary compiler, `diary.render_month` | `title` only | none | no | + +The diary reading step (`diary.Reading`) returns `mode`, +`spoken_date`, `addressee`, `gist` and `moments`. It returns no +persons, no tags and no facts. + +### Effect on retrieval + +Observed on 2026-09-19 with the agent on a diary question: + +| Step | Model calls | Reason | +|---|---|---| +| search finds the month page | 1 | the page matches the keywords | +| grep the month page | 1-2 | the hit is one line without its day | +| read sections of the page | 1-2 | to find the day and the full entry | +| answer | 1 | | + +Each extra call costs 5-25 s of prefill on the local model. A search +hit that already holds the entry, its day, its people and its facts +removes the grep and read calls. + +## The pattern + +### Frontmatter + +| Field | OKF | Content | Source | +|---|---|---|---| +| `type` | required | `document`, `note`, `bookmark`, `email`, `diary-entry`, ... | writer | +| `title` | standard | short name of the item | classifier | +| `timestamp` | standard | when it happened (not when it was filed) | writer; for the diary, `date_for` | +| `description` | standard | one sentence: what this is | classifier | +| `resource` | standard | link to the original: Paperless document, audio, Matrix event | writer | +| `tags` | standard | ontology tags: `Topic:`, `Person:` | classifier | +| `persons` | custom | family members the item is about | classifier | +| type-specific | custom | `document_type`, `paperless_id`, `capture_id`, ... | writer | + +The field renames in `open-knowledge-format.md` ("Changes", items 1-4: +`added` to `timestamp`, `source_uri` to `resource`, `type` everywhere) +are part of this pattern. + +### Body + +```markdown +> [!summary] +> One to three sentences, narrative, in the family's language. +> +> - Fact with a name, date, number or place +> - Fact ... + + +``` + +Rules, same as for documents today: + +- Facts anchor on a name, date, number or place. A sentence without one + is summary, not a fact. +- Summary and facts are generated. The content below them is the + source. Quotes come only from the content (the diary rule in + `diary.py`: "The diary quotes, it does not invent"). +- One classifier vocabulary. The diary reading step uses the same + ontology section (`ontology.classifier_prompt_section`) as the + document classifier, so `Topic:Health` means the same on a letter + from the doctor and on a diary entry. + +## Diary: cards in the vault, pages as projections + +The diary is personal and must read like one: day headings, the +family's own words, a short narrative, links to the audio. Tags, facts +and frontmatter do not belong on the page the family reads +(`diary-journal.md`: the Diary is personal and verbatim, the Journal is +operational and factual). + +Search and the agent need the machine fields. Two stored artifacts, +one for people and one for machines, would be two sources of truth: a +correction would have to be made in both. So there is one source, the +**card**, and every view is generated from it. + +### Flow + +``` +memo, photo or note in the memories room + -> after the day's messages settle: extract one card per entry, + write it to the vault + -> the memory bot posts the card as a thread reply; the thread root + is the entry's last message in the main timeline + -> a family member replies in that thread to correct it + -> the card is extracted again with the correction; the correction wins + -> the diary compiler reads the cards and builds the warm pages (brain) +``` + +This is the flow the archivist already runs for documents: file, +reply with what was extracted, correct by a reply in the thread. The +correction pass starts from the state the person saw +(`_initial_classification_block` in `stacklets/docs/bot/pipeline.py`). + +| Layer | Where | Role | Changed by | +|---|---|---|---| +| Card | vault (source) | one file per entry: machine frontmatter, summary and facts, the verbatim text or transcript, the audio link | the card extractor; corrections by thread reply | +| Card post | memories room, in the thread on the entry's last main-timeline message | shows the family what was extracted; the same thread is the place to correct it | the memory bot (edits its own post, `m.replace`) | +| Diary pages | brain (projection) | the warm month, year and index pages, composed from the cards | nobody; rebuilt from the cards | +| Search, agent, `stack memory ask`, OKF export | read the cards through the brain mirror | find and answer | nobody | + +This is ADR-011's rule. A diary entry is a record, a thing that +happened, so its card lives in the vault. The month page can be +rebuilt losslessly, so it lives in the brain. Today the diary exists +only in the brain, which is also why an edit on a diary page would be +lost on the next compile. + +### Rules + +- **Thread on the entry's last message.** The card is a thread reply, + and the thread root is the last message of the entry in the main + timeline. That is the natural point: the entry is complete there, + and the thread marker sits on it. The root must be a main-timeline + message, because Matrix threads cannot be nested. The memories room + keeps the family's own messages in its main timeline; cards and + corrections stay in threads. +- **Corrections in the same thread.** A family member replies in the + card's thread, as for documents. +- **Extract after the messages settle.** The compiler attaches late + captions, replies and split uploads within time windows + (`join_fragments`, `REMARK_WINDOW_S`, sync bursts). Cards are made in + a nightly run or after a quiet period, not per message. +- **Late additions edit the card.** A message that joins an entry after + its card was posted updates the card in the vault and edits the bot's + post (`m.replace`). It does not post again. +- **One card per entry.** A card that covers several messages goes in + the thread of the last of them in the main timeline at extraction + time. A message that is itself a reply inside a thread is never the + root. +- **The card holds the source text.** The transcript or text word for + word, and the audio link. The diary compiler reads only the cards, so + its quotes come from the same source as the facts. +- **Corrections are explicit.** A correction arrives as a thread reply, + so the extractor knows it is human input and keeps it on every later + pass. It never removes a heading or shortens the body + (augmentation-strict, `open-knowledge-format.md`, Tier 1, item 3), + and the write seam enforces this in code. +- **The memory stacklet owns it.** The memories room and the diary + belong to memory, not to the archivist, which owns Paperless + (AGENTS.md, principle 5). The thread-correction handling becomes a + shared piece both bots use, not a copy. + +### Card layout + +``` +family/diary/2026/06/2026-06-12-.md one card (vault) +``` + +Example card (demo family): + +```markdown +--- +type: diary-entry +title: Seepferdchen +timestamp: 2026-06-12 +description: Bart passes his first swimming badge at the public pool. +persons: + - Bart + - Marge +tags: + - Topic:Sport + - Person:Bart +resource: https://matrix.to/#/!room/$event +--- + +> [!summary] +> Marge records Bart after his swimming test at the public pool. +> +> - Bart: Seepferdchen passed +> - Place: Springfield public pool + + +``` + +The month page shows the transcript, the quotes and the audio link +from each card under its day heading. It does not show the +frontmatter, the summary callout or the facts. The diary compiler has +no model step of its own for entries; it formats cards. The month +digest paragraph stays generated, from the cards of that month. + +### Path identity + +The path must not change when a card is extracted again. The slug +comes from the entry's date and a short hash of its first event id. It +does not come from the generated title, because a correction can +change the title. + +### Matrix load + +At about 5 entries a day, the memories room gets about 1,800 threads +a year, one per card. Threads are ordinary events with an `m.thread` +relation, and Element loads the thread list lazily, so this is +expected to be fine. +Measure it before rollout: seed a test-rig room with a year of +synthetic entries and cards (`tools/family-memories`), then time room +open and thread-list scrolling in Element. + +## What is a diary entry + +The entry unit is an open decision, and the pattern depends on it. + +Today the compiler's unit is a **message group** (`diary.Entry`, +`compile_entries`): one recording, photo or text, with its split +uploads joined (`join_fragments`) and its replies and captions attached +(`refers_to`). The **day** is only the page layout: `render_month` puts +the entries under day headings. + +| Unit | For | Against | +|---|---|---| +| Message group (current `Entry`) | Exists and is tested. One source, so quotes stay safe. Stable identity from the first event id. | Many small files. A memo and a photo of the same occasion become two entries. | +| Day | Matches how the family reads the diary. One file per day. | A day mixes unrelated occasions (a zoo visit and a doctor's appointment); tags and facts become a mix, and a hit returns the whole day. | +| Occasion (model-grouped) | One file per thing that happened, across messages and across days. Best unit for tags, facts and retrieval. | Needs a grouping step with a model. New failure mode: wrong merges. Identity is harder when a later message joins an occasion. | + +Criteria for the decision: + +1. Retrieval: rank of the answering entry for the questions in the + agent's search log (`[memory_search]` lines since 2026-09-19). +2. Purity: share of entries whose tags and facts describe one + occasion only. +3. Correction: one correction touches one card. +4. Stability: two compiles of the same room give the same paths. + +The month page groups by day whatever the record unit is, so the +reading experience does not depend on this choice. + +## Build as a separate compiler, then compare + +The current diary compiler stays as it is. A second compiler is built +next to it, with the same inputs, and the two outputs are compared +before one replaces the other. + +| | Current compiler | New compiler | +|---|---|---| +| Input | room events, cached readings (`diary_store`) | the same | +| Reading step | `mode`, `spoken_date`, `addressee`, `gist`, `moments` | card extraction: the same, plus `title`, `description`, `persons`, `tags`, `summary`, `facts`; cached per entry | +| Entry unit | message group, rendered by day | the unit under test (see above); the compiler takes it as a parameter | +| Output | month pages (brain) | cards in a separate vault tree, card posts in a test room, month pages composed from the cards in a separate brain tree | +| Tests | `tests/stacklets/test_memory_diary.py` | new module tests, written first | + +### Comparison + +| Measure | How | +|---|---| +| Retrieval | replay the questions from the agent's search log against both trees; rank of the answering page or entry | +| Agent cost | the same questions through the agent: model calls, wall time, prompt tokens | +| OKF conformance | the validator from `open-knowledge-format.md` ("Build", item 6) on both trees | +| Reading | the family reads both month pages for the same month | +| Cost | model calls and time for a full compile and for an incremental run | +| Stability | paths after two compiles of the same room | +| Corrections | a thread reply changes the card, and the next compile shows it on the diary page | +| Matrix load | room open and thread-list time in Element with a year of card threads | + +The first runs use the synthetic corpus in `tools/family-memories`. +A run on the family's own room writes cards into the production +vault and posts into the memories room. It needs the owner's decision +first. + +## Search changes that go with it + +- Search matches `description` as it matches `title` and tag values + today. +- Matches in the summary callout rank above matches in the body. +- Pages that are not diary entries (topic pages, emails, long + documents) are split into heading sections for ranking, so a hit is + the section that matched. + +## Open questions + +1. The entry unit (see above). +2. Cadence of card extraction: nightly, or after a quiet period per + room, and how long a card stays open for late additions. +3. The memory bot becomes a vault writer. ADR-011 lists three writers + (archivist, CLI, humans): the memory bot joins them, or it files + through the CLI's write path. +4. OKF `index.md` against the diary's `about.md`. Quartz renders a + folder's `index.md` without a body (`diary.pages_for`). The OKF + exporter (`open-knowledge-format.md`, "Build", item 5) can write + `index.md` for export and keep `about.md` for the wiki. +5. Replace the current diary tree, or keep both until the family + agrees.