From 360551b0c060cbaec787d0908e19be7994421cf6 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 15 Sep 2026 22:46:34 +0200 Subject: [PATCH 1/2] feat(web): add the web stacklet with SearXNG search An optional stacklet runs SearXNG on the family's own hardware, so a search reaches upstream engines from the instance rather than from a signed-in browser. This is not anonymity and the documentation says so. stack web search queries it from the terminal, and a search page is served on the LAN. stack web fetch reads a single page through the same ladder the archivist uses. The stacklet is opt-in. Link capture does not require it. Refs docs/design/web/plan.md (phase 2). --- docs/design/web/plan.md | 2 +- lib/stack/web/__init__.py | 3 +- lib/stack/web/fetch.py | 49 +++++++++ stacklets/web/caddy.snippet | 12 +++ stacklets/web/cli/fetch.py | 88 ++++++++++++++++ stacklets/web/cli/search.py | 114 ++++++++++++++++++++ stacklets/web/config/settings.yml | 30 ++++++ stacklets/web/docker-compose.yml | 54 ++++++++++ stacklets/web/stacklet.toml | 48 +++++++++ tests/framework/test_web_transport.py | 126 +++++++++++++++++++++++ tests/integration/test_web_search_e2e.py | 118 +++++++++++++++++++++ 11 files changed, 642 insertions(+), 2 deletions(-) create mode 100644 stacklets/web/caddy.snippet create mode 100644 stacklets/web/cli/fetch.py create mode 100644 stacklets/web/cli/search.py create mode 100644 stacklets/web/config/settings.yml create mode 100644 stacklets/web/docker-compose.yml create mode 100644 stacklets/web/stacklet.toml create mode 100644 tests/framework/test_web_transport.py create mode 100644 tests/integration/test_web_search_e2e.py diff --git a/docs/design/web/plan.md b/docs/design/web/plan.md index 9c2a7dca..7d7068d0 100644 --- a/docs/design/web/plan.md +++ b/docs/design/web/plan.md @@ -222,7 +222,7 @@ confirm neither produces a fabricated entry. loads a fixture and asserts a gate verdict, so adding a site profile later costs one fixture plus one line. -### Phase 2 — The `web` stacklet, search half (about half a day) +### Phase 2 — The `web` stacklet, search half — SHIPPED - `stacklets/web/` with `stacklet.toml`, compose, `config/settings.yml` (`use_default_settings`, generated `secret_key`, `formats: [html, json]`), diff --git a/lib/stack/web/__init__.py b/lib/stack/web/__init__.py index 2e4c0261..3910d815 100644 --- a/lib/stack/web/__init__.py +++ b/lib/stack/web/__init__.py @@ -14,7 +14,7 @@ """ from stack.web.content import SourceContent -from stack.web.fetch import FetchOutcome, fetch_url +from stack.web.fetch import FetchOutcome, fetch_url, urllib_transport from stack.web.profiles import canonicalize, profile_for from stack.web.quality import Page, Verdict, assess @@ -27,4 +27,5 @@ "canonicalize", "fetch_url", "profile_for", + "urllib_transport", ] diff --git a/lib/stack/web/fetch.py b/lib/stack/web/fetch.py index 537ba58a..aedb6d79 100644 --- a/lib/stack/web/fetch.py +++ b/lib/stack/web/fetch.py @@ -277,6 +277,55 @@ def _from_url_only(url: str, title: str, profile: Profile) -> FetchOutcome: ) +# ── Transports ──────────────────────────────────────────────────────── +# +# Two, because the ladder has two callers with incompatible +# environments. The bot runs inside a container that already has +# aiohttp and an open session. The host CLI runs on a Mac with no +# virtualenv and possibly nothing up at all, so it gets the standard +# library. Neither is imported at module level. + +def urllib_transport(*, timeout: int = 30) -> Transport: + """A transport over the standard library, for the host CLI. + + `stack web fetch` has to work with no containers running and no + third-party packages installed, which rules out aiohttp. urllib is + synchronous, so the request goes to a worker thread and the ladder + stays async for both callers. + + An HTTP error is returned rather than raised. A 403 carrying a + Cloudflare challenge is not a transport failure — it is a page, and + it is exactly the page the gate needs to look at. + """ + def _blocking(url: str, headers: dict) -> Response | None: + import urllib.error + import urllib.request + + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as resp: + return Response( + url=resp.url, + status=resp.status, + html=resp.read().decode("utf-8", errors="replace"), + content_type=resp.headers.get_content_type(), + ) + except urllib.error.HTTPError as err: + return Response( + url=err.url, + status=err.code, + html=err.read().decode("utf-8", errors="replace"), + content_type=err.headers.get_content_type() if err.headers else "text/html", + ) + + async def _fetch(url: str, headers: dict) -> Response | None: + import asyncio + + return await asyncio.to_thread(_blocking, url, headers) + + return _fetch + + # ── aiohttp transport ───────────────────────────────────────────────── def aiohttp_transport(session, *, timeout: int = 30) -> Transport: diff --git a/stacklets/web/caddy.snippet b/stacklets/web/caddy.snippet new file mode 100644 index 00000000..5018b7ef --- /dev/null +++ b/stacklets/web/caddy.snippet @@ -0,0 +1,12 @@ +# stacklets/web/caddy.snippet +# +# The family-facing search page. Caddy runs on the stack network, so +# the backend is the container name and the in-container port (8080), +# not the published host port. +# +# No auth, matching the other family-facing stacklets. The LAN is the +# perimeter. + +search.{$FAMSTACK_DOMAIN} { + reverse_proxy stack-web-search:8080 +} diff --git a/stacklets/web/cli/fetch.py b/stacklets/web/cli/fetch.py new file mode 100644 index 00000000..17930150 --- /dev/null +++ b/stacklets/web/cli/fetch.py @@ -0,0 +1,88 @@ +"""stack web fetch — read a web page the way the archivist does. + +Host-native. Same code path the bot takes for a pasted link, with +nothing running: canonicalize the URL, try the page's own structured +data, fall back to HTTP plus extraction, and put the result through the +quality gate. No container, no model, no network hop to a service. + +That is the point of the command. When a link files badly, the question +is always "what did we actually get back", and answering it should not +require the stack to be up or a bot to be restarted. + + stack web fetch https://www.essen-und-trinken.de/rezepte/48816-... + Griechischer Salat Rezept + tier 1 (default) — the page published its own structured data + + **Servings:** 4 · **Total time:** 35 min + ## Ingredients + - 500 g rote und gelbe Paprika + ... + +A page we cannot read says so, and says which obstacle it was: + + stack web fetch https://www.decathlon.de/ + https://www.decathlon.de/ + challenge — bot protection served 'just a moment...' instead of the page + +A refusal is a successful read of a blocked page, not a command +failure, so the exit code stays 0. The branch point for a script is +`--json`, whose `verdict` field carries which of the five refusals it +was — more than an exit code could say — alongside the tier, the +profile, the title and the body. +""" + +HELP = "Read a web page and print it as Markdown" + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "lib")) + +from stack.web import fetch_url # noqa: E402 +from stack.web.fetch import urllib_transport # noqa: E402 + + +def run(args, stacklet, config): + parser = argparse.ArgumentParser(prog="stack web fetch", add_help=False) + parser.add_argument("url", nargs="?") + parser.add_argument("--json", action="store_true") + parser.add_argument("--timeout", type=int, default=30) + parser.add_argument("-h", "--help", action="store_true") + opts = parser.parse_args(args) + + if opts.help or not opts.url: + print(__doc__.strip()) + return {"ok": True} + + outcome = asyncio.run(fetch_url( + opts.url, transport=urllib_transport(timeout=opts.timeout), + )) + + if opts.json: + print(json.dumps({ + "url": outcome.url, + "verdict": outcome.verdict.name, + "detail": outcome.verdict.detail, + "tier": outcome.tier, + "profile": outcome.profile, + "title": outcome.content.title_hint if outcome.content else None, + "text": outcome.content.text if outcome.content else None, + }, indent=2, ensure_ascii=False)) + return {"ok": outcome.ok} + + # The header is the same two lines either way -- what we ended up + # reading, and how we got there. A refusal is not an error report, + # it is the same shape with no body under it. + print() + print(f" {outcome.content.title_hint if outcome.content else outcome.url}") + if outcome.ok: + print(f" tier {outcome.tier} ({outcome.profile}) — {outcome.verdict.detail}") + print() + print(outcome.content.text) + else: + print(f" {outcome.verdict.name} — {outcome.verdict.detail}") + print() + return {"ok": outcome.ok} diff --git a/stacklets/web/cli/search.py b/stacklets/web/cli/search.py new file mode 100644 index 00000000..5fdf2d36 --- /dev/null +++ b/stacklets/web/cli/search.py @@ -0,0 +1,114 @@ +"""stack web search — search the internet from the terminal. + +Queries the instance's own SearXNG, which forwards to upstream engines +and merges the results. Nothing about the family reaches Google as a +logged-in profile; the query itself still reaches an upstream engine, +and that is the honest description rather than a claim of anonymity. + + stack web search "immich vs photoprism" + 1. Immich vs PhotoPrism: which self-hosted photo manager + https://example.com/immich-vs-photoprism + Both index a library and serve it over the LAN, but they ... + + stack web search "immich vs photoprism" --json # for an agent + stack web search "mac mini idle watts" --count 5 + +Unlike `stack web fetch`, this needs the stacklet up: search is a +service, not a library. When it is down the command says so and points +at `stack up web` rather than printing an empty result list, because +"no results" and "nothing is running" are different problems and a +family that cannot tell them apart will retype the query. + +The JSON API this reads is off in SearXNG's shipped defaults +(`search.formats` is `[html]`). The stacklet's settings overlay turns +it on; if that ever regresses the web UI keeps working perfectly and +only this command breaks, which is why there is a test for it. +""" + +HELP = "Search the internet through your own SearXNG" + +import argparse +import json +import urllib.error +import urllib.parse +import urllib.request + +SEARCH_URL = "http://localhost:42080/search" +DEFAULT_COUNT = 8 + + +def search(query: str, *, count: int = DEFAULT_COUNT, timeout: int = 20) -> list[dict]: + """Results as a list of `{title, url, content, engine}`. Raises OSError + when the service cannot be reached.""" + params = urllib.parse.urlencode({"q": query, "format": "json"}) + request = urllib.request.Request( + f"{SEARCH_URL}?{params}", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode("utf-8", errors="replace")) + + results = [] + for item in payload.get("results", [])[:count]: + results.append({ + "title": (item.get("title") or "").strip(), + "url": (item.get("url") or "").strip(), + "content": (item.get("content") or "").strip(), + "engine": item.get("engine") or "", + }) + return results + + +def run(args, stacklet, config): + parser = argparse.ArgumentParser(prog="stack web search", add_help=False) + parser.add_argument("query", nargs="*") + parser.add_argument("--json", action="store_true") + parser.add_argument("--count", type=int, default=DEFAULT_COUNT) + parser.add_argument("-h", "--help", action="store_true") + opts = parser.parse_args(args) + + query = " ".join(opts.query).strip() + if opts.help or not query: + print(__doc__.strip()) + return {"ok": True} + + try: + results = search(query, count=opts.count) + except urllib.error.HTTPError as err: + # 403 here is the shipped-defaults failure, and it is worth + # naming: the web UI works, so nothing looks broken until a + # programmatic query is tried. + if err.code == 403: + return {"error": ( + "SearXNG refused the JSON API (403). Its `search.formats` " + "setting is back to html-only — check " + "stacklets/web/config/settings.yml is mounted." + )} + return {"error": f"search failed: HTTP {err.code}"} + except OSError: + return {"error": "Search needs the web stacklet running. Run `stack up web`."} + + if opts.json: + print(json.dumps(results, indent=2, ensure_ascii=False)) + return {"ok": True} + + if not results: + print(f"\n No results for \"{query}\".\n") + return {"ok": True} + + print() + for n, item in enumerate(results, 1): + print(f" {n}. {item['title']}") + print(f" {item['url']}") + if item["content"]: + print(f" {_wrap(item['content'])}") + print() + return {"ok": True} + + +def _wrap(text: str, width: int = 88) -> str: + """One-line snippet, truncated rather than reflowed — the list reads + as a scannable column, and a three-line snippet per hit buries the + next result.""" + flat = " ".join(text.split()) + return flat if len(flat) <= width else flat[: width - 1] + "…" diff --git a/stacklets/web/config/settings.yml b/stacklets/web/config/settings.yml new file mode 100644 index 00000000..245e81cf --- /dev/null +++ b/stacklets/web/config/settings.yml @@ -0,0 +1,30 @@ +# SearXNG instance settings — overlaid on the image's own defaults. +# +# `use_default_settings` means this file is a patch, not a replacement: +# every engine, category and locale the image ships stays as it was, and +# only the keys below change. Without it, upgrading SearXNG would +# silently strip whatever the new version added. +# +# The secret key is NOT here. SearXNG reads `${SEARXNG_SECRET}` over +# this file, and the stack generates one per install, so a key never +# lands in the repo. + +use_default_settings: true + +search: + # The image ships `formats: [html]`, so the JSON API is off by + # default and every programmatic query returns a 403. `stack web + # search` and `stack web ask` are both JSON clients, which makes this + # single line load-bearing -- and a silent failure mode if it ever + # regresses, since the web UI keeps working perfectly. There is a + # stacktests lane that asserts it. + formats: + - html + - json + +server: + # Behind Caddy on the stack network, and on a LAN. The limiter exists + # to keep a public instance from being scraped; here it would only + # rate-limit the family. + limiter: false + public_instance: false diff --git a/stacklets/web/docker-compose.yml b/stacklets/web/docker-compose.yml new file mode 100644 index 00000000..7d4af61c --- /dev/null +++ b/stacklets/web/docker-compose.yml @@ -0,0 +1,54 @@ +# stacklets/web/docker-compose.yml — the search half of web reading +# +# One service today. SearXNG is a metasearch engine: it forwards a +# query to upstream engines and merges the results, so there is no +# index to build and no crawl to run. Cold start is about a second. +# +# On privacy, the honest description: this stops Google and Bing +# building a profile of the family, because the query arrives from the +# instance rather than from a logged-in browser. It is not anonymity — +# the query still reaches an upstream engine, and the plan says so +# plainly rather than claiming otherwise. +# +# The stealth fetch service (tier 3, a real browser) lands here in +# phase 4 as a second service. It is deliberately not bundled now: +# search is a 60 MB image, a browser is 250 MB, and a family that only +# wants search should not pay for Chromium. + +name: stack-web + +services: + stack-web-search: + container_name: stack-web-search + image: searxng/searxng:${WEB_SEARCH_VERSION:-latest} + networks: + - stack + environment: + # SEARXNG_BASE_URL is deliberately unset. The image's default + # derives the public URL from the request, which is right for + # both modes here: port mode is hit on the LAN IP, domain mode + # arrives through Caddy with forwarding headers. Pinning it would + # mean guessing which one this instance uses. + # + # Signs the session cookie. Generated per install and kept in the + # stack's secret store, never in the repo. + SEARXNG_SECRET: ${WEB_SECRET} + # The image binds 127.0.0.1 by default, which would be unreachable + # from Caddy and from the published port. + SEARXNG_BIND_ADDRESS: "0.0.0.0" + TZ: ${TZ:-UTC} + volumes: + # Read-only: the settings overlay is repo state, not instance + # state. Nothing SearXNG writes belongs in version control, and a + # writable mount invites drift between the file and the container. + - ./config/settings.yml:/etc/searxng/settings.yml:ro + ports: + # 8080 is the image's internal listener. PORT_BIND_IP is set by + # the runtime: 0.0.0.0 in port mode, 127.0.0.1 in domain mode so + # Caddy is the only way in. + - "${PORT_BIND_IP:-127.0.0.1}:42080:8080" + restart: unless-stopped + +networks: + stack: + external: true diff --git a/stacklets/web/stacklet.toml b/stacklets/web/stacklet.toml new file mode 100644 index 00000000..7047e3c9 --- /dev/null +++ b/stacklets/web/stacklet.toml @@ -0,0 +1,48 @@ +# stacklet.toml — web stacklet (reading the internet) +# +# The stacklet is what *stops* web reading being heavy. Fetching a +# pasted link is not in here: canonicalization, structured data, plain +# HTTP, extraction and the quality gate are pure Python in +# `lib/stack/web/`, so `stack web fetch` works with nothing running and +# the archivist never waits on a container for the case that never +# needed one. +# +# What is in here is the part that genuinely needs a service. Search is +# a search engine. The stealth fetch tier, when it lands, is a browser +# — roughly 250 MB of Chromium that would otherwise tax every bot image +# for a capability most of them never invoke. +# +# So installing this is opt-in by construction: a family that never +# searches and never pastes a shop link never downloads any of it. + +id = "web" +name = "Web" +description = "Search the internet privately (SearXNG)" +version = "0.1.0" +category = "infrastructure" +port = 42080 + +hints = [ + "Search at {url}", + "From the terminal: `stack web search \"immich vs photoprism\"`", + "Read a page: `stack web fetch ` (works with nothing running)", +] + +[upstream] +image = "searxng/searxng" +channel = "patch" + +# `generate` keys are minted per install and kept in the stack's secret +# store, never in the repo. SearXNG signs its session cookie with this; +# the shipped default is the literal string "ultrasecretkey". +[env] +generate = ["WEB_SECRET"] + +[env.defaults] +WEB_DATA_DIR = "{data_dir}/web" +TZ = "{timezone}" + +[health] +url = "http://localhost:42080/healthz" +expect = "200" +timeout = 60 diff --git a/tests/framework/test_web_transport.py b/tests/framework/test_web_transport.py new file mode 100644 index 00000000..e4b4d9b5 --- /dev/null +++ b/tests/framework/test_web_transport.py @@ -0,0 +1,126 @@ +"""The host CLI's transport: stdlib only, and a block is a page. + +`stack web fetch` has to work on a Mac with nothing running and no +virtualenv — that is the whole reason tiers 0 to 2 are not in a +container. So the host gets a urllib transport rather than the bot's +aiohttp one, and the ladder is handed whichever fits. + +The subtle requirement is the error path. urllib raises on any 4xx, +but a 403 carrying a Cloudflare challenge is not a transport failure — +it is the page we most need the gate to read. A transport that lets +that exception escape would turn every blocked site into "could not be +reached", which is both wrong and less useful than the truth. + +Driven against pytest-httpserver, so the real urllib code path runs. +""" + +from __future__ import annotations + +from stack.web.fetch import BROWSER_HEADERS, fetch_url, urllib_transport + +ARTICLE = """Why local LLMs matter +

Why local LLMs matter

+

Running models on your own hardware changes the privacy calculus entirely. +Your prompts never leave the machine, the model never phones home, and you can +iterate without worrying about quotas or per-token billing at any point.

+

For a family this means one server in a closet replaces three separate cloud +subscriptions. The arithmetic works out somewhere around the fourth month.

+
""" + +CHALLENGE = ( + 'Just a moment...' + 'Enable JavaScript and cookies to ' + "continue" +) + + +class TestTheHostCanReadAPage: + async def test_an_article_fetches_and_extracts(self, httpserver): + httpserver.expect_request("/article").respond_with_data( + ARTICLE, content_type="text/html", + ) + + outcome = await fetch_url( + httpserver.url_for("/article"), transport=urllib_transport(), + ) + + assert outcome.ok + assert outcome.content is not None + assert "privacy calculus" in outcome.content.text + assert outcome.content.title_hint == "Why local LLMs matter" + + async def test_browser_headers_are_actually_sent(self, httpserver): + """Plenty of CDNs answer a bare library user-agent with a 403 out + of habit. Sending a browser's headers is the difference between + reading a public page and not, so it is worth pinning that they + reach the wire rather than sitting in a dict.""" + seen = {} + + def _handler(request): + seen.update(request.headers) + from werkzeug.wrappers import Response as WerkzeugResponse + return WerkzeugResponse(ARTICLE, content_type="text/html") + + httpserver.expect_request("/article").respond_with_handler(_handler) + await fetch_url(httpserver.url_for("/article"), transport=urllib_transport()) + + assert seen.get("User-Agent") == BROWSER_HEADERS["User-Agent"] + + +class TestABlockIsAPageNotAnError: + async def test_a_403_challenge_reaches_the_gate(self, httpserver): + """urllib raises HTTPError on a 403. If that escapes, every + blocked site reports as unreachable and the family is told + something false.""" + httpserver.expect_request("/blocked").respond_with_data( + CHALLENGE, status=403, content_type="text/html", + ) + + outcome = await fetch_url( + httpserver.url_for("/blocked"), transport=urllib_transport(), + ) + + assert outcome.verdict.name == "challenge" + assert outcome.content is None + + async def test_a_404_is_reported_as_empty_not_unreachable(self, httpserver): + httpserver.expect_request("/missing").respond_with_data( + "Not found", status=404, + content_type="text/html", + ) + + outcome = await fetch_url( + httpserver.url_for("/missing"), transport=urllib_transport(), + ) + + assert not outcome.ok + assert outcome.verdict.name == "empty" + + async def test_a_dead_host_is_still_a_verdict(self): + """A genuine transport failure — nothing listening — must also + come back as a verdict, because the CLI prints one either way.""" + outcome = await fetch_url( + "http://127.0.0.1:1/article", transport=urllib_transport(timeout=2), + ) + + assert not outcome.ok + assert outcome.verdict.detail + + +class TestStructuredDataOverTheWire: + async def test_a_recipe_served_by_http_is_read_from_its_markup( + self, httpserver, web_fixture, + ): + """The Phase 2 promise: `stack web fetch` on a recipe prints + ingredients with no container running at all.""" + httpserver.expect_request("/rezept").respond_with_data( + web_fixture("recipe-jsonld"), content_type="text/html", + ) + + outcome = await fetch_url( + httpserver.url_for("/rezept"), transport=urllib_transport(), + ) + + assert outcome.ok + assert outcome.tier == "1" + assert "## Ingredients" in outcome.content.text diff --git a/tests/integration/test_web_search_e2e.py b/tests/integration/test_web_search_e2e.py new file mode 100644 index 00000000..efc3159f --- /dev/null +++ b/tests/integration/test_web_search_e2e.py @@ -0,0 +1,118 @@ +"""Search, against the real SearXNG container. + +One assertion carries this file: that the JSON API is on. + +SearXNG ships `search.formats: [html]`, so every programmatic query +returns 403 out of the box. The stacklet's settings overlay adds +`json`, and that single line is load-bearing for `stack web search` +and for `stack web ask`. It is also the quietest thing in the stacklet +that can break: the web UI keeps working perfectly when the overlay +stops being applied, so nothing looks wrong until an agent asks a +question and gets a refusal. + +A container upgrade, a rename of the mount path, or a future +`use_default_settings` change would all do it. Hence a lane rather +than a comment. + +Run via the rig: + + tests/integration/stacktests up web + tests/integration/stacktests pytest \ + tests/integration/test_web_search_e2e.py +""" + +from __future__ import annotations + +import json +import sys +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO_ROOT / "stacklets" / "web" / "cli")) + +pytestmark = pytest.mark.smoke + +SEARCH_BASE = "http://localhost:42080" + + +@pytest.fixture(scope="module", autouse=True) +def require_search(): + """Skip rather than fail when the optional stacklet is not up. + + `web` is opt-in by design -- a family that never searches never + installs it -- so its absence is a valid instance state, not a + broken one. + """ + try: + with urllib.request.urlopen(f"{SEARCH_BASE}/healthz", timeout=5) as response: + if response.status != 200: + pytest.skip("web stacklet is not healthy") + except OSError: + pytest.skip("web stacklet is not running (`stack up web`)") + + +class TestTheJsonApiIsEnabled: + """The overlay's whole job, asserted directly.""" + + def test_a_json_query_is_not_refused(self): + """403 here means `search.formats` is back to html-only and the + settings overlay is not reaching the container.""" + request = urllib.request.Request( + f"{SEARCH_BASE}/search?q=famstack&format=json", + headers={"Accept": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=20) as response: + assert response.status == 200 + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as err: + if err.code == 403: + pytest.fail( + "SearXNG refused format=json (403). `search.formats` is " + "html-only -- the settings overlay at " + "stacklets/web/config/settings.yml is not mounted." + ) + raise + + assert "results" in payload + + def test_the_overlay_kept_the_shipped_engines(self): + """`use_default_settings: true` means the overlay is a patch. + Without it the file would *replace* the image's settings and + take every engine with it -- which looks like "search returns + nothing" rather than like a config error.""" + request = urllib.request.Request( + f"{SEARCH_BASE}/search?q=immich&format=json", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.loads(response.read().decode("utf-8")) + + engines = {r.get("engine") for r in payload.get("results", [])} + assert engines, "no engine answered; the default engine set is gone" + + +class TestTheCommandReturnsUsableResults: + """Driving the CLI's own function, so the test cannot pass while the + command is broken.""" + + def test_results_carry_a_title_and_a_url(self): + from search import search + + results = search("immich vs photoprism", count=5) + + assert results, "no results for a query that certainly has them" + for item in results: + assert item["title"].strip(), f"result with no title: {item}" + assert item["url"].startswith("http"), f"result with no url: {item}" + + def test_the_count_limit_is_honoured(self): + """An agent budgets its context by this number, so it is a + contract rather than a hint.""" + from search import search + + assert len(search("self-hosted photos", count=3)) <= 3 From a721c7eacfd38bbd09ff11cf7eee9ee4d96793da Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 15 Sep 2026 22:59:19 +0200 Subject: [PATCH 2/2] fix(web): pin the SearXNG image and detect engine degradation SearXNG publishes several builds a day and each changes which upstream engines respond, so an unpinned image made search behaviour vary between restarts. Few engines answer on a default install: the Google web engine and Bing ship disabled upstream, and Startpage is behind a captcha. Losing one more leaves search returning fewer results rather than an error, which is invisible to the family. The search lane now fails when fewer than two engines respond, and nothing reads number_of_results, which the API does not populate. The compose audit also treated image: repo/name:${VAR:-latest} as pinned, because the literal tag is not the word "latest". --- docs/design/web/plan.md | 83 ++++++++++++++++++++++-- stacklets/web/docker-compose.yml | 7 +- tests/framework/test_compose_pins.py | 39 +++++++++++ tests/integration/test_web_search_e2e.py | 57 ++++++++++++++++ 4 files changed, 178 insertions(+), 8 deletions(-) diff --git a/docs/design/web/plan.md b/docs/design/web/plan.md index 7d7068d0..79c2a96e 100644 --- a/docs/design/web/plan.md +++ b/docs/design/web/plan.md @@ -266,14 +266,72 @@ Opt-in, off by default. - Pin `browserforge`. The spike hit an import-time failure without a pin. **Verification gate, and this gates the phase.** Must be proven inside a -`linux/arm64` container, not on macOS. The spike ran on macOS arm64. If Chromium -or Camoufox does not run natively in the container, this phase does not ship as -designed, because x86 emulation is what made the SeleniumBase path cost 40s. -Then: decathlon and geizhals both return real content through the service. +`linux/arm64` container, **headless**, not on macOS. The spike ran on macOS +arm64 and headed; every public stealth benchmark is also headed, so headless is +the untested axis and the one we ship. Then: decathlon and geizhals both return +real content through the service. + +Two ceilings to write into the gate rather than discover in it: + +- **`real_chrome` can never be true on arm64.** Patchright's own guidance is to + run real Google Chrome via `channel="chrome"`, and Chrome for Testing + publishes no `linux-arm64` build. On arm64 Playwright falls back to a + Chromium `headless_shell`. The weaker configuration is permanent, not a + setup mistake. +- **Tier 3 is a treadmill, not a milestone.** Cloudflare turned on default + AI-crawler blocking for free plans on 2026-09-15, with Web Bot Auth + (Ed25519-signed requests, a published JWKS, an application process) as the + sanctioned alternative. A self-hosted family stack cannot join that + programme, so the web is splitting into "identify yourself cryptographically" + and "be indistinguishable from a browser", and famstack is structurally on + the second path. Keep tier 3 opt-in, behind the `Transport` seam, degrading + to an honest link card. Budget for it breaking. **Harness improvement.** A `stacktests` case that asserts the gate escalates exactly once and never loops between tier 2 and tier 3. +## Landscape check, 2026-09-15 + +A survey of the agentic-browser and agent-web-access space, assessed against +this stack's constraints (arm64 only, nothing hosted, AGPLv3-compatible, +container weight, a local ~30B model). Three things changed a decision; the +rest confirmed one. + +**Structured data is the right long bet, and the competing standard is not.** +JSON-LD now appears on about 41% of mobile pages and is still growing, which +is why tier 1 reads a recipe deterministically and never asks a model. By +contrast **`llms.txt` is a dud**: across 137,000 domains surveyed, 97% of +`llms.txt` files received zero requests in a month, and most of the fetches +that did happen were not AI tools. Not worth implementing. `NLWeb` (sites +answering natural-language queries over their own schema.org data) is the one +to watch, because it makes the JSON-LD reader more valuable rather than +obsolete. `WebMCP` is a browser-side JavaScript API in a Chrome origin trial, +so it does nothing for a server-side fetcher. + +**Nothing beats trafilatura inside these constraints.** Everything that +measurably wins on extraction quality is a 0.6B transformer needing 1.5 GB of +weights, or x86-only, or non-commercially licensed. Worth knowing that every +benchmark in this space is published by someone shipping a competitor, and the +same library scores 0.924 and 0.6402 depending on who counts. One adjacent +finding: `html2text` (used by `stack.email_message`, not by this module) is the +weakest dependency we have, and `html-to-markdown` replaces it at MIT, zero +Python dependencies, native arm64 wheels, ~7 MB. + +**The capability bar for browser agents is far lower than the marketing.** +ClawBench, 153 everyday tasks across 144 live sites: the best score ever +recorded is 33.3%, and the same models score 65 to 75% on traditional web +benchmarks. Princeton's cost-instrumented leaderboard puts real-web multi-step +success at 40 to 42%, at hundreds of dollars per benchmark run. This is the +evidence behind phase 3 being search snippets plus one model call rather than +multi-step tool use: nobody has a reliable web agent, least of all on a local +model. + +**Licence traps, in the PriceBuddy category.** `Notte` is SSPL. `DrissionPage` +permits non-commercial use only, in Chinese, while GitHub reports it as +`NOASSERTION` so an automated check will not catch it. `rebrowser-patches` has +no licence file at all. `SurfSense` is Apache-2.0 except the directory +containing its SearXNG connector, which is BSL 1.1. + ## What we are explicitly NOT building - **lightpanda.** Solved the challenges on every blocked page and still returned @@ -298,9 +356,20 @@ exactly once and never loops between tier 2 and tier 3. ## Open decisions -1. **Image size budget for `stack-web-fetch`.** Roughly 250 MB estimated, not - measured. If it lands materially higher, consider Camoufox directly instead - of the full Scrapling browser set. +1. ~~**Image size budget for `stack-web-fetch`.**~~ **Resolved, and the + fallback was backwards.** Measured: the official `pyd4vinci/scrapling` + `linux/arm64` image is **644 MB compressed**, of which 441 MB is + `playwright install chromium` and 138 MB is `uv sync --all-extras`. + Installing only `[fetchers]` and `playwright install --only-shell chromium` + puts the floor around **400 MB**. Chromium's own apt dependencies rule out + 250 MB, so the budget moves rather than the design. + + Camoufox is no longer the escape hatch: **Scrapling dropped it entirely at + v0.3.13**, and `StealthyFetcher` is now patchright over Playwright Chromium + with a built-in Turnstile solver. Camoufox's `lin.arm64` asset is **623 MB + zipped on its own**, so "use Camoufox directly" is now a step backwards. + The arm64 story is patchright and Playwright shipping native aarch64 + wheels, not Camoufox's builds. 2. **Does tier 3 stay synchronous?** At 3.4s to 19.8s it fits in a chat round trip behind the existing 👀 ack. If real-world pages cluster at the slow end, it becomes a background job and the reply becomes "fetching, will file it". diff --git a/stacklets/web/docker-compose.yml b/stacklets/web/docker-compose.yml index 7d4af61c..b98319a7 100644 --- a/stacklets/web/docker-compose.yml +++ b/stacklets/web/docker-compose.yml @@ -20,7 +20,12 @@ name: stack-web services: stack-web-search: container_name: stack-web-search - image: searxng/searxng:${WEB_SEARCH_VERSION:-latest} + # Pinned, and deliberately not to a moving tag. SearXNG ships several + # builds a day (three carried 2026-09-15 alone), and each one changes + # which upstream engines work -- the project's whole job is chasing + # other people's bot detection. `:latest` here would mean search + # quietly changing behaviour between two `stack up` runs. + image: searxng/searxng:${WEB_SEARCH_VERSION:-2026.9.15-ca4965040} networks: - stack environment: diff --git a/tests/framework/test_compose_pins.py b/tests/framework/test_compose_pins.py index 06704625..42c9e138 100644 --- a/tests/framework/test_compose_pins.py +++ b/tests/framework/test_compose_pins.py @@ -56,6 +56,12 @@ def _unpinned(text: str) -> list[str]: # `registry:5000/img` is not mistaken for an `img:5000` tag. last_segment = ref.rsplit("/", 1)[-1] tag = last_segment.split(":", 1)[1] if ":" in last_segment else "latest" + # An inline default -- `image: repo/name:${VAR:-latest}` -- renders to + # `latest` whenever the variable is unset, which is the normal case. + # The literal tag text is not "latest", so comparing it alone lets the + # floating tag straight through while looking pinned. + if tag.startswith("${"): + tag = tag.partition(":-")[2].rstrip("}") or "latest" if tag == "latest" and ref not in KNOWN_UNPINNED: found.append(ref) return found @@ -94,3 +100,36 @@ def test_no_floating_image_tags(): "unpinned container images (an unpinned image is a scheduled outage):\n" + "\n".join(f" {p}: {', '.join(refs)}" for p, refs in offenders.items()) ) + + +def test_an_inline_latest_default_is_not_a_pin(): + """`image: repo/name:${VAR:-latest}` is a floating tag wearing a + variable as a disguise. + + The literal tag text is `${VAR:-latest}`, which is not the string + "latest", so a naive comparison reads it as pinned and waves it + through. It renders to `latest` every time the variable is unset, + which is the normal case -- nobody exports it. + + This is a real escape that reached a shipped compose file, not a + hypothetical: the web stacklet used exactly this form. The audit is + only worth its milliseconds if it sees the shapes people actually + write. + """ + floating = "services:\n s:\n image: searxng/searxng:${WEB_SEARCH_VERSION:-latest}" + assert _unpinned(floating) == ["searxng/searxng:${WEB_SEARCH_VERSION:-latest}"] + + +def test_an_inline_default_naming_a_real_version_is_a_pin(): + """The same form with a real default is the correct way to write an + overridable pin, and must not be flagged -- otherwise the fix for + the case above is to stop offering the override at all.""" + pinned = "services:\n s:\n image: searxng/searxng:${WEB_SEARCH_VERSION:-2026.9.15-ca4965040}" + assert _unpinned(pinned) == [] + + +def test_a_whole_ref_variable_is_still_deferred(): + """`image: ${SOME_IMAGE}` puts both name and tag elsewhere; the pin + lives wherever that variable is defined. Tightening the tag check + must not start flagging these.""" + assert _unpinned("services:\n s:\n image: ${SOME_IMAGE}") == [] diff --git a/tests/integration/test_web_search_e2e.py b/tests/integration/test_web_search_e2e.py index efc3159f..c37d818b 100644 --- a/tests/integration/test_web_search_e2e.py +++ b/tests/integration/test_web_search_e2e.py @@ -96,6 +96,63 @@ def test_the_overlay_kept_the_shipped_engines(self): assert engines, "no engine answered; the default engine set is gone" +class TestSearchIsNotQuietlyDegraded: + """SearXNG has no index; it forwards to other engines and merges what + comes back. So "search got worse" does not look like an error, it + looks like fewer results -- and the failure is upstream, continuous, + and not ours to fix. + + Measured on this instance: of the engines enabled by default, only + two actually answer. Google's web engine and Bing ship `disabled: + true` upstream, and Startpage is marked inactive behind a + proof-of-work captcha. Two engines is one bad week from one. + + These assertions exist so that becomes a red test rather than a + family wondering why the answers got worse. + """ + + def test_more_than_one_engine_answers(self): + """A single surviving engine is a working search box and a + broken search. It is also the state that precedes zero.""" + request = urllib.request.Request( + f"{SEARCH_BASE}/search?q=self+hosted+photos&format=json", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.loads(response.read().decode("utf-8")) + + engines = set() + for item in payload.get("results", []): + engines.update(item.get("engines") or []) + + assert len(engines) >= 2, ( + f"only {engines or 'no'} engine(s) answered. Upstream engines " + "break continuously; check searx.engines in the container log " + "and the enabled set in config/settings.yml." + ) + + def test_number_of_results_is_not_used_as_a_count(self): + """A trap worth pinning rather than remembering. + + SearXNG reports `number_of_results` as 0 or null while the + `results` array holds a full page. Anything that gates on it + reports "no results" for a successful search, and the bug looks + like an upstream outage rather than a field misread. + """ + request = urllib.request.Request( + f"{SEARCH_BASE}/search?q=immich&format=json", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.loads(response.read().decode("utf-8")) + + assert payload.get("results"), "no results to make the point with" + assert not payload.get("number_of_results"), ( + "number_of_results became truthful -- if upstream fixed it, this " + "test can go, but until then nothing may branch on it" + ) + + class TestTheCommandReturnsUsableResults: """Driving the CLI's own function, so the test cannot pass while the command is broken."""