diff --git a/docs/design/web/plan.md b/docs/design/web/plan.md
index 19bb2773..9c2a7dca 100644
--- a/docs/design/web/plan.md
+++ b/docs/design/web/plan.md
@@ -131,6 +131,37 @@ Other measurements:
Camoufox publishes `lin.arm64`; Playwright's Chromium builds arm64. This is the
single biggest reason Scrapling wins.
+## Re-validation, 2026-09-15
+
+Re-probed before building Phase 1, same two UAs, anonymous, German IP.
+Three findings held, one moved.
+
+| Claim | Still true? |
+|---|---|
+| decathlon.de blocks a plain fetch | yes — 403, Cloudflare `Just a moment...` |
+| Recipe JSON-LD on essen-und-trinken / einfachkochen | yes — full `Recipe`, yield, totalTime, 11 and 14 ingredients, steps, nutrition |
+| geizhals.de returns 403 | **no** — the *homepage* serves 200 and its real listing. The measured 403 was a product/listing URL; the row overstated it as the whole domain |
+| `old.reddit.com` + Chrome UA returns the post | **no — reversed** |
+
+**Reddit now walls anonymous readers.** `old.reddit.com` answers a 302 to
+`/login/?reason=lor2`, and `www.reddit.com` serves an 8 KB JavaScript
+shell. The `.json` endpoint is 403 with a 190 KB HTML block page.
+
+This is worth more than a corrected row, because of the *shape* of the
+failure. The login redirect ends on HTTP 200 with a 320 KB body and a
+friendly `Welcome to Reddit` — which is exactly what a
+"did the extractor return a string?" success check reads as an article.
+The drift did not break the gate's design, it validated it: `login` was
+already in the verdict set, and the landing URL is the only honest
+signal on that page. The gate reads the URL a fetch *ended* on for
+precisely this reason.
+
+The reddit profile keeps its `old.reddit` rewrite anyway. It no longer
+recovers the post, but it moves the reported reason from `empty` ("the
+page was blank") to `login` ("reddit wants you signed in"), which is the
+one a person can act on — and it starts working again unchanged if
+reddit relaxes.
+
## Decision: one stacklet, but not for everything
The tempting version is a `web` stacklet that owns all web operations including
@@ -158,7 +189,7 @@ construction: a family that never pastes a shop link never downloads Chromium.
## Phases
-### Phase 1 — Framework module and the gate (about 1 day)
+### Phase 1 — Framework module and the gate — SHIPPED
The whole fix for both reported bugs, with no new container.
diff --git a/lib/stack/web/__init__.py b/lib/stack/web/__init__.py
new file mode 100644
index 00000000..2e4c0261
--- /dev/null
+++ b/lib/stack/web/__init__.py
@@ -0,0 +1,30 @@
+"""How famstack reads the web.
+
+A pasted link becomes a good vault entry or an honest link card, and
+never a cookie policy. The ladder that decides which is in `fetch`; the
+rule that nothing unjudged reaches the vault is in `quality`.
+
+ content SourceContent, the shape every capture source produces
+ quality the gate: ok / challenge / login / consent / paywall / empty
+ profiles per-domain rules — canonical URL, extraction tuning
+ structured JSON-LD Recipe and Article, read without an LLM
+ fetch the ladder that runs them in order
+
+See `docs/design/web/plan.md` for the measurements behind each tier.
+"""
+
+from stack.web.content import SourceContent
+from stack.web.fetch import FetchOutcome, fetch_url
+from stack.web.profiles import canonicalize, profile_for
+from stack.web.quality import Page, Verdict, assess
+
+__all__ = [
+ "FetchOutcome",
+ "Page",
+ "SourceContent",
+ "Verdict",
+ "assess",
+ "canonicalize",
+ "fetch_url",
+ "profile_for",
+]
diff --git a/lib/stack/web/content.py b/lib/stack/web/content.py
new file mode 100644
index 00000000..ae61569a
--- /dev/null
+++ b/lib/stack/web/content.py
@@ -0,0 +1,37 @@
+"""`SourceContent` — the classifier's input, normalized across sources.
+
+Promoted here from `stacklets/docs/bot/extractors.py` because it stopped
+being one stacklet's type the moment a second consumer needed it: the
+host CLI (`stack web fetch`) builds one without the archivist running at
+all. Same move as `stack.email_message`, same reason -- the framework
+owns the shared shape, each stacklet owns its own mapping into it.
+
+The archivist's classifier does not care whether text arrived from a
+photographed receipt that Paperless OCR'd, a pasted URL that trafilatura
+rendered, or a wall of text somebody typed. This is what all of those
+agree to produce.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class SourceContent:
+ """The classifier's input, normalized across source types.
+
+ `text` is the body the classifier reads — Markdown when the
+ extractor can produce it, plain text otherwise. `title_hint` is
+ whatever the source advertised as a title (HTML ``, first
+ body line, filename); the classifier may overwrite it with
+ something more useful. `source_uri` is the canonical pointer
+ back to the origin (`https://...`, `paperless://42`,
+ `matrix:<event-id>` — caller decides the scheme), captured into
+ the mirror's frontmatter for round-tripping. None means the
+ capture has no upstream pointer (a pure pasted note).
+ """
+ text: str
+ mime: str = "text/plain"
+ title_hint: str | None = None
+ source_uri: str | None = None
diff --git a/lib/stack/web/fetch.py b/lib/stack/web/fetch.py
new file mode 100644
index 00000000..537ba58a
--- /dev/null
+++ b/lib/stack/web/fetch.py
@@ -0,0 +1,306 @@
+"""The fetch ladder — cheap first, expensive only on proven failure.
+
+Four tiers, and the whole design is in which order they run:
+
+ 0 canonicalize strip trackers, rewrite hosts ~0 ms
+ 1 structured the page's own JSON-LD ~0.5 s
+ 2 plain HTTP browser headers, then trafilatura ~0.5 s
+ 3 stealth a real browser, in the web stacklet 3.4-19.8 s
+
+Tiers 0 to 2 are pure Python over bytes. They carry the common case,
+they need nothing running, and they are why `stack web fetch` works on
+a machine with no containers up. Tier 3 is the only step that needs a
+browser, it lives in an optional stacklet, and it is reached only when
+the gate says bot protection is in the way — never by default, and
+never for a failure a browser cannot fix.
+
+The gate runs after every tier that produces content, including tier 3.
+An expensive fetch does not get to skip the check: a stealth browser
+can be served a challenge page too, and an unjudged 19-second result is
+no better than an unjudged instant one.
+
+Transport is injected rather than imported. The bot has an aiohttp
+session already open and should reuse it; the host CLI has no aiohttp
+at all. Both hand in a callable, so this module depends on neither.
+
+trafilatura is imported lazily for the same reason: the gate, the
+profiles and the JSON-LD reader are stdlib, and a caller that never
+reaches tier 2 never needs the dependency.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Awaitable, Callable
+
+from stack.web.content import SourceContent
+from stack.web.profiles import Profile, canonicalize, profile_for, title_from_url
+from stack.web.quality import Page, Verdict, assess, page_title
+
+# The headers a browser sends. Not a disguise — plenty of CDNs answer a
+# bare library user-agent with a 403 out of habit, and this is the
+# difference between reading a public page and not. Sites that actually
+# check (decathlon) are unmoved by it; that is what tier 3 is for.
+BROWSER_HEADERS = {
+ "User-Agent": (
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+ "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
+ ),
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
+}
+
+
+@dataclass
+class Response:
+ """What a transport hands back. Deliberately minimal — the ladder
+ needs the landing URL (a login redirect is only visible there), the
+ status, the body and the content type, and nothing else."""
+
+ url: str
+ status: int
+ html: str
+ content_type: str = "text/html"
+
+
+# A transport takes a URL plus headers and returns a Response, or None
+# when the request could not be made at all (DNS, timeout, refused).
+Transport = Callable[[str, dict], Awaitable["Response | None"]]
+
+
+@dataclass
+class FetchOutcome:
+ """The result of running the ladder.
+
+ `content` is present only when the gate passed. `verdict` is always
+ present and always explains itself, so a caller rendering a link
+ card has something true to say about why there is no entry.
+
+ `tier` records which rung produced the outcome, for the logs: a
+ site that starts needing tier 3 is worth noticing.
+ """
+
+ verdict: Verdict
+ content: SourceContent | None = None
+ url: str = ""
+ tier: str = ""
+ profile: str = "default"
+
+ @property
+ def ok(self) -> bool:
+ return self.content is not None and self.verdict.ok
+
+
+# ── Tier 2 extraction ─────────────────────────────────────────────────
+
+def extract_body(
+ html: str, *, favor_recall: bool = False, include_comments: bool = False,
+) -> str | None:
+ """HTML to Markdown via trafilatura, tuned by profile.
+
+ Precision is the default because most pages are articles surrounded
+ by furniture. Recall is for pages whose content *is* the discussion
+ under them — measured on a Reddit post, precision returned 821
+ characters of sidebar and recall returned the 5513-character post.
+ The per-domain profile decides; this function only applies it.
+
+ Returns None when trafilatura is unavailable or finds no body. The
+ caller treats both as "tier 2 produced nothing" and lets the gate
+ name it, so a missing dependency degrades to a link card rather
+ than an exception.
+ """
+ try:
+ import trafilatura
+ except ImportError:
+ return None
+
+ kwargs = {
+ "output_format": "markdown",
+ "include_links": True,
+ "include_images": False,
+ "include_tables": True,
+ "include_comments": include_comments,
+ }
+ # trafilatura rejects both favour flags at once; recall wins where a
+ # profile asked for it, precision is the default everywhere else.
+ if favor_recall:
+ kwargs["favor_recall"] = True
+ else:
+ kwargs["favor_precision"] = True
+
+ body = trafilatura.extract(html, **kwargs)
+ return body.strip() if body and body.strip() else None
+
+
+def _is_html(content_type: str) -> bool:
+ ctype = (content_type or "").lower()
+ return ctype.startswith("text/html") or ctype.startswith("application/xhtml")
+
+
+# ── The ladder ────────────────────────────────────────────────────────
+
+async def fetch_url(
+ url: str,
+ *,
+ transport: Transport,
+ stealth: Transport | None = None,
+) -> FetchOutcome:
+ """Run the ladder and return a judged outcome. Never raises.
+
+ `stealth` is tier 3. Passing None — which is what happens when the
+ `web` stacklet is not installed — means a challenge is terminal and
+ the caller renders a link card saying so. That is the honest
+ degradation: the family is told the site blocked us, rather than
+ getting a silent failure or an entry built from the challenge page.
+ """
+ target = canonicalize(url)
+ profile = profile_for(target)
+
+ # Tier 0.5 — a shell page whose subject is in the path. Google Maps
+ # is the case: fetching it is pointless, the document is an empty
+ # application and the place name is already in the URL we were
+ # handed. Answering from the URL beats answering from the page.
+ shell_title = title_from_url(target)
+
+ response = await _safe(transport, target)
+ if response is None:
+ if shell_title:
+ return _from_url_only(target, shell_title, profile)
+ return FetchOutcome(
+ verdict=Verdict("empty", "the site could not be reached"),
+ url=target, tier="2", profile=profile.name,
+ )
+
+ outcome = _judge(response, profile)
+ if outcome.ok or not outcome.verdict.escalate or stealth is None:
+ if not outcome.ok and shell_title:
+ return _from_url_only(target, shell_title, profile)
+ return outcome
+
+ # Tier 3 — only for a challenge, only once. There is no loop back to
+ # tier 2: if the browser is also served a challenge, the answer is
+ # that we cannot read this page.
+ stealthed = await _safe(stealth, response.url or target)
+ if stealthed is None:
+ return outcome
+ escalated = _judge(stealthed, profile, tier="3")
+ return escalated if escalated.ok else outcome
+
+
+async def _safe(transport: Transport, url: str) -> Response | None:
+ """Run a transport, turning any transport-level failure into None.
+
+ The ladder's contract is that it never raises; a caller rendering a
+ chat reply has nothing useful to do with a socket error.
+ """
+ try:
+ return await transport(url, dict(BROWSER_HEADERS))
+ except Exception: # noqa: BLE001 — transports raise library-specific errors
+ return None
+
+
+def _judge(response: Response, profile: Profile, *, tier: str = "2") -> FetchOutcome:
+ """Tiers 1 and 2 over one response, then the gate.
+
+ Structured data is tried first and short-circuits: when a site hands
+ us its own `Recipe` object there is no reason to guess at the
+ rendered page, and no reason to ask the gate whether the guess was
+ any good.
+ """
+ if not _is_html(response.content_type):
+ return FetchOutcome(
+ verdict=Verdict("empty", f"the URL served {response.content_type}, not a web page"),
+ url=response.url, tier=tier, profile=profile.name,
+ )
+
+ structured = _structured(response)
+ if structured is not None:
+ return FetchOutcome(
+ verdict=Verdict("ok", "the page published its own structured data"),
+ content=structured, url=response.url, tier="1", profile=profile.name,
+ )
+
+ body = extract_body(
+ response.html,
+ favor_recall=profile.favor_recall,
+ include_comments=profile.include_comments,
+ )
+ page = Page(
+ url=response.url, status=response.status, html=response.html,
+ text=body or "", content_type=response.content_type,
+ )
+ verdict = assess(page, min_chars=profile.min_chars)
+ if not verdict.ok:
+ detail = verdict.detail
+ if profile.blocked_note:
+ detail = f"{detail} — {profile.blocked_note}"
+ return FetchOutcome(
+ verdict=Verdict(verdict.name, detail),
+ url=response.url, tier=tier, profile=profile.name,
+ )
+
+ return FetchOutcome(
+ verdict=verdict,
+ content=SourceContent(
+ text=body or "",
+ mime="text/html",
+ title_hint=page_title(response.html),
+ source_uri=response.url,
+ ),
+ url=response.url, tier=tier, profile=profile.name,
+ )
+
+
+def _structured(response: Response) -> SourceContent | None:
+ from stack.web.structured import read_structured
+
+ return read_structured(response.html, url=response.url)
+
+
+def _from_url_only(url: str, title: str, profile: Profile) -> FetchOutcome:
+ """An entry built from the URL, for pages that have no text to read.
+
+ The body is the one true sentence we can say about the place: its
+ name and where the link points. The classifier gets a real title
+ instead of "Google Maps", and the family gets an entry they can
+ find again.
+ """
+ return FetchOutcome(
+ verdict=Verdict("ok", "the page has no text; its subject came from the URL"),
+ content=SourceContent(
+ text=f"{title}\n\n{url}",
+ mime="text/markdown",
+ title_hint=title,
+ source_uri=url,
+ ),
+ url=url, tier="0", profile=profile.name,
+ )
+
+
+# ── aiohttp transport ─────────────────────────────────────────────────
+
+def aiohttp_transport(session, *, timeout: int = 30) -> Transport:
+ """A transport over an already-open aiohttp session.
+
+ The bot keeps one session for its lifetime; handing it in rather
+ than opening a new one per fetch keeps connection reuse and, more
+ importantly, keeps this module free of an aiohttp import at module
+ level so the host CLI can import the ladder without it.
+ """
+ async def _fetch(url: str, headers: dict) -> Response | None:
+ import aiohttp
+
+ async with session.get(
+ url,
+ timeout=aiohttp.ClientTimeout(total=timeout),
+ allow_redirects=True,
+ headers=headers,
+ ) as resp:
+ return Response(
+ url=str(resp.url),
+ status=resp.status,
+ html=await resp.text(errors="replace"),
+ content_type=resp.content_type or "",
+ )
+
+ return _fetch
diff --git a/lib/stack/web/profiles.py b/lib/stack/web/profiles.py
new file mode 100644
index 00000000..540b6d71
--- /dev/null
+++ b/lib/stack/web/profiles.py
@@ -0,0 +1,176 @@
+"""Per-domain rules — the part of web reading that is not general.
+
+Two things vary by site and nothing else does: which URL actually
+serves the content, and how hard to pull on the extractor. Both are
+small, both are data, and both belong somewhere a new site costs one
+entry rather than a branch in the fetch ladder.
+
+Tier 0 is the canonical URL. It is free, it runs before any request,
+and it is the single highest-yield step in the whole ladder: stripping
+a tracker parameter turns two URLs into one cache key, and rewriting a
+host can turn a JavaScript shell into a document.
+
+Stdlib only: the host CLI runs this without a virtualenv.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from urllib.parse import parse_qsl, unquote, urlencode, urlsplit, urlunsplit
+
+from stack.web.quality import DEFAULT_MIN_CHARS
+
+
+# ── Tracking parameters ───────────────────────────────────────────────
+#
+# Campaign and referrer tags identify the person who shared the link,
+# not the page. Dropping them is a privacy measure as much as a
+# normalisation one: the URL ends up in the vault and in chat, and it
+# should not carry "Homer clicked this from a newsletter" with it.
+
+_TRACKING_PREFIXES = ("utm_", "pk_", "mtm_", "matomo_", "ga_", "_hs")
+_TRACKING_PARAMS = frozenset({
+ "fbclid", "gclid", "dclid", "msclkid", "twclid", "igshid", "ttclid",
+ "mc_cid", "mc_eid", "ref", "ref_src", "ref_url", "referrer", "source",
+ "share_id", "si", "spm", "cmpid", "icid", "trk", "yclid", "vero_id",
+})
+
+
+def _is_tracking(key: str) -> bool:
+ lowered = key.lower()
+ return lowered in _TRACKING_PARAMS or lowered.startswith(_TRACKING_PREFIXES)
+
+
+def strip_tracking(url: str) -> str:
+ """Drop campaign/referrer parameters, keep everything else.
+
+ Conservative on purpose. A query string is often load-bearing (an
+ article id, a search term, a page number), so only known-tracking
+ keys go; anything unrecognised stays.
+ """
+ split = urlsplit(url)
+ if not split.query:
+ return url
+ kept = [(k, v) for k, v in parse_qsl(split.query, keep_blank_values=True)
+ if not _is_tracking(k)]
+ return urlunsplit(split._replace(query=urlencode(kept)))
+
+
+# ── Profiles ──────────────────────────────────────────────────────────
+
+@dataclass(frozen=True)
+class Profile:
+ """What we know about reading one family of sites.
+
+ `hosts` are matched as suffixes, so a profile for `reddit.com`
+ covers `old.reddit.com` and `www.reddit.com` alike.
+
+ `rewrite_host` swaps the host before fetching. `favor_recall` and
+ `include_comments` are handed to trafilatura: precision is the right
+ default for an article, recall is the right default for a page whose
+ content *is* the discussion under it.
+
+ `min_chars` lowers the gate's empty floor for sites that genuinely
+ publish short pages.
+
+ `blocked_note` is prose shown to the family when this site fails the
+ gate. Generic advice ("the site blocked us") is worse than naming
+ the actual reason where we already know it.
+ """
+
+ name: str
+ hosts: tuple[str, ...] = ()
+ rewrite_host: str | None = None
+ favor_recall: bool = False
+ include_comments: bool = False
+ min_chars: int = DEFAULT_MIN_CHARS
+ blocked_note: str | None = None
+ # Path prefixes whose content lives in the URL rather than the body.
+ # A Google Maps place is the worked example: the document is an
+ # empty application shell, and the place name is right there in the
+ # path we were given.
+ title_from_path: tuple[str, ...] = field(default=())
+
+
+DEFAULT = Profile(name="default")
+
+PROFILES: tuple[Profile, ...] = (
+ # Reddit serves anonymous readers a JavaScript shell on `www` and,
+ # since measurement, a 302 to `/login?reason=lor2` on `old`. Neither
+ # yields a post. The rewrite is kept anyway, and deliberately: it
+ # moves the failure from `empty` ("the page was blank", true but
+ # unhelpful) to `login` ("reddit wants you signed in", actionable).
+ # If reddit relaxes anonymous access, the rewrite starts working
+ # again with no change here. Recall settings are ready for that day
+ # — measured at the time: precision returned 821 characters of
+ # sidebar, recall returned the 5513-character post.
+ Profile(
+ name="reddit",
+ hosts=("reddit.com",),
+ rewrite_host="old.reddit.com",
+ favor_recall=True,
+ include_comments=True,
+ blocked_note="reddit no longer serves posts to readers who are not signed in",
+ ),
+ # Google Maps place URLs are an application shell: HTTP 200, a full
+ # document, and no prose but the site-wide meta description. The
+ # place name is in the path, so the URL is the content.
+ Profile(
+ name="google-maps",
+ hosts=("google.com", "google.de", "maps.app.goo.gl"),
+ title_from_path=("/maps/place/",),
+ blocked_note="Google Maps renders in the browser, so there is no page text to file",
+ ),
+)
+
+
+def profile_for(url: str) -> Profile:
+ """The profile governing this URL, or the default.
+
+ Host suffix match, so `old.reddit.com` and `www.reddit.com` both
+ resolve to the reddit profile.
+ """
+ host = urlsplit(url).netloc.lower().split(":")[0]
+ for profile in PROFILES:
+ if any(host == h or host.endswith("." + h) for h in profile.hosts):
+ return profile
+ return DEFAULT
+
+
+# ── Tier 0 ────────────────────────────────────────────────────────────
+
+def canonicalize(url: str) -> str:
+ """The URL we should actually fetch. Costs nothing, runs first.
+
+ Strips tracking parameters and applies the profile's host rewrite.
+ Shortener resolution is *not* done here: it needs a request, so it
+ belongs to the fetch step, and this function stays pure.
+ """
+ cleaned = strip_tracking(url.strip())
+ profile = profile_for(cleaned)
+ if profile.rewrite_host:
+ split = urlsplit(cleaned)
+ if split.netloc.lower() != profile.rewrite_host:
+ cleaned = urlunsplit(split._replace(netloc=profile.rewrite_host))
+ return cleaned
+
+
+def title_from_url(url: str) -> str | None:
+ """A human title recovered from the path, for shell pages.
+
+ Returns None unless the URL matches a profile's `title_from_path`,
+ so this never guesses at an ordinary article URL — a slug makes a
+ poor title when the document has a real one.
+ """
+ profile = profile_for(url)
+ if not profile.title_from_path:
+ return None
+ path = urlsplit(url).path
+ for prefix in profile.title_from_path:
+ if prefix not in path:
+ continue
+ tail = path.split(prefix, 1)[1].split("/")[0]
+ name = unquote(tail).replace("+", " ").strip()
+ if name:
+ return name
+ return None
diff --git a/lib/stack/web/quality.py b/lib/stack/web/quality.py
new file mode 100644
index 00000000..93c64d03
--- /dev/null
+++ b/lib/stack/web/quality.py
@@ -0,0 +1,219 @@
+"""The quality gate — what may become a vault entry, and what may not.
+
+The archivist's URL capture used to succeed whenever the extractor
+returned a non-empty string. Three different pages return a non-empty
+string without being articles:
+
+ a Cloudflare interstitial 403, "Just a moment...", a spinner
+ a login wall 200, 320 KB, titled "Welcome to Reddit"
+ a JavaScript shell 200, a full document, no prose
+
+All three were filed, summarised by the classifier, and given a title.
+The family ended up with wiki entries about cookie policies. This
+module is the fix: nothing reaches the vault without being named first.
+
+The verdict is a value rather than a boolean because two readers need
+it. The chat reply says something different for "this site wants you
+logged in" than for "this site is checking your browser", and whoever
+reads the logs later needs to know which sites fail which way.
+
+ ok there is a body worth filing
+ challenge bot protection is between us and the page
+ login the site redirected us to a sign-in
+ consent a cookie or consent wall is being served instead
+ paywall the content exists but is not ours to read
+ empty we got a page and it had nothing on it
+
+Only `challenge` is worth escalating to a browser tier. The rest are
+terminal no matter how expensively we fetch them, which is what keeps
+tier 3 from being a retry loop.
+
+Detection keys on structure, never on prose. An article *about*
+Cloudflare contains every word its challenge page does; what it does not
+contain is Cloudflare's DOM ids, its challenge host, or that exact
+`<title>`. Where a structural signal exists off-page -- the URL a
+redirect chain ended on -- it is preferred, because it is the one part
+of a JavaScript shell that cannot be styled away.
+
+Stdlib only: the host CLI runs this without a virtualenv.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+# A body shorter than this is furniture — a nav rail, a cookie notice,
+# a subreddit sidebar. Measured: trafilatura on a blocked Reddit page
+# returned 821 characters of sidebar and read as content to everything
+# downstream. Callers may lower it per profile; see `assess`.
+DEFAULT_MIN_CHARS = 250
+
+
+@dataclass(frozen=True)
+class Page:
+ """One fetched page, as the gate sees it.
+
+ `url` is the URL the fetch *ended* on, not the one it started
+ with. A login wall is a redirect, so the requested URL says nothing
+ and the landing URL says everything.
+
+ `text` is the extracted body when tier 2 has run, and empty before
+ that. `html` is always the raw document: the structural markers the
+ gate reads (a `<title>`, a challenge script host) are stripped out
+ by extraction, so both are needed.
+ """
+
+ url: str
+ status: int = 200
+ html: str = ""
+ text: str = ""
+ content_type: str = "text/html"
+
+
+@dataclass(frozen=True)
+class Verdict:
+ """A named outcome plus the sentence that explains it."""
+
+ name: str
+ detail: str
+
+ @property
+ def ok(self) -> bool:
+ return self.name == "ok"
+
+ @property
+ def escalate(self) -> bool:
+ """Whether a browser tier could plausibly do better.
+
+ Only bot protection qualifies. A paywall, a login wall and an
+ empty page cost the same in a headless browser as they do over
+ plain HTTP, so escalating them would buy a slower failure.
+ """
+ return self.name == "challenge"
+
+
+# ── Challenge ─────────────────────────────────────────────────────────
+#
+# Cloudflare's interstitial is the one we measured, and it identifies
+# itself three ways that survive a localisation change: the element id
+# it puts its error text in, the host it loads the widget from, and the
+# managed-challenge title. Any one is conclusive; prose is not used.
+
+_CHALLENGE_MARKERS = (
+ "challenges.cloudflare.com",
+ "challenge-platform",
+ "cf-browser-verification",
+ "challenge-error-text",
+ "_cf_chl_opt",
+)
+
+# Exact titles served by challenge pages. Compared whole, so a post
+# titled "How Cloudflare's Just a moment page works" does not match.
+_CHALLENGE_TITLES = {
+ "just a moment...",
+ "just a moment",
+ "attention required! | cloudflare",
+ "access denied",
+ "checking your browser before accessing",
+ "one more step",
+}
+
+_TITLE_RE = re.compile(r"<title[^>]*>([^<]*)", re.IGNORECASE | re.DOTALL)
+
+
+def page_title(html: str) -> str | None:
+ """The document's ``, trimmed. None when absent or blank."""
+ match = _TITLE_RE.search(html or "")
+ if not match:
+ return None
+ return match.group(1).strip() or None
+
+
+def _is_challenge(page: Page) -> str | None:
+ title = (page_title(page.html) or "").strip().lower()
+ if title in _CHALLENGE_TITLES:
+ return f"bot protection served {title!r} instead of the page"
+ for marker in _CHALLENGE_MARKERS:
+ if marker in page.html:
+ return f"bot protection detected ({marker})"
+ return None
+
+
+# ── Login ─────────────────────────────────────────────────────────────
+#
+# Detected from the landing URL, because that is where the signal
+# actually is. Reddit's wall is the worked example: HTTP 200, a large
+# document, a friendly title, and the only honest thing about it is
+# that the redirect chain ended on `/login`.
+
+_LOGIN_PATHS = ("/login", "/signin", "/sign-in", "/sign_in", "/auth/login", "/accounts/login")
+
+
+def _is_login(page: Page) -> str | None:
+ split = urlsplit(page.url or "")
+ path = split.path.rstrip("/").lower()
+ if any(path == p or path.endswith(p) for p in _LOGIN_PATHS):
+ return f"the site redirected to a sign-in page ({split.netloc}{split.path})"
+ if page.status in (401, 403) and "login" in split.query.lower():
+ return "the site requires a sign-in"
+ return None
+
+
+# ── Consent ───────────────────────────────────────────────────────────
+#
+# A consent *wall* is a different host, not a banner. Nearly every
+# European site carries a banner in the same document as its content;
+# treating those as blocks would reject most of the web. So the signal
+# is the dedicated consent host a site bounces to.
+
+_CONSENT_HOSTS = ("consent.google.com", "consent.youtube.com", "consent.yahoo.com")
+
+
+def _is_consent(page: Page) -> str | None:
+ host = urlsplit(page.url or "").netloc.lower()
+ if host in _CONSENT_HOSTS:
+ return f"the site served a consent wall at {host}"
+ return None
+
+
+# ── Paywall ───────────────────────────────────────────────────────────
+#
+# 402 is the unambiguous case and the only one detected here. Prose
+# detection ("Subscribe to continue reading") is deliberately absent:
+# most paywalled pages carry that string *alongside* a readable
+# excerpt, and an excerpt is still worth filing.
+
+def _is_paywall(page: Page) -> str | None:
+ if page.status == 402:
+ return "the site returned 402 Payment Required"
+ return None
+
+
+# ── The gate ──────────────────────────────────────────────────────────
+
+def assess(page: Page, *, min_chars: int = DEFAULT_MIN_CHARS) -> Verdict:
+ """Classify a fetched page. Never raises.
+
+ Order is deliberate. A challenge page is also short, and a login
+ wall is also nearly empty, so `empty` has to be the last thing
+ checked -- otherwise every block would be reported as "the page was
+ blank", which is true and useless.
+
+ `min_chars` is the floor below which a body is assumed to be
+ furniture rather than an article. It is an argument so a site
+ profile can lower it for a domain that legitimately publishes short
+ pages, instead of that domain having to skip the gate.
+ """
+ for check in (_is_challenge, _is_login, _is_consent, _is_paywall):
+ detail = check(page)
+ if detail:
+ return Verdict(check.__name__.removeprefix("_is_"), detail)
+
+ body = (page.text or "").strip()
+ if len(body) < min_chars:
+ got = f"{len(body)} characters" if body else "nothing"
+ return Verdict("empty", f"the page yielded {got}, below the {min_chars}-character floor")
+
+ return Verdict("ok", f"extracted {len(body)} characters")
diff --git a/lib/stack/web/structured.py b/lib/stack/web/structured.py
new file mode 100644
index 00000000..9a881ade
--- /dev/null
+++ b/lib/stack/web/structured.py
@@ -0,0 +1,236 @@
+"""JSON-LD — the content a site already handed us, structured.
+
+Recipe sites publish schema.org markup because Google asks them to.
+That markup is better than anything an extractor can recover from the
+rendered page: the ingredients are a list rather than a paragraph, the
+quantities are attached to them, and the steps are in order. It is also
+free, deterministic, and needs no model.
+
+Measured on two German recipe sites: both served a complete `Recipe`
+object to a plain fetch with no bot wall — yield, total time, 11 to 14
+ingredients with quantities, the instruction steps, and nutrition.
+
+So this runs before general extraction, and when it hits, the page is
+never parsed as prose at all.
+
+The awkward part is that a listing page carries `Recipe` objects too:
+"our 30 best apple cakes" embeds thirty of them, and filing that as a
+recipe would produce an entry with the ingredients of whichever one
+happened to be first. So a candidate is only accepted when it looks
+like the page's *subject* — it must carry both ingredients and steps.
+
+Stdlib only: the host CLI runs this without a virtualenv.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Iterator
+
+from stack.web.content import SourceContent
+
+_LD_BLOCK_RE = re.compile(
+ r'<script[^>]*type\s*=\s*["\']application/ld\+json["\'][^>]*>(.*?)</script>',
+ re.IGNORECASE | re.DOTALL,
+)
+
+
+def _blocks(html: str) -> Iterator[Any]:
+ """Every parseable ld+json payload in the document.
+
+ Sites ship malformed JSON more often than you would hope (trailing
+ commas, unescaped newlines in a description). One bad block must not
+ cost us a good one, so parse failures are skipped silently.
+ """
+ for raw in _LD_BLOCK_RE.findall(html or ""):
+ try:
+ yield json.loads(raw)
+ except (ValueError, TypeError):
+ continue
+
+
+def _objects(node: Any) -> Iterator[dict]:
+ """Walk a payload yielding every object in it.
+
+ JSON-LD nests three ways in the wild — a bare object, a top-level
+ array, and a `@graph` list — and sites mix them. Walking everything
+ is shorter than handling each shape and cannot miss one.
+ """
+ if isinstance(node, dict):
+ yield node
+ for value in node.values():
+ yield from _objects(value)
+ elif isinstance(node, list):
+ for item in node:
+ yield from _objects(item)
+
+
+def _has_type(obj: dict, wanted: str) -> bool:
+ raw = obj.get("@type")
+ types = raw if isinstance(raw, list) else [raw]
+ return any(isinstance(t, str) and t.lower() == wanted.lower() for t in types)
+
+
+def _text(value: Any) -> str:
+ """Flatten a schema.org value into a string.
+
+ Handles the shapes sites actually publish: a plain string, a bare
+ number (`"recipeYield": 4` is common — the field is typed as text
+ in the spec and half the web ignores that), an object carrying a
+ `name`/`text`, or a list of any of those.
+ """
+ if isinstance(value, str):
+ return value.strip()
+ if isinstance(value, bool):
+ return ""
+ if isinstance(value, (int, float)):
+ return str(value)
+ if isinstance(value, dict):
+ for key in ("text", "name", "description"):
+ got = value.get(key)
+ if isinstance(got, str) and got.strip():
+ return got.strip()
+ return ""
+ if isinstance(value, list):
+ return "\n".join(filter(None, (_text(v) for v in value)))
+ return ""
+
+
+def _steps(value: Any) -> list[str]:
+ """Instruction steps, flattened out of however they were nested.
+
+ `recipeInstructions` is a list of strings, a list of `HowToStep`
+ objects, a list of `HowToSection` objects each holding steps, or a
+ single paragraph. Sections are flattened rather than preserved: the
+ vault entry is prose, not a structured recipe format.
+ """
+ if isinstance(value, str):
+ return [line.strip() for line in value.splitlines() if line.strip()]
+ if isinstance(value, dict):
+ if _has_type(value, "HowToSection"):
+ return _steps(value.get("itemListElement") or [])
+ got = _text(value)
+ return [got] if got else []
+ if isinstance(value, list):
+ out: list[str] = []
+ for item in value:
+ out.extend(_steps(item))
+ return out
+ return []
+
+
+def _duration(iso: Any) -> str | None:
+ """`PT1H15M` as `1 h 15 min`. Returns None for anything else.
+
+ Deliberately narrow: ISO 8601 durations have a full grammar, and
+ recipes only ever use hours and minutes. Anything unrecognised is
+ dropped rather than guessed at.
+ """
+ if not isinstance(iso, str):
+ return None
+ match = re.fullmatch(r"PT(?:(\d+)H)?(?:(\d+)M)?", iso.strip(), re.IGNORECASE)
+ if not match or not any(match.groups()):
+ return None
+ hours, minutes = match.groups()
+ parts = []
+ if hours:
+ parts.append(f"{int(hours)} h")
+ if minutes:
+ parts.append(f"{int(minutes)} min")
+ return " ".join(parts)
+
+
+# ── Recipe ────────────────────────────────────────────────────────────
+
+def _recipe_markdown(obj: dict) -> str | None:
+ """Render a `Recipe` object as the Markdown that goes in the vault.
+
+ None when the object is a mention rather than the page's subject —
+ a listing page's thumbnails carry a name and an image but no
+ ingredients, and filing one of those would produce an entry named
+ after a recipe it does not contain.
+ """
+ ingredients = [_text(i) for i in (obj.get("recipeIngredient") or [])]
+ ingredients = [i for i in ingredients if i]
+ steps = [s for s in _steps(obj.get("recipeInstructions") or []) if s]
+ if not ingredients or not steps:
+ return None
+
+ lines: list[str] = []
+ description = _text(obj.get("description"))
+ if description:
+ lines += [description, ""]
+
+ facts = []
+ servings = _text(obj.get("recipeYield"))
+ if servings:
+ facts.append(f"**Servings:** {servings}")
+ total = _duration(obj.get("totalTime")) or _duration(obj.get("cookTime"))
+ if total:
+ facts.append(f"**Total time:** {total}")
+ if facts:
+ lines += [" · ".join(facts), ""]
+
+ lines += ["## Ingredients", ""]
+ lines += [f"- {i}" for i in ingredients]
+ lines += ["", "## Instructions", ""]
+ lines += [f"{n}. {s}" for n, s in enumerate(steps, 1)]
+
+ return "\n".join(lines).strip()
+
+
+# ── Article ───────────────────────────────────────────────────────────
+
+def _article_markdown(obj: dict) -> str | None:
+ """A `NewsArticle`/`Article` body, when the site publishes one.
+
+ Most do not — `articleBody` is optional and usually omitted, which
+ is why this returns None far more often than the recipe path and
+ why general extraction still exists. When it is present it is the
+ cleanest possible body: exactly what the publisher considers the
+ article, with no navigation to strip.
+ """
+ body = _text(obj.get("articleBody"))
+ if not body:
+ return None
+ lines = []
+ description = _text(obj.get("description"))
+ if description and description not in body:
+ lines += [description, ""]
+ lines.append(body)
+ return "\n".join(lines).strip()
+
+
+# ── Entry point ───────────────────────────────────────────────────────
+
+_READERS = (
+ ("Recipe", _recipe_markdown),
+ ("Article", _article_markdown),
+ ("NewsArticle", _article_markdown),
+ ("BlogPosting", _article_markdown),
+)
+
+
+def read_structured(html: str, *, url: str | None = None) -> SourceContent | None:
+ """The page's own structured data as a `SourceContent`, if usable.
+
+ Returns None whenever the markup is absent, unparseable, or present
+ but not about this page — the caller falls through to general
+ extraction, which is the common case.
+ """
+ for payload in _blocks(html):
+ for obj in _objects(payload):
+ for wanted, render in _READERS:
+ if not _has_type(obj, wanted):
+ continue
+ body = render(obj)
+ if not body:
+ continue
+ return SourceContent(
+ text=body,
+ mime="text/markdown",
+ title_hint=_text(obj.get("headline") or obj.get("name")) or None,
+ source_uri=url,
+ )
+ return None
diff --git a/stacklets/docs/bot/archivist.py b/stacklets/docs/bot/archivist.py
index 6e9faaee..e9826da0 100644
--- a/stacklets/docs/bot/archivist.py
+++ b/stacklets/docs/bot/archivist.py
@@ -32,6 +32,7 @@
import time
from contextlib import contextmanager
from pathlib import Path
+from urllib.parse import urlsplit
import aiohttp
import yaml
@@ -2406,6 +2407,15 @@ async def _reply_for_capture(
transcript=o.transcript,
todo_link=self._todo_link(o),
)
+ # A link card filed, but the family should hear why it has no
+ # summary -- and hear which obstacle it was. "Reddit wants you
+ # signed in" is something a person can act on; "couldn't read
+ # that link" is not.
+ if o.blocked_reason:
+ host = urlsplit(o.display_link or "").netloc or "the site"
+ note = self.t(f"capture_blocked_{o.blocked_reason}", host=host)
+ reply = f"{note}\n\n{reply}"
+
metadata = (
{"dev.famstack.event": o.envelope} if o.envelope else None
)
diff --git a/stacklets/docs/bot/capture_pipeline.py b/stacklets/docs/bot/capture_pipeline.py
index 7e6a504e..30ab9a38 100644
--- a/stacklets/docs/bot/capture_pipeline.py
+++ b/stacklets/docs/bot/capture_pipeline.py
@@ -19,7 +19,8 @@
import datetime as _dt
import io
import re
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
+from urllib.parse import unquote, urlsplit
from loguru import logger
@@ -103,6 +104,12 @@ class CaptureOutcome:
envelope: dict | None = None
transcript: str | None = None
failure_reason: str | None = None
+ # Set when a URL capture filed as a *link card* rather than a summary:
+ # the gate's verdict name (`challenge`, `login`, `consent`, `paywall`,
+ # `empty`). The entry exists either way -- this is what the reply layer
+ # uses to say why there is a link but no summary. None on every other
+ # capture, including a URL that read cleanly.
+ blocked_reason: str | None = None
# The vault bucket the capture filed under (`entity_slug`): a topic path
# like `family/camping` or a bare personal bucket like `homer`. The reply
# renderer uses it to build a `/go/topic/<scope>/todo` link, but only for
@@ -110,6 +117,37 @@ class CaptureOutcome:
scope: str | None = None
+def _link_card(url: str, user_hint: str | None) -> SourceContent:
+ """A `SourceContent` for a page we could not read.
+
+ The body is the sender's own words plus the link, and deliberately
+ nothing else. It is tempting to write "this page was blocked" into
+ it, but the body is the classifier's input: say that, and the entry
+ comes back titled "Blocked page" instead of "camping gear list".
+ Why it was blocked belongs in the chat reply, not in the vault.
+
+ The title hint falls back through the same reasoning -- the sender's
+ first line, then the URL's last path segment, then the host. Each is
+ a worse guess than the one before it, and all three beat a summary
+ of a cookie banner.
+ """
+ hint = (user_hint or "").strip()
+ split = urlsplit(url)
+ slug = [p for p in split.path.split("/") if p]
+ title = None
+ if hint:
+ title = hint.splitlines()[0].strip()[:120]
+ elif slug:
+ title = unquote(slug[-1]).replace("-", " ").replace("_", " ").strip() or None
+ if not title:
+ title = split.netloc
+
+ body = f"{hint}\n\n{url}" if hint else url
+ return SourceContent(
+ text=body, mime="text/markdown", title_hint=title, source_uri=url,
+ )
+
+
class CapturePipeline:
"""Captures a URL (bookmark) or pasted text (note) into the vault."""
@@ -175,17 +213,27 @@ async def capture_url(
out. Empty/None leaves the prompt unchanged.
"""
await notifier.acknowledge()
- source = await self._url_extractor.extract(url)
+ outcome = await self._url_extractor.fetch(url)
+
+ # A page we could not read still files. Dropping it on the floor
+ # was the old behaviour and it lost two things worth keeping: the
+ # link itself, and whatever the sender wrote around it -- which is
+ # often the more useful half ("gear list for the camping trip").
+ # The gate's verdict rides along so the reply can name the
+ # obstacle instead of guessing.
+ blocked_reason = None
+ source = outcome.content
if source is None:
- return CaptureOutcome(
- status="extract_failed", failure_reason="url",
- )
- return await self._publish(
+ source = _link_card(outcome.url or url, user_hint)
+ blocked_reason = outcome.verdict.name
+
+ result = await self._publish(
source=source, kind="bookmark", sender_mxid=sender_mxid,
display_link=url, actor=sender_mxid,
capture_id=capture_id, seed_topics=seed_topics,
bucket=bucket, user_hint=user_hint,
)
+ return replace(result, blocked_reason=blocked_reason) if blocked_reason else result
async def capture_text(
self, *, text: str, sender_mxid: str,
diff --git a/stacklets/docs/bot/extractors.py b/stacklets/docs/bot/extractors.py
index 02bb3f76..4b13317b 100644
--- a/stacklets/docs/bot/extractors.py
+++ b/stacklets/docs/bot/extractors.py
@@ -17,16 +17,16 @@
poll → OCR) because it's deeply Paperless-shaped; unifying it into a
single backend would be churn without a real second consumer.
-trafilatura is imported lazily so a Python test environment that
-doesn't exercise URL extraction doesn't need the dep installed. In
-production the bot-runner image always carries it (declared in
-`stacklets/core/bot-runner/requirements.txt`).
+`SourceContent` and the whole URL-reading ladder now live in
+`lib/stack/web/`. They moved the moment a second consumer appeared:
+the host CLI reads a URL with no archivist running. `UrlExtractor`
+stays here as the bot's binding to it — an aiohttp session in, a
+`SourceContent` out — so nothing upstream had to change.
"""
from __future__ import annotations
import re
-from dataclasses import dataclass
import aiohttp
from loguru import logger
@@ -35,28 +35,10 @@
# here so existing `from extractors import parse_email, ParsedEmail`
# callers and tests keep working after the move.
from stack.email_message import ParsedEmail, parse_email # noqa: F401
-
-
-# ── SourceContent ────────────────────────────────────────────────────────
-
-@dataclass
-class SourceContent:
- """The classifier's input, normalized across source types.
-
- `text` is the body the classifier reads — Markdown when the
- extractor can produce it, plain text otherwise. `title_hint` is
- whatever the source advertised as a title (HTML `<title>`, first
- body line, filename); the classifier may overwrite it with
- something more useful. `source_uri` is the canonical pointer
- back to the origin (`https://...`, `paperless://42`,
- `matrix:<event-id>` — caller decides the scheme), captured into
- the mirror's frontmatter for round-tripping. None means the
- capture has no upstream pointer (a pure pasted note).
- """
- text: str
- mime: str = "text/plain"
- title_hint: str | None = None
- source_uri: str | None = None
+# Same arrangement for the capture types: the framework owns the shared
+# shape, this module owns the docs-side mapping into it.
+from stack.web import FetchOutcome, SourceContent # noqa: F401
+from stack.web.fetch import aiohttp_transport, fetch_url
# ── Shared helpers ───────────────────────────────────────────────────────
@@ -96,82 +78,50 @@ def _first_url(text: str) -> str | None:
# ── UrlExtractor ─────────────────────────────────────────────────────────
class UrlExtractor:
- """Fetch a URL and convert the HTML body to Markdown via trafilatura.
-
- Failure paths return None, never raise — the caller renders a
- single "couldn't capture this" reply regardless of whether the
- server 500'd, the host was unreachable, or trafilatura couldn't
- find a body. Logging surfaces the distinction for debugging.
-
- Non-HTML content types are rejected at the gate. PDFs reach the
- archivist through a separate path (`_handle_url`) that uploads to
- Paperless; this extractor's job is web articles only.
+ """The bot's binding to the framework's fetch ladder.
+
+ The reading itself — canonicalize, structured data, HTTP,
+ extraction, and the quality gate — is `stack.web.fetch`. This
+ supplies the one thing the framework deliberately does not have: a
+ transport. The bot already keeps an aiohttp session open for its
+ lifetime, so it hands that in rather than opening a second one.
+
+ Two ways out, for two callers. `fetch` returns the full outcome
+ including the gate's verdict, which is what the capture pipeline
+ needs to explain a refusal to the family. `extract` keeps the older
+ "content or nothing" shape for callers that only care whether there
+ was something to file.
+
+ Tier 3 (a real browser, in the optional `web` stacklet) is not
+ wired up yet. Until it is, a challenge page is terminal and the
+ family is told the site blocked us — which is the honest answer,
+ and a better one than the fabricated entry they used to get.
"""
def __init__(self, http: aiohttp.ClientSession, *, timeout: int = 30):
self.http = http
self.timeout = timeout
- async def extract(self, url: str) -> SourceContent | None:
- html = await self._fetch_html(url)
- if html is None:
- return None
-
- try:
- import trafilatura
- except ImportError:
- logger.error(
- "[extractor] trafilatura not installed — "
- "URL captures require the bot-runner image",
- )
- return None
-
- body = trafilatura.extract(
- html, output_format="markdown",
- include_links=True, include_images=False,
- include_tables=True,
- favor_precision=True,
- )
- if not body or not body.strip():
- logger.info("[extractor] {} → trafilatura returned no body", url)
- return None
-
- return SourceContent(
- text=body.strip(),
- mime="text/html",
- title_hint=_html_title(html),
- source_uri=url,
+ async def fetch(self, url: str) -> FetchOutcome:
+ """Read a URL and return a judged outcome. Never raises."""
+ outcome = await fetch_url(
+ url, transport=aiohttp_transport(self.http, timeout=self.timeout),
)
+ if outcome.ok:
+ logger.info(
+ "[extractor] {} → tier {} ({}), {} chars",
+ url, outcome.tier, outcome.profile, len(outcome.content.text),
+ )
+ else:
+ logger.info(
+ "[extractor] {} → {}: {}",
+ url, outcome.verdict.name, outcome.verdict.detail,
+ )
+ return outcome
- async def _fetch_html(self, url: str) -> str | None:
- """GET the URL. Returns the body text on success, None on any
- failure (non-200, non-HTML, transport error)."""
- try:
- async with self.http.get(
- url,
- timeout=aiohttp.ClientTimeout(total=self.timeout),
- allow_redirects=True,
- headers={"User-Agent": "famstack-archivist/1.0"},
- ) as resp:
- if resp.status != 200:
- logger.info(
- "[extractor] {} → HTTP {}", url, resp.status,
- )
- return None
- ctype = (resp.content_type or "").lower()
- if not (
- ctype.startswith("text/html")
- or ctype.startswith("application/xhtml")
- ):
- logger.info(
- "[extractor] {} → non-HTML content_type={}",
- url, ctype,
- )
- return None
- return await resp.text()
- except (aiohttp.ClientError, OSError) as e:
- logger.warning("[extractor] {} → fetch failed: {}", url, e)
- return None
+ async def extract(self, url: str) -> SourceContent | None:
+ """The fetched body, or None when there was nothing to file."""
+ return (await self.fetch(url)).content
# ── TextExtractor ────────────────────────────────────────────────────────
diff --git a/stacklets/docs/bot/messages/archivist.yml b/stacklets/docs/bot/messages/archivist.yml
index ce326967..32d1a710 100644
--- a/stacklets/docs/bot/messages/archivist.yml
+++ b/stacklets/docs/bot/messages/archivist.yml
@@ -85,6 +85,15 @@ en:
# URL capture (knowledge rooms — links become summarized notes, not Paperless docs)
capture_failed: "\u274C Couldn't read that link. Either the host is unreachable or there's no article body to extract."
capture_failed_binary: "\u274C Couldn't read that file."
+ # A blocked page still files: the link and whatever the sender wrote
+ # around it become a link card, and one of these lines says why there
+ # is no summary. Naming the obstacle beats "couldn't read that link" —
+ # the family can decide whether to open it themselves.
+ capture_blocked_challenge: "\U0001F512 {host} checks for bots, so I couldn't read the page. Saved the link."
+ capture_blocked_login: "\U0001F512 {host} only shows this to people who are signed in. Saved the link."
+ capture_blocked_consent: "\U0001F512 {host} served a consent wall instead of the page. Saved the link."
+ capture_blocked_paywall: "\U0001F512 {host} is paywalled. Saved the link."
+ capture_blocked_empty: "\U0001F4C4 No article text on that page. Saved the link."
capture_llm_failed: "\U0001F4DD Captured the page, but classification skipped: {error}"
capture_no_mirror: "\u274C Captures need the `code` stacklet up so I can write to the memory vault. Run `stack up code`."
captured: "\u2705 Captured: {title}"
@@ -288,6 +297,11 @@ de:
# URL-Capture (Wissensräume — Links werden zu zusammengefassten Notizen, nicht zu Paperless-Dokumenten)
capture_failed: "\u274C Konnte den Link nicht lesen. Entweder ist die Seite nicht erreichbar, oder es gibt keinen Artikel-Inhalt zum Extrahieren."
capture_failed_binary: "\u274C Konnte diese Datei nicht lesen."
+ capture_blocked_challenge: "\U0001F512 {host} prüft auf Bots, deshalb konnte ich die Seite nicht lesen. Link gespeichert."
+ capture_blocked_login: "\U0001F512 {host} zeigt das nur angemeldeten Personen. Link gespeichert."
+ capture_blocked_consent: "\U0001F512 {host} hat statt der Seite eine Zustimmungs-Abfrage geliefert. Link gespeichert."
+ capture_blocked_paywall: "\U0001F512 {host} ist hinter einer Bezahlschranke. Link gespeichert."
+ capture_blocked_empty: "\U0001F4C4 Auf der Seite gab es keinen Artikeltext. Link gespeichert."
capture_llm_failed: "\U0001F4DD Seite gespeichert, aber Klassifizierung übersprungen: {error}"
capture_no_mirror: "\u274C Captures brauchen das `code`-Stacklet, damit ich in den Memory-Vault schreiben kann. Starte `stack up code`."
captured: "\u2705 Gespeichert: {title}"
diff --git a/tests/conftest.py b/tests/conftest.py
index dc2f1b4b..ed68915a 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -183,3 +183,43 @@ def git_unreachable_remote_clone(_git_states, tmp_path) -> GitPair:
gone = tmp_path / "moved-away.git"
_git("-C", str(pair.local), "remote", "set-url", "origin", str(gone))
return GitPair(remote=gone, local=pair.local)
+
+
+# ── Web capture fixtures ──────────────────────────────────────────────
+#
+# Real pages, captured from the live sites the web plan measured, so a
+# detector cannot be tuned against HTML that was written to satisfy it.
+# `tests/fixtures/web/README.md` records where each came from and what
+# was stripped. Both test trees use these: the gate lives in the
+# framework, the extractor that calls it lives in a stacklet.
+
+_WEB_FIXTURES = Path(__file__).parent / "fixtures" / "web"
+
+
+@pytest.fixture(scope="session")
+def web_fixture():
+ """Load a captured page by name (no `.html` suffix)."""
+ def _load(name: str) -> str:
+ path = _WEB_FIXTURES / f"{name}.html"
+ if not path.exists():
+ available = ", ".join(sorted(p.stem for p in _WEB_FIXTURES.glob("*.html")))
+ raise FileNotFoundError(f"no web fixture {name!r}; have: {available}")
+ return path.read_text(encoding="utf-8")
+
+ return _load
+
+
+@pytest.fixture(scope="session")
+def extracted():
+ """Run the real tier-2 extraction over fixture HTML.
+
+ Tests assert on what trafilatura actually produces rather than on a
+ stand-in for it, so a library upgrade that changes extraction shows
+ up here instead of in production.
+ """
+ from stack.web.fetch import extract_body
+
+ def _extract(html: str, **kwargs) -> str:
+ return extract_body(html, **kwargs) or ""
+
+ return _extract
diff --git a/tests/fixtures/web/README.md b/tests/fixtures/web/README.md
new file mode 100644
index 00000000..43a43184
--- /dev/null
+++ b/tests/fixtures/web/README.md
@@ -0,0 +1,53 @@
+# Web capture fixtures
+
+Real pages, captured from the live sites the web plan measured.
+
+They are here because a fixture written next to the detector that reads
+it proves only that the two were written together. Every claim the gate
+makes about how a site blocks is checkable against what the site
+actually served.
+
+## Provenance
+
+Captured 2026-09-15 with a Chrome user agent, anonymous (no cookies, no
+session), from a German residential IP.
+
+| Fixture | Source | Served |
+|---|---|---|
+| `cloudflare-challenge.html` | `https://www.decathlon.de/` | HTTP 403, Cloudflare managed challenge, `<title>Just a moment...` |
+| `reddit-login-wall.html` | `https://old.reddit.com/r/selfhosted/` | HTTP 302 to `/login/?reason=lor2`, then HTTP 200, `Welcome to Reddit` |
+| `google-maps-shell.html` | `https://www.google.com/maps/place/Brandenburger+Tor/` | HTTP 200, application shell, no prose but the site-wide meta description |
+| `recipe-jsonld.html` | `https://www.essen-und-trinken.de/rezepte/48816-rzpt-griechischer-salat` | HTTP 200, complete schema.org `Recipe` |
+| `shop-listing-ok.html` | `https://geizhals.de/` | HTTP 200, real listing, cookie banner in the same document |
+
+The last one is the negative control: a page carrying a consent banner
+that must still pass the gate, because the content is right there.
+
+## What was stripped
+
+Each file had inline `