From 56781e9bd3017b9525cd309349397c27d29131cd Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 15 Sep 2026 22:14:34 +0200 Subject: [PATCH] feat(capture): file a link card when a page cannot be read A pasted link that a site refuses to serve produced either a silent failure or an entry summarising a consent banner. Fetched pages are now classified before anything is filed, and a page that cannot be read still files as a link card carrying the text the sender wrote with it. The chat reply names the obstacle: bot protection, sign-in, consent wall, paywall, or a page with no text. Recipes are read from the page's own structured data, so quantities and steps come from the publisher rather than from a summary. Application shell pages whose subject appears in the URL are answered from the URL. Refs docs/design/web/plan.md (phase 1). --- docs/design/web/plan.md | 33 +- lib/stack/web/__init__.py | 30 + lib/stack/web/content.py | 37 + lib/stack/web/fetch.py | 306 ++ lib/stack/web/profiles.py | 176 + lib/stack/web/quality.py | 219 + lib/stack/web/structured.py | 236 + stacklets/docs/bot/archivist.py | 10 + stacklets/docs/bot/capture_pipeline.py | 60 +- stacklets/docs/bot/extractors.py | 140 +- stacklets/docs/bot/messages/archivist.yml | 14 + tests/conftest.py | 40 + tests/fixtures/web/README.md | 53 + tests/fixtures/web/cloudflare-challenge.html | 1 + tests/fixtures/web/google-maps-shell.html | 1 + tests/fixtures/web/recipe-jsonld.html | 1501 +++++++ tests/fixtures/web/reddit-login-wall.html | 322 ++ tests/fixtures/web/shop-listing-ok.html | 4237 ++++++++++++++++++ tests/framework/test_web_fetch.py | 298 ++ tests/framework/test_web_profiles.py | 122 + tests/framework/test_web_quality.py | 221 + tests/framework/test_web_structured.py | 192 + tests/stacklets/test_archivist_routing.py | 15 + tests/stacklets/test_capture_pipeline.py | 114 +- tools/web/capture-fixture.py | 93 + 25 files changed, 8347 insertions(+), 124 deletions(-) create mode 100644 lib/stack/web/__init__.py create mode 100644 lib/stack/web/content.py create mode 100644 lib/stack/web/fetch.py create mode 100644 lib/stack/web/profiles.py create mode 100644 lib/stack/web/quality.py create mode 100644 lib/stack/web/structured.py create mode 100644 tests/fixtures/web/README.md create mode 100644 tests/fixtures/web/cloudflare-challenge.html create mode 100644 tests/fixtures/web/google-maps-shell.html create mode 100644 tests/fixtures/web/recipe-jsonld.html create mode 100644 tests/fixtures/web/reddit-login-wall.html create mode 100644 tests/fixtures/web/shop-listing-ok.html create mode 100644 tests/framework/test_web_fetch.py create mode 100644 tests/framework/test_web_profiles.py create mode 100644 tests/framework/test_web_quality.py create mode 100644 tests/framework/test_web_structured.py create mode 100755 tools/web/capture-fixture.py diff --git a/docs/design/web/plan.md b/docs/design/web/plan.md index 19bb2773..9c2a7dca 100644 --- a/docs/design/web/plan.md +++ b/docs/design/web/plan.md @@ -131,6 +131,37 @@ Other measurements: Camoufox publishes `lin.arm64`; Playwright's Chromium builds arm64. This is the single biggest reason Scrapling wins. +## Re-validation, 2026-09-15 + +Re-probed before building Phase 1, same two UAs, anonymous, German IP. +Three findings held, one moved. + +| Claim | Still true? | +|---|---| +| decathlon.de blocks a plain fetch | yes — 403, Cloudflare `Just a moment...` | +| Recipe JSON-LD on essen-und-trinken / einfachkochen | yes — full `Recipe`, yield, totalTime, 11 and 14 ingredients, steps, nutrition | +| geizhals.de returns 403 | **no** — the *homepage* serves 200 and its real listing. The measured 403 was a product/listing URL; the row overstated it as the whole domain | +| `old.reddit.com` + Chrome UA returns the post | **no — reversed** | + +**Reddit now walls anonymous readers.** `old.reddit.com` answers a 302 to +`/login/?reason=lor2`, and `www.reddit.com` serves an 8 KB JavaScript +shell. The `.json` endpoint is 403 with a 190 KB HTML block page. + +This is worth more than a corrected row, because of the *shape* of the +failure. The login redirect ends on HTTP 200 with a 320 KB body and a +friendly `Welcome to Reddit` — which is exactly what a +"did the extractor return a string?" success check reads as an article. +The drift did not break the gate's design, it validated it: `login` was +already in the verdict set, and the landing URL is the only honest +signal on that page. The gate reads the URL a fetch *ended* on for +precisely this reason. + +The reddit profile keeps its `old.reddit` rewrite anyway. It no longer +recovers the post, but it moves the reported reason from `empty` ("the +page was blank") to `login` ("reddit wants you signed in"), which is the +one a person can act on — and it starts working again unchanged if +reddit relaxes. + ## Decision: one stacklet, but not for everything The tempting version is a `web` stacklet that owns all web operations including @@ -158,7 +189,7 @@ construction: a family that never pastes a shop link never downloads Chromium. ## Phases -### Phase 1 — Framework module and the gate (about 1 day) +### Phase 1 — Framework module and the gate — SHIPPED The whole fix for both reported bugs, with no new container. diff --git a/lib/stack/web/__init__.py b/lib/stack/web/__init__.py new file mode 100644 index 00000000..2e4c0261 --- /dev/null +++ b/lib/stack/web/__init__.py @@ -0,0 +1,30 @@ +"""How famstack reads the web. + +A pasted link becomes a good vault entry or an honest link card, and +never a cookie policy. The ladder that decides which is in `fetch`; the +rule that nothing unjudged reaches the vault is in `quality`. + + content SourceContent, the shape every capture source produces + quality the gate: ok / challenge / login / consent / paywall / empty + profiles per-domain rules — canonical URL, extraction tuning + structured JSON-LD Recipe and Article, read without an LLM + fetch the ladder that runs them in order + +See `docs/design/web/plan.md` for the measurements behind each tier. +""" + +from stack.web.content import SourceContent +from stack.web.fetch import FetchOutcome, fetch_url +from stack.web.profiles import canonicalize, profile_for +from stack.web.quality import Page, Verdict, assess + +__all__ = [ + "FetchOutcome", + "Page", + "SourceContent", + "Verdict", + "assess", + "canonicalize", + "fetch_url", + "profile_for", +] diff --git a/lib/stack/web/content.py b/lib/stack/web/content.py new file mode 100644 index 00000000..ae61569a --- /dev/null +++ b/lib/stack/web/content.py @@ -0,0 +1,37 @@ +"""`SourceContent` — the classifier's input, normalized across sources. + +Promoted here from `stacklets/docs/bot/extractors.py` because it stopped +being one stacklet's type the moment a second consumer needed it: the +host CLI (`stack web fetch`) builds one without the archivist running at +all. Same move as `stack.email_message`, same reason -- the framework +owns the shared shape, each stacklet owns its own mapping into it. + +The archivist's classifier does not care whether text arrived from a +photographed receipt that Paperless OCR'd, a pasted URL that trafilatura +rendered, or a wall of text somebody typed. This is what all of those +agree to produce. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class SourceContent: + """The classifier's input, normalized across source types. + + `text` is the body the classifier reads — Markdown when the + extractor can produce it, plain text otherwise. `title_hint` is + whatever the source advertised as a title (HTML ``, first + body line, filename); the classifier may overwrite it with + something more useful. `source_uri` is the canonical pointer + back to the origin (`https://...`, `paperless://42`, + `matrix:<event-id>` — caller decides the scheme), captured into + the mirror's frontmatter for round-tripping. None means the + capture has no upstream pointer (a pure pasted note). + """ + text: str + mime: str = "text/plain" + title_hint: str | None = None + source_uri: str | None = None diff --git a/lib/stack/web/fetch.py b/lib/stack/web/fetch.py new file mode 100644 index 00000000..537ba58a --- /dev/null +++ b/lib/stack/web/fetch.py @@ -0,0 +1,306 @@ +"""The fetch ladder — cheap first, expensive only on proven failure. + +Four tiers, and the whole design is in which order they run: + + 0 canonicalize strip trackers, rewrite hosts ~0 ms + 1 structured the page's own JSON-LD ~0.5 s + 2 plain HTTP browser headers, then trafilatura ~0.5 s + 3 stealth a real browser, in the web stacklet 3.4-19.8 s + +Tiers 0 to 2 are pure Python over bytes. They carry the common case, +they need nothing running, and they are why `stack web fetch` works on +a machine with no containers up. Tier 3 is the only step that needs a +browser, it lives in an optional stacklet, and it is reached only when +the gate says bot protection is in the way — never by default, and +never for a failure a browser cannot fix. + +The gate runs after every tier that produces content, including tier 3. +An expensive fetch does not get to skip the check: a stealth browser +can be served a challenge page too, and an unjudged 19-second result is +no better than an unjudged instant one. + +Transport is injected rather than imported. The bot has an aiohttp +session already open and should reuse it; the host CLI has no aiohttp +at all. Both hand in a callable, so this module depends on neither. + +trafilatura is imported lazily for the same reason: the gate, the +profiles and the JSON-LD reader are stdlib, and a caller that never +reaches tier 2 never needs the dependency. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Awaitable, Callable + +from stack.web.content import SourceContent +from stack.web.profiles import Profile, canonicalize, profile_for, title_from_url +from stack.web.quality import Page, Verdict, assess, page_title + +# The headers a browser sends. Not a disguise — plenty of CDNs answer a +# bare library user-agent with a 403 out of habit, and this is the +# difference between reading a public page and not. Sites that actually +# check (decathlon) are unmoved by it; that is what tier 3 is for. +BROWSER_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", +} + + +@dataclass +class Response: + """What a transport hands back. Deliberately minimal — the ladder + needs the landing URL (a login redirect is only visible there), the + status, the body and the content type, and nothing else.""" + + url: str + status: int + html: str + content_type: str = "text/html" + + +# A transport takes a URL plus headers and returns a Response, or None +# when the request could not be made at all (DNS, timeout, refused). +Transport = Callable[[str, dict], Awaitable["Response | None"]] + + +@dataclass +class FetchOutcome: + """The result of running the ladder. + + `content` is present only when the gate passed. `verdict` is always + present and always explains itself, so a caller rendering a link + card has something true to say about why there is no entry. + + `tier` records which rung produced the outcome, for the logs: a + site that starts needing tier 3 is worth noticing. + """ + + verdict: Verdict + content: SourceContent | None = None + url: str = "" + tier: str = "" + profile: str = "default" + + @property + def ok(self) -> bool: + return self.content is not None and self.verdict.ok + + +# ── Tier 2 extraction ───────────────────────────────────────────────── + +def extract_body( + html: str, *, favor_recall: bool = False, include_comments: bool = False, +) -> str | None: + """HTML to Markdown via trafilatura, tuned by profile. + + Precision is the default because most pages are articles surrounded + by furniture. Recall is for pages whose content *is* the discussion + under them — measured on a Reddit post, precision returned 821 + characters of sidebar and recall returned the 5513-character post. + The per-domain profile decides; this function only applies it. + + Returns None when trafilatura is unavailable or finds no body. The + caller treats both as "tier 2 produced nothing" and lets the gate + name it, so a missing dependency degrades to a link card rather + than an exception. + """ + try: + import trafilatura + except ImportError: + return None + + kwargs = { + "output_format": "markdown", + "include_links": True, + "include_images": False, + "include_tables": True, + "include_comments": include_comments, + } + # trafilatura rejects both favour flags at once; recall wins where a + # profile asked for it, precision is the default everywhere else. + if favor_recall: + kwargs["favor_recall"] = True + else: + kwargs["favor_precision"] = True + + body = trafilatura.extract(html, **kwargs) + return body.strip() if body and body.strip() else None + + +def _is_html(content_type: str) -> bool: + ctype = (content_type or "").lower() + return ctype.startswith("text/html") or ctype.startswith("application/xhtml") + + +# ── The ladder ──────────────────────────────────────────────────────── + +async def fetch_url( + url: str, + *, + transport: Transport, + stealth: Transport | None = None, +) -> FetchOutcome: + """Run the ladder and return a judged outcome. Never raises. + + `stealth` is tier 3. Passing None — which is what happens when the + `web` stacklet is not installed — means a challenge is terminal and + the caller renders a link card saying so. That is the honest + degradation: the family is told the site blocked us, rather than + getting a silent failure or an entry built from the challenge page. + """ + target = canonicalize(url) + profile = profile_for(target) + + # Tier 0.5 — a shell page whose subject is in the path. Google Maps + # is the case: fetching it is pointless, the document is an empty + # application and the place name is already in the URL we were + # handed. Answering from the URL beats answering from the page. + shell_title = title_from_url(target) + + response = await _safe(transport, target) + if response is None: + if shell_title: + return _from_url_only(target, shell_title, profile) + return FetchOutcome( + verdict=Verdict("empty", "the site could not be reached"), + url=target, tier="2", profile=profile.name, + ) + + outcome = _judge(response, profile) + if outcome.ok or not outcome.verdict.escalate or stealth is None: + if not outcome.ok and shell_title: + return _from_url_only(target, shell_title, profile) + return outcome + + # Tier 3 — only for a challenge, only once. There is no loop back to + # tier 2: if the browser is also served a challenge, the answer is + # that we cannot read this page. + stealthed = await _safe(stealth, response.url or target) + if stealthed is None: + return outcome + escalated = _judge(stealthed, profile, tier="3") + return escalated if escalated.ok else outcome + + +async def _safe(transport: Transport, url: str) -> Response | None: + """Run a transport, turning any transport-level failure into None. + + The ladder's contract is that it never raises; a caller rendering a + chat reply has nothing useful to do with a socket error. + """ + try: + return await transport(url, dict(BROWSER_HEADERS)) + except Exception: # noqa: BLE001 — transports raise library-specific errors + return None + + +def _judge(response: Response, profile: Profile, *, tier: str = "2") -> FetchOutcome: + """Tiers 1 and 2 over one response, then the gate. + + Structured data is tried first and short-circuits: when a site hands + us its own `Recipe` object there is no reason to guess at the + rendered page, and no reason to ask the gate whether the guess was + any good. + """ + if not _is_html(response.content_type): + return FetchOutcome( + verdict=Verdict("empty", f"the URL served {response.content_type}, not a web page"), + url=response.url, tier=tier, profile=profile.name, + ) + + structured = _structured(response) + if structured is not None: + return FetchOutcome( + verdict=Verdict("ok", "the page published its own structured data"), + content=structured, url=response.url, tier="1", profile=profile.name, + ) + + body = extract_body( + response.html, + favor_recall=profile.favor_recall, + include_comments=profile.include_comments, + ) + page = Page( + url=response.url, status=response.status, html=response.html, + text=body or "", content_type=response.content_type, + ) + verdict = assess(page, min_chars=profile.min_chars) + if not verdict.ok: + detail = verdict.detail + if profile.blocked_note: + detail = f"{detail} — {profile.blocked_note}" + return FetchOutcome( + verdict=Verdict(verdict.name, detail), + url=response.url, tier=tier, profile=profile.name, + ) + + return FetchOutcome( + verdict=verdict, + content=SourceContent( + text=body or "", + mime="text/html", + title_hint=page_title(response.html), + source_uri=response.url, + ), + url=response.url, tier=tier, profile=profile.name, + ) + + +def _structured(response: Response) -> SourceContent | None: + from stack.web.structured import read_structured + + return read_structured(response.html, url=response.url) + + +def _from_url_only(url: str, title: str, profile: Profile) -> FetchOutcome: + """An entry built from the URL, for pages that have no text to read. + + The body is the one true sentence we can say about the place: its + name and where the link points. The classifier gets a real title + instead of "Google Maps", and the family gets an entry they can + find again. + """ + return FetchOutcome( + verdict=Verdict("ok", "the page has no text; its subject came from the URL"), + content=SourceContent( + text=f"{title}\n\n{url}", + mime="text/markdown", + title_hint=title, + source_uri=url, + ), + url=url, tier="0", profile=profile.name, + ) + + +# ── aiohttp transport ───────────────────────────────────────────────── + +def aiohttp_transport(session, *, timeout: int = 30) -> Transport: + """A transport over an already-open aiohttp session. + + The bot keeps one session for its lifetime; handing it in rather + than opening a new one per fetch keeps connection reuse and, more + importantly, keeps this module free of an aiohttp import at module + level so the host CLI can import the ladder without it. + """ + async def _fetch(url: str, headers: dict) -> Response | None: + import aiohttp + + async with session.get( + url, + timeout=aiohttp.ClientTimeout(total=timeout), + allow_redirects=True, + headers=headers, + ) as resp: + return Response( + url=str(resp.url), + status=resp.status, + html=await resp.text(errors="replace"), + content_type=resp.content_type or "", + ) + + return _fetch diff --git a/lib/stack/web/profiles.py b/lib/stack/web/profiles.py new file mode 100644 index 00000000..540b6d71 --- /dev/null +++ b/lib/stack/web/profiles.py @@ -0,0 +1,176 @@ +"""Per-domain rules — the part of web reading that is not general. + +Two things vary by site and nothing else does: which URL actually +serves the content, and how hard to pull on the extractor. Both are +small, both are data, and both belong somewhere a new site costs one +entry rather than a branch in the fetch ladder. + +Tier 0 is the canonical URL. It is free, it runs before any request, +and it is the single highest-yield step in the whole ladder: stripping +a tracker parameter turns two URLs into one cache key, and rewriting a +host can turn a JavaScript shell into a document. + +Stdlib only: the host CLI runs this without a virtualenv. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from urllib.parse import parse_qsl, unquote, urlencode, urlsplit, urlunsplit + +from stack.web.quality import DEFAULT_MIN_CHARS + + +# ── Tracking parameters ─────────────────────────────────────────────── +# +# Campaign and referrer tags identify the person who shared the link, +# not the page. Dropping them is a privacy measure as much as a +# normalisation one: the URL ends up in the vault and in chat, and it +# should not carry "Homer clicked this from a newsletter" with it. + +_TRACKING_PREFIXES = ("utm_", "pk_", "mtm_", "matomo_", "ga_", "_hs") +_TRACKING_PARAMS = frozenset({ + "fbclid", "gclid", "dclid", "msclkid", "twclid", "igshid", "ttclid", + "mc_cid", "mc_eid", "ref", "ref_src", "ref_url", "referrer", "source", + "share_id", "si", "spm", "cmpid", "icid", "trk", "yclid", "vero_id", +}) + + +def _is_tracking(key: str) -> bool: + lowered = key.lower() + return lowered in _TRACKING_PARAMS or lowered.startswith(_TRACKING_PREFIXES) + + +def strip_tracking(url: str) -> str: + """Drop campaign/referrer parameters, keep everything else. + + Conservative on purpose. A query string is often load-bearing (an + article id, a search term, a page number), so only known-tracking + keys go; anything unrecognised stays. + """ + split = urlsplit(url) + if not split.query: + return url + kept = [(k, v) for k, v in parse_qsl(split.query, keep_blank_values=True) + if not _is_tracking(k)] + return urlunsplit(split._replace(query=urlencode(kept))) + + +# ── Profiles ────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class Profile: + """What we know about reading one family of sites. + + `hosts` are matched as suffixes, so a profile for `reddit.com` + covers `old.reddit.com` and `www.reddit.com` alike. + + `rewrite_host` swaps the host before fetching. `favor_recall` and + `include_comments` are handed to trafilatura: precision is the right + default for an article, recall is the right default for a page whose + content *is* the discussion under it. + + `min_chars` lowers the gate's empty floor for sites that genuinely + publish short pages. + + `blocked_note` is prose shown to the family when this site fails the + gate. Generic advice ("the site blocked us") is worse than naming + the actual reason where we already know it. + """ + + name: str + hosts: tuple[str, ...] = () + rewrite_host: str | None = None + favor_recall: bool = False + include_comments: bool = False + min_chars: int = DEFAULT_MIN_CHARS + blocked_note: str | None = None + # Path prefixes whose content lives in the URL rather than the body. + # A Google Maps place is the worked example: the document is an + # empty application shell, and the place name is right there in the + # path we were given. + title_from_path: tuple[str, ...] = field(default=()) + + +DEFAULT = Profile(name="default") + +PROFILES: tuple[Profile, ...] = ( + # Reddit serves anonymous readers a JavaScript shell on `www` and, + # since measurement, a 302 to `/login?reason=lor2` on `old`. Neither + # yields a post. The rewrite is kept anyway, and deliberately: it + # moves the failure from `empty` ("the page was blank", true but + # unhelpful) to `login` ("reddit wants you signed in", actionable). + # If reddit relaxes anonymous access, the rewrite starts working + # again with no change here. Recall settings are ready for that day + # — measured at the time: precision returned 821 characters of + # sidebar, recall returned the 5513-character post. + Profile( + name="reddit", + hosts=("reddit.com",), + rewrite_host="old.reddit.com", + favor_recall=True, + include_comments=True, + blocked_note="reddit no longer serves posts to readers who are not signed in", + ), + # Google Maps place URLs are an application shell: HTTP 200, a full + # document, and no prose but the site-wide meta description. The + # place name is in the path, so the URL is the content. + Profile( + name="google-maps", + hosts=("google.com", "google.de", "maps.app.goo.gl"), + title_from_path=("/maps/place/",), + blocked_note="Google Maps renders in the browser, so there is no page text to file", + ), +) + + +def profile_for(url: str) -> Profile: + """The profile governing this URL, or the default. + + Host suffix match, so `old.reddit.com` and `www.reddit.com` both + resolve to the reddit profile. + """ + host = urlsplit(url).netloc.lower().split(":")[0] + for profile in PROFILES: + if any(host == h or host.endswith("." + h) for h in profile.hosts): + return profile + return DEFAULT + + +# ── Tier 0 ──────────────────────────────────────────────────────────── + +def canonicalize(url: str) -> str: + """The URL we should actually fetch. Costs nothing, runs first. + + Strips tracking parameters and applies the profile's host rewrite. + Shortener resolution is *not* done here: it needs a request, so it + belongs to the fetch step, and this function stays pure. + """ + cleaned = strip_tracking(url.strip()) + profile = profile_for(cleaned) + if profile.rewrite_host: + split = urlsplit(cleaned) + if split.netloc.lower() != profile.rewrite_host: + cleaned = urlunsplit(split._replace(netloc=profile.rewrite_host)) + return cleaned + + +def title_from_url(url: str) -> str | None: + """A human title recovered from the path, for shell pages. + + Returns None unless the URL matches a profile's `title_from_path`, + so this never guesses at an ordinary article URL — a slug makes a + poor title when the document has a real one. + """ + profile = profile_for(url) + if not profile.title_from_path: + return None + path = urlsplit(url).path + for prefix in profile.title_from_path: + if prefix not in path: + continue + tail = path.split(prefix, 1)[1].split("/")[0] + name = unquote(tail).replace("+", " ").strip() + if name: + return name + return None diff --git a/lib/stack/web/quality.py b/lib/stack/web/quality.py new file mode 100644 index 00000000..93c64d03 --- /dev/null +++ b/lib/stack/web/quality.py @@ -0,0 +1,219 @@ +"""The quality gate — what may become a vault entry, and what may not. + +The archivist's URL capture used to succeed whenever the extractor +returned a non-empty string. Three different pages return a non-empty +string without being articles: + + a Cloudflare interstitial 403, "Just a moment...", a spinner + a login wall 200, 320 KB, titled "Welcome to Reddit" + a JavaScript shell 200, a full document, no prose + +All three were filed, summarised by the classifier, and given a title. +The family ended up with wiki entries about cookie policies. This +module is the fix: nothing reaches the vault without being named first. + +The verdict is a value rather than a boolean because two readers need +it. The chat reply says something different for "this site wants you +logged in" than for "this site is checking your browser", and whoever +reads the logs later needs to know which sites fail which way. + + ok there is a body worth filing + challenge bot protection is between us and the page + login the site redirected us to a sign-in + consent a cookie or consent wall is being served instead + paywall the content exists but is not ours to read + empty we got a page and it had nothing on it + +Only `challenge` is worth escalating to a browser tier. The rest are +terminal no matter how expensively we fetch them, which is what keeps +tier 3 from being a retry loop. + +Detection keys on structure, never on prose. An article *about* +Cloudflare contains every word its challenge page does; what it does not +contain is Cloudflare's DOM ids, its challenge host, or that exact +`<title>`. Where a structural signal exists off-page -- the URL a +redirect chain ended on -- it is preferred, because it is the one part +of a JavaScript shell that cannot be styled away. + +Stdlib only: the host CLI runs this without a virtualenv. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from urllib.parse import urlsplit + +# A body shorter than this is furniture — a nav rail, a cookie notice, +# a subreddit sidebar. Measured: trafilatura on a blocked Reddit page +# returned 821 characters of sidebar and read as content to everything +# downstream. Callers may lower it per profile; see `assess`. +DEFAULT_MIN_CHARS = 250 + + +@dataclass(frozen=True) +class Page: + """One fetched page, as the gate sees it. + + `url` is the URL the fetch *ended* on, not the one it started + with. A login wall is a redirect, so the requested URL says nothing + and the landing URL says everything. + + `text` is the extracted body when tier 2 has run, and empty before + that. `html` is always the raw document: the structural markers the + gate reads (a `<title>`, a challenge script host) are stripped out + by extraction, so both are needed. + """ + + url: str + status: int = 200 + html: str = "" + text: str = "" + content_type: str = "text/html" + + +@dataclass(frozen=True) +class Verdict: + """A named outcome plus the sentence that explains it.""" + + name: str + detail: str + + @property + def ok(self) -> bool: + return self.name == "ok" + + @property + def escalate(self) -> bool: + """Whether a browser tier could plausibly do better. + + Only bot protection qualifies. A paywall, a login wall and an + empty page cost the same in a headless browser as they do over + plain HTTP, so escalating them would buy a slower failure. + """ + return self.name == "challenge" + + +# ── Challenge ───────────────────────────────────────────────────────── +# +# Cloudflare's interstitial is the one we measured, and it identifies +# itself three ways that survive a localisation change: the element id +# it puts its error text in, the host it loads the widget from, and the +# managed-challenge title. Any one is conclusive; prose is not used. + +_CHALLENGE_MARKERS = ( + "challenges.cloudflare.com", + "challenge-platform", + "cf-browser-verification", + "challenge-error-text", + "_cf_chl_opt", +) + +# Exact titles served by challenge pages. Compared whole, so a post +# titled "How Cloudflare's Just a moment page works" does not match. +_CHALLENGE_TITLES = { + "just a moment...", + "just a moment", + "attention required! | cloudflare", + "access denied", + "checking your browser before accessing", + "one more step", +} + +_TITLE_RE = re.compile(r"<title[^>]*>([^<]*)", re.IGNORECASE | re.DOTALL) + + +def page_title(html: str) -> str | None: + """The document's ``, trimmed. None when absent or blank.""" + match = _TITLE_RE.search(html or "") + if not match: + return None + return match.group(1).strip() or None + + +def _is_challenge(page: Page) -> str | None: + title = (page_title(page.html) or "").strip().lower() + if title in _CHALLENGE_TITLES: + return f"bot protection served {title!r} instead of the page" + for marker in _CHALLENGE_MARKERS: + if marker in page.html: + return f"bot protection detected ({marker})" + return None + + +# ── Login ───────────────────────────────────────────────────────────── +# +# Detected from the landing URL, because that is where the signal +# actually is. Reddit's wall is the worked example: HTTP 200, a large +# document, a friendly title, and the only honest thing about it is +# that the redirect chain ended on `/login`. + +_LOGIN_PATHS = ("/login", "/signin", "/sign-in", "/sign_in", "/auth/login", "/accounts/login") + + +def _is_login(page: Page) -> str | None: + split = urlsplit(page.url or "") + path = split.path.rstrip("/").lower() + if any(path == p or path.endswith(p) for p in _LOGIN_PATHS): + return f"the site redirected to a sign-in page ({split.netloc}{split.path})" + if page.status in (401, 403) and "login" in split.query.lower(): + return "the site requires a sign-in" + return None + + +# ── Consent ─────────────────────────────────────────────────────────── +# +# A consent *wall* is a different host, not a banner. Nearly every +# European site carries a banner in the same document as its content; +# treating those as blocks would reject most of the web. So the signal +# is the dedicated consent host a site bounces to. + +_CONSENT_HOSTS = ("consent.google.com", "consent.youtube.com", "consent.yahoo.com") + + +def _is_consent(page: Page) -> str | None: + host = urlsplit(page.url or "").netloc.lower() + if host in _CONSENT_HOSTS: + return f"the site served a consent wall at {host}" + return None + + +# ── Paywall ─────────────────────────────────────────────────────────── +# +# 402 is the unambiguous case and the only one detected here. Prose +# detection ("Subscribe to continue reading") is deliberately absent: +# most paywalled pages carry that string *alongside* a readable +# excerpt, and an excerpt is still worth filing. + +def _is_paywall(page: Page) -> str | None: + if page.status == 402: + return "the site returned 402 Payment Required" + return None + + +# ── The gate ────────────────────────────────────────────────────────── + +def assess(page: Page, *, min_chars: int = DEFAULT_MIN_CHARS) -> Verdict: + """Classify a fetched page. Never raises. + + Order is deliberate. A challenge page is also short, and a login + wall is also nearly empty, so `empty` has to be the last thing + checked -- otherwise every block would be reported as "the page was + blank", which is true and useless. + + `min_chars` is the floor below which a body is assumed to be + furniture rather than an article. It is an argument so a site + profile can lower it for a domain that legitimately publishes short + pages, instead of that domain having to skip the gate. + """ + for check in (_is_challenge, _is_login, _is_consent, _is_paywall): + detail = check(page) + if detail: + return Verdict(check.__name__.removeprefix("_is_"), detail) + + body = (page.text or "").strip() + if len(body) < min_chars: + got = f"{len(body)} characters" if body else "nothing" + return Verdict("empty", f"the page yielded {got}, below the {min_chars}-character floor") + + return Verdict("ok", f"extracted {len(body)} characters") diff --git a/lib/stack/web/structured.py b/lib/stack/web/structured.py new file mode 100644 index 00000000..9a881ade --- /dev/null +++ b/lib/stack/web/structured.py @@ -0,0 +1,236 @@ +"""JSON-LD — the content a site already handed us, structured. + +Recipe sites publish schema.org markup because Google asks them to. +That markup is better than anything an extractor can recover from the +rendered page: the ingredients are a list rather than a paragraph, the +quantities are attached to them, and the steps are in order. It is also +free, deterministic, and needs no model. + +Measured on two German recipe sites: both served a complete `Recipe` +object to a plain fetch with no bot wall — yield, total time, 11 to 14 +ingredients with quantities, the instruction steps, and nutrition. + +So this runs before general extraction, and when it hits, the page is +never parsed as prose at all. + +The awkward part is that a listing page carries `Recipe` objects too: +"our 30 best apple cakes" embeds thirty of them, and filing that as a +recipe would produce an entry with the ingredients of whichever one +happened to be first. So a candidate is only accepted when it looks +like the page's *subject* — it must carry both ingredients and steps. + +Stdlib only: the host CLI runs this without a virtualenv. +""" + +from __future__ import annotations + +import json +import re +from typing import Any, Iterator + +from stack.web.content import SourceContent + +_LD_BLOCK_RE = re.compile( + r'<script[^>]*type\s*=\s*["\']application/ld\+json["\'][^>]*>(.*?)</script>', + re.IGNORECASE | re.DOTALL, +) + + +def _blocks(html: str) -> Iterator[Any]: + """Every parseable ld+json payload in the document. + + Sites ship malformed JSON more often than you would hope (trailing + commas, unescaped newlines in a description). One bad block must not + cost us a good one, so parse failures are skipped silently. + """ + for raw in _LD_BLOCK_RE.findall(html or ""): + try: + yield json.loads(raw) + except (ValueError, TypeError): + continue + + +def _objects(node: Any) -> Iterator[dict]: + """Walk a payload yielding every object in it. + + JSON-LD nests three ways in the wild — a bare object, a top-level + array, and a `@graph` list — and sites mix them. Walking everything + is shorter than handling each shape and cannot miss one. + """ + if isinstance(node, dict): + yield node + for value in node.values(): + yield from _objects(value) + elif isinstance(node, list): + for item in node: + yield from _objects(item) + + +def _has_type(obj: dict, wanted: str) -> bool: + raw = obj.get("@type") + types = raw if isinstance(raw, list) else [raw] + return any(isinstance(t, str) and t.lower() == wanted.lower() for t in types) + + +def _text(value: Any) -> str: + """Flatten a schema.org value into a string. + + Handles the shapes sites actually publish: a plain string, a bare + number (`"recipeYield": 4` is common — the field is typed as text + in the spec and half the web ignores that), an object carrying a + `name`/`text`, or a list of any of those. + """ + if isinstance(value, str): + return value.strip() + if isinstance(value, bool): + return "" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, dict): + for key in ("text", "name", "description"): + got = value.get(key) + if isinstance(got, str) and got.strip(): + return got.strip() + return "" + if isinstance(value, list): + return "\n".join(filter(None, (_text(v) for v in value))) + return "" + + +def _steps(value: Any) -> list[str]: + """Instruction steps, flattened out of however they were nested. + + `recipeInstructions` is a list of strings, a list of `HowToStep` + objects, a list of `HowToSection` objects each holding steps, or a + single paragraph. Sections are flattened rather than preserved: the + vault entry is prose, not a structured recipe format. + """ + if isinstance(value, str): + return [line.strip() for line in value.splitlines() if line.strip()] + if isinstance(value, dict): + if _has_type(value, "HowToSection"): + return _steps(value.get("itemListElement") or []) + got = _text(value) + return [got] if got else [] + if isinstance(value, list): + out: list[str] = [] + for item in value: + out.extend(_steps(item)) + return out + return [] + + +def _duration(iso: Any) -> str | None: + """`PT1H15M` as `1 h 15 min`. Returns None for anything else. + + Deliberately narrow: ISO 8601 durations have a full grammar, and + recipes only ever use hours and minutes. Anything unrecognised is + dropped rather than guessed at. + """ + if not isinstance(iso, str): + return None + match = re.fullmatch(r"PT(?:(\d+)H)?(?:(\d+)M)?", iso.strip(), re.IGNORECASE) + if not match or not any(match.groups()): + return None + hours, minutes = match.groups() + parts = [] + if hours: + parts.append(f"{int(hours)} h") + if minutes: + parts.append(f"{int(minutes)} min") + return " ".join(parts) + + +# ── Recipe ──────────────────────────────────────────────────────────── + +def _recipe_markdown(obj: dict) -> str | None: + """Render a `Recipe` object as the Markdown that goes in the vault. + + None when the object is a mention rather than the page's subject — + a listing page's thumbnails carry a name and an image but no + ingredients, and filing one of those would produce an entry named + after a recipe it does not contain. + """ + ingredients = [_text(i) for i in (obj.get("recipeIngredient") or [])] + ingredients = [i for i in ingredients if i] + steps = [s for s in _steps(obj.get("recipeInstructions") or []) if s] + if not ingredients or not steps: + return None + + lines: list[str] = [] + description = _text(obj.get("description")) + if description: + lines += [description, ""] + + facts = [] + servings = _text(obj.get("recipeYield")) + if servings: + facts.append(f"**Servings:** {servings}") + total = _duration(obj.get("totalTime")) or _duration(obj.get("cookTime")) + if total: + facts.append(f"**Total time:** {total}") + if facts: + lines += [" · ".join(facts), ""] + + lines += ["## Ingredients", ""] + lines += [f"- {i}" for i in ingredients] + lines += ["", "## Instructions", ""] + lines += [f"{n}. {s}" for n, s in enumerate(steps, 1)] + + return "\n".join(lines).strip() + + +# ── Article ─────────────────────────────────────────────────────────── + +def _article_markdown(obj: dict) -> str | None: + """A `NewsArticle`/`Article` body, when the site publishes one. + + Most do not — `articleBody` is optional and usually omitted, which + is why this returns None far more often than the recipe path and + why general extraction still exists. When it is present it is the + cleanest possible body: exactly what the publisher considers the + article, with no navigation to strip. + """ + body = _text(obj.get("articleBody")) + if not body: + return None + lines = [] + description = _text(obj.get("description")) + if description and description not in body: + lines += [description, ""] + lines.append(body) + return "\n".join(lines).strip() + + +# ── Entry point ─────────────────────────────────────────────────────── + +_READERS = ( + ("Recipe", _recipe_markdown), + ("Article", _article_markdown), + ("NewsArticle", _article_markdown), + ("BlogPosting", _article_markdown), +) + + +def read_structured(html: str, *, url: str | None = None) -> SourceContent | None: + """The page's own structured data as a `SourceContent`, if usable. + + Returns None whenever the markup is absent, unparseable, or present + but not about this page — the caller falls through to general + extraction, which is the common case. + """ + for payload in _blocks(html): + for obj in _objects(payload): + for wanted, render in _READERS: + if not _has_type(obj, wanted): + continue + body = render(obj) + if not body: + continue + return SourceContent( + text=body, + mime="text/markdown", + title_hint=_text(obj.get("headline") or obj.get("name")) or None, + source_uri=url, + ) + return None diff --git a/stacklets/docs/bot/archivist.py b/stacklets/docs/bot/archivist.py index 6e9faaee..e9826da0 100644 --- a/stacklets/docs/bot/archivist.py +++ b/stacklets/docs/bot/archivist.py @@ -32,6 +32,7 @@ import time from contextlib import contextmanager from pathlib import Path +from urllib.parse import urlsplit import aiohttp import yaml @@ -2406,6 +2407,15 @@ async def _reply_for_capture( transcript=o.transcript, todo_link=self._todo_link(o), ) + # A link card filed, but the family should hear why it has no + # summary -- and hear which obstacle it was. "Reddit wants you + # signed in" is something a person can act on; "couldn't read + # that link" is not. + if o.blocked_reason: + host = urlsplit(o.display_link or "").netloc or "the site" + note = self.t(f"capture_blocked_{o.blocked_reason}", host=host) + reply = f"{note}\n\n{reply}" + metadata = ( {"dev.famstack.event": o.envelope} if o.envelope else None ) diff --git a/stacklets/docs/bot/capture_pipeline.py b/stacklets/docs/bot/capture_pipeline.py index 7e6a504e..30ab9a38 100644 --- a/stacklets/docs/bot/capture_pipeline.py +++ b/stacklets/docs/bot/capture_pipeline.py @@ -19,7 +19,8 @@ import datetime as _dt import io import re -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from urllib.parse import unquote, urlsplit from loguru import logger @@ -103,6 +104,12 @@ class CaptureOutcome: envelope: dict | None = None transcript: str | None = None failure_reason: str | None = None + # Set when a URL capture filed as a *link card* rather than a summary: + # the gate's verdict name (`challenge`, `login`, `consent`, `paywall`, + # `empty`). The entry exists either way -- this is what the reply layer + # uses to say why there is a link but no summary. None on every other + # capture, including a URL that read cleanly. + blocked_reason: str | None = None # The vault bucket the capture filed under (`entity_slug`): a topic path # like `family/camping` or a bare personal bucket like `homer`. The reply # renderer uses it to build a `/go/topic/<scope>/todo` link, but only for @@ -110,6 +117,37 @@ class CaptureOutcome: scope: str | None = None +def _link_card(url: str, user_hint: str | None) -> SourceContent: + """A `SourceContent` for a page we could not read. + + The body is the sender's own words plus the link, and deliberately + nothing else. It is tempting to write "this page was blocked" into + it, but the body is the classifier's input: say that, and the entry + comes back titled "Blocked page" instead of "camping gear list". + Why it was blocked belongs in the chat reply, not in the vault. + + The title hint falls back through the same reasoning -- the sender's + first line, then the URL's last path segment, then the host. Each is + a worse guess than the one before it, and all three beat a summary + of a cookie banner. + """ + hint = (user_hint or "").strip() + split = urlsplit(url) + slug = [p for p in split.path.split("/") if p] + title = None + if hint: + title = hint.splitlines()[0].strip()[:120] + elif slug: + title = unquote(slug[-1]).replace("-", " ").replace("_", " ").strip() or None + if not title: + title = split.netloc + + body = f"{hint}\n\n{url}" if hint else url + return SourceContent( + text=body, mime="text/markdown", title_hint=title, source_uri=url, + ) + + class CapturePipeline: """Captures a URL (bookmark) or pasted text (note) into the vault.""" @@ -175,17 +213,27 @@ async def capture_url( out. Empty/None leaves the prompt unchanged. """ await notifier.acknowledge() - source = await self._url_extractor.extract(url) + outcome = await self._url_extractor.fetch(url) + + # A page we could not read still files. Dropping it on the floor + # was the old behaviour and it lost two things worth keeping: the + # link itself, and whatever the sender wrote around it -- which is + # often the more useful half ("gear list for the camping trip"). + # The gate's verdict rides along so the reply can name the + # obstacle instead of guessing. + blocked_reason = None + source = outcome.content if source is None: - return CaptureOutcome( - status="extract_failed", failure_reason="url", - ) - return await self._publish( + source = _link_card(outcome.url or url, user_hint) + blocked_reason = outcome.verdict.name + + result = await self._publish( source=source, kind="bookmark", sender_mxid=sender_mxid, display_link=url, actor=sender_mxid, capture_id=capture_id, seed_topics=seed_topics, bucket=bucket, user_hint=user_hint, ) + return replace(result, blocked_reason=blocked_reason) if blocked_reason else result async def capture_text( self, *, text: str, sender_mxid: str, diff --git a/stacklets/docs/bot/extractors.py b/stacklets/docs/bot/extractors.py index 02bb3f76..4b13317b 100644 --- a/stacklets/docs/bot/extractors.py +++ b/stacklets/docs/bot/extractors.py @@ -17,16 +17,16 @@ poll → OCR) because it's deeply Paperless-shaped; unifying it into a single backend would be churn without a real second consumer. -trafilatura is imported lazily so a Python test environment that -doesn't exercise URL extraction doesn't need the dep installed. In -production the bot-runner image always carries it (declared in -`stacklets/core/bot-runner/requirements.txt`). +`SourceContent` and the whole URL-reading ladder now live in +`lib/stack/web/`. They moved the moment a second consumer appeared: +the host CLI reads a URL with no archivist running. `UrlExtractor` +stays here as the bot's binding to it — an aiohttp session in, a +`SourceContent` out — so nothing upstream had to change. """ from __future__ import annotations import re -from dataclasses import dataclass import aiohttp from loguru import logger @@ -35,28 +35,10 @@ # here so existing `from extractors import parse_email, ParsedEmail` # callers and tests keep working after the move. from stack.email_message import ParsedEmail, parse_email # noqa: F401 - - -# ── SourceContent ──────────────────────────────────────────────────────── - -@dataclass -class SourceContent: - """The classifier's input, normalized across source types. - - `text` is the body the classifier reads — Markdown when the - extractor can produce it, plain text otherwise. `title_hint` is - whatever the source advertised as a title (HTML `<title>`, first - body line, filename); the classifier may overwrite it with - something more useful. `source_uri` is the canonical pointer - back to the origin (`https://...`, `paperless://42`, - `matrix:<event-id>` — caller decides the scheme), captured into - the mirror's frontmatter for round-tripping. None means the - capture has no upstream pointer (a pure pasted note). - """ - text: str - mime: str = "text/plain" - title_hint: str | None = None - source_uri: str | None = None +# Same arrangement for the capture types: the framework owns the shared +# shape, this module owns the docs-side mapping into it. +from stack.web import FetchOutcome, SourceContent # noqa: F401 +from stack.web.fetch import aiohttp_transport, fetch_url # ── Shared helpers ─────────────────────────────────────────────────────── @@ -96,82 +78,50 @@ def _first_url(text: str) -> str | None: # ── UrlExtractor ───────────────────────────────────────────────────────── class UrlExtractor: - """Fetch a URL and convert the HTML body to Markdown via trafilatura. - - Failure paths return None, never raise — the caller renders a - single "couldn't capture this" reply regardless of whether the - server 500'd, the host was unreachable, or trafilatura couldn't - find a body. Logging surfaces the distinction for debugging. - - Non-HTML content types are rejected at the gate. PDFs reach the - archivist through a separate path (`_handle_url`) that uploads to - Paperless; this extractor's job is web articles only. + """The bot's binding to the framework's fetch ladder. + + The reading itself — canonicalize, structured data, HTTP, + extraction, and the quality gate — is `stack.web.fetch`. This + supplies the one thing the framework deliberately does not have: a + transport. The bot already keeps an aiohttp session open for its + lifetime, so it hands that in rather than opening a second one. + + Two ways out, for two callers. `fetch` returns the full outcome + including the gate's verdict, which is what the capture pipeline + needs to explain a refusal to the family. `extract` keeps the older + "content or nothing" shape for callers that only care whether there + was something to file. + + Tier 3 (a real browser, in the optional `web` stacklet) is not + wired up yet. Until it is, a challenge page is terminal and the + family is told the site blocked us — which is the honest answer, + and a better one than the fabricated entry they used to get. """ def __init__(self, http: aiohttp.ClientSession, *, timeout: int = 30): self.http = http self.timeout = timeout - async def extract(self, url: str) -> SourceContent | None: - html = await self._fetch_html(url) - if html is None: - return None - - try: - import trafilatura - except ImportError: - logger.error( - "[extractor] trafilatura not installed — " - "URL captures require the bot-runner image", - ) - return None - - body = trafilatura.extract( - html, output_format="markdown", - include_links=True, include_images=False, - include_tables=True, - favor_precision=True, - ) - if not body or not body.strip(): - logger.info("[extractor] {} → trafilatura returned no body", url) - return None - - return SourceContent( - text=body.strip(), - mime="text/html", - title_hint=_html_title(html), - source_uri=url, + async def fetch(self, url: str) -> FetchOutcome: + """Read a URL and return a judged outcome. Never raises.""" + outcome = await fetch_url( + url, transport=aiohttp_transport(self.http, timeout=self.timeout), ) + if outcome.ok: + logger.info( + "[extractor] {} → tier {} ({}), {} chars", + url, outcome.tier, outcome.profile, len(outcome.content.text), + ) + else: + logger.info( + "[extractor] {} → {}: {}", + url, outcome.verdict.name, outcome.verdict.detail, + ) + return outcome - async def _fetch_html(self, url: str) -> str | None: - """GET the URL. Returns the body text on success, None on any - failure (non-200, non-HTML, transport error).""" - try: - async with self.http.get( - url, - timeout=aiohttp.ClientTimeout(total=self.timeout), - allow_redirects=True, - headers={"User-Agent": "famstack-archivist/1.0"}, - ) as resp: - if resp.status != 200: - logger.info( - "[extractor] {} → HTTP {}", url, resp.status, - ) - return None - ctype = (resp.content_type or "").lower() - if not ( - ctype.startswith("text/html") - or ctype.startswith("application/xhtml") - ): - logger.info( - "[extractor] {} → non-HTML content_type={}", - url, ctype, - ) - return None - return await resp.text() - except (aiohttp.ClientError, OSError) as e: - logger.warning("[extractor] {} → fetch failed: {}", url, e) - return None + async def extract(self, url: str) -> SourceContent | None: + """The fetched body, or None when there was nothing to file.""" + return (await self.fetch(url)).content # ── TextExtractor ──────────────────────────────────────────────────────── diff --git a/stacklets/docs/bot/messages/archivist.yml b/stacklets/docs/bot/messages/archivist.yml index ce326967..32d1a710 100644 --- a/stacklets/docs/bot/messages/archivist.yml +++ b/stacklets/docs/bot/messages/archivist.yml @@ -85,6 +85,15 @@ en: # URL capture (knowledge rooms — links become summarized notes, not Paperless docs) capture_failed: "\u274C Couldn't read that link. Either the host is unreachable or there's no article body to extract." capture_failed_binary: "\u274C Couldn't read that file." + # A blocked page still files: the link and whatever the sender wrote + # around it become a link card, and one of these lines says why there + # is no summary. Naming the obstacle beats "couldn't read that link" — + # the family can decide whether to open it themselves. + capture_blocked_challenge: "\U0001F512 {host} checks for bots, so I couldn't read the page. Saved the link." + capture_blocked_login: "\U0001F512 {host} only shows this to people who are signed in. Saved the link." + capture_blocked_consent: "\U0001F512 {host} served a consent wall instead of the page. Saved the link." + capture_blocked_paywall: "\U0001F512 {host} is paywalled. Saved the link." + capture_blocked_empty: "\U0001F4C4 No article text on that page. Saved the link." capture_llm_failed: "\U0001F4DD Captured the page, but classification skipped: {error}" capture_no_mirror: "\u274C Captures need the `code` stacklet up so I can write to the memory vault. Run `stack up code`." captured: "\u2705 Captured: {title}" @@ -288,6 +297,11 @@ de: # URL-Capture (Wissensräume — Links werden zu zusammengefassten Notizen, nicht zu Paperless-Dokumenten) capture_failed: "\u274C Konnte den Link nicht lesen. Entweder ist die Seite nicht erreichbar, oder es gibt keinen Artikel-Inhalt zum Extrahieren." capture_failed_binary: "\u274C Konnte diese Datei nicht lesen." + capture_blocked_challenge: "\U0001F512 {host} prüft auf Bots, deshalb konnte ich die Seite nicht lesen. Link gespeichert." + capture_blocked_login: "\U0001F512 {host} zeigt das nur angemeldeten Personen. Link gespeichert." + capture_blocked_consent: "\U0001F512 {host} hat statt der Seite eine Zustimmungs-Abfrage geliefert. Link gespeichert." + capture_blocked_paywall: "\U0001F512 {host} ist hinter einer Bezahlschranke. Link gespeichert." + capture_blocked_empty: "\U0001F4C4 Auf der Seite gab es keinen Artikeltext. Link gespeichert." capture_llm_failed: "\U0001F4DD Seite gespeichert, aber Klassifizierung übersprungen: {error}" capture_no_mirror: "\u274C Captures brauchen das `code`-Stacklet, damit ich in den Memory-Vault schreiben kann. Starte `stack up code`." captured: "\u2705 Gespeichert: {title}" diff --git a/tests/conftest.py b/tests/conftest.py index dc2f1b4b..ed68915a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -183,3 +183,43 @@ def git_unreachable_remote_clone(_git_states, tmp_path) -> GitPair: gone = tmp_path / "moved-away.git" _git("-C", str(pair.local), "remote", "set-url", "origin", str(gone)) return GitPair(remote=gone, local=pair.local) + + +# ── Web capture fixtures ────────────────────────────────────────────── +# +# Real pages, captured from the live sites the web plan measured, so a +# detector cannot be tuned against HTML that was written to satisfy it. +# `tests/fixtures/web/README.md` records where each came from and what +# was stripped. Both test trees use these: the gate lives in the +# framework, the extractor that calls it lives in a stacklet. + +_WEB_FIXTURES = Path(__file__).parent / "fixtures" / "web" + + +@pytest.fixture(scope="session") +def web_fixture(): + """Load a captured page by name (no `.html` suffix).""" + def _load(name: str) -> str: + path = _WEB_FIXTURES / f"{name}.html" + if not path.exists(): + available = ", ".join(sorted(p.stem for p in _WEB_FIXTURES.glob("*.html"))) + raise FileNotFoundError(f"no web fixture {name!r}; have: {available}") + return path.read_text(encoding="utf-8") + + return _load + + +@pytest.fixture(scope="session") +def extracted(): + """Run the real tier-2 extraction over fixture HTML. + + Tests assert on what trafilatura actually produces rather than on a + stand-in for it, so a library upgrade that changes extraction shows + up here instead of in production. + """ + from stack.web.fetch import extract_body + + def _extract(html: str, **kwargs) -> str: + return extract_body(html, **kwargs) or "" + + return _extract diff --git a/tests/fixtures/web/README.md b/tests/fixtures/web/README.md new file mode 100644 index 00000000..43a43184 --- /dev/null +++ b/tests/fixtures/web/README.md @@ -0,0 +1,53 @@ +# Web capture fixtures + +Real pages, captured from the live sites the web plan measured. + +They are here because a fixture written next to the detector that reads +it proves only that the two were written together. Every claim the gate +makes about how a site blocks is checkable against what the site +actually served. + +## Provenance + +Captured 2026-09-15 with a Chrome user agent, anonymous (no cookies, no +session), from a German residential IP. + +| Fixture | Source | Served | +|---|---|---| +| `cloudflare-challenge.html` | `https://www.decathlon.de/` | HTTP 403, Cloudflare managed challenge, `<title>Just a moment...` | +| `reddit-login-wall.html` | `https://old.reddit.com/r/selfhosted/` | HTTP 302 to `/login/?reason=lor2`, then HTTP 200, `Welcome to Reddit` | +| `google-maps-shell.html` | `https://www.google.com/maps/place/Brandenburger+Tor/` | HTTP 200, application shell, no prose but the site-wide meta description | +| `recipe-jsonld.html` | `https://www.essen-und-trinken.de/rezepte/48816-rzpt-griechischer-salat` | HTTP 200, complete schema.org `Recipe` | +| `shop-listing-ok.html` | `https://geizhals.de/` | HTTP 200, real listing, cookie banner in the same document | + +The last one is the negative control: a page carrying a consent banner +that must still pass the gate, because the content is right there. + +## What was stripped + +Each file had inline ` \ No newline at end of file diff --git a/tests/fixtures/web/google-maps-shell.html b/tests/fixtures/web/google-maps-shell.html new file mode 100644 index 00000000..4291114e --- /dev/null +++ b/tests/fixtures/web/google-maps-shell.html @@ -0,0 +1 @@ + Google Maps \ No newline at end of file diff --git a/tests/fixtures/web/recipe-jsonld.html b/tests/fixtures/web/recipe-jsonld.html new file mode 100644 index 00000000..01e32e35 --- /dev/null +++ b/tests/fixtures/web/recipe-jsonld.html @@ -0,0 +1,1501 @@ + + + + + + +Griechischer Salat Rezept - [ESSEN UND TRINKEN] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+Zum Inhalt +
+Ein Produkt von + +
+ +
+ +
+ +
+
+ +
+
+
Anzeige
+
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+
+ +
+
+
+
    +
  • Rezepte

  • Griechischer Salat Rezept

    • +
+
+

Griechischer Salat +

+
+
+
+
+
+ + +

+ +(242 Bewertungen) + +

+
+
+ +
+
+

+ + +
+
+ +
+ + + +
+
+
+
+
+
+
+
+Griechischer Salat +
+
+Foto: http://www.colourbox.de/ +
+
+Griechischer Salat kombiniert starke Zutaten: cremigen Feta, knackige Gurke, frische Paprika und zarte Oliven. Getoppt von feinen Zwiebeln. Das schmeckt nach Urlaub! +
+
+Folgen Sie uns bei Google +
+
+
+
Koch/Köchin: essen-und-trinken.de +
+
+ +
+
Anzeige
+ +
+
+
+
+Fertig in +35 Minuten +
+
+

Schwierigkeit

+

einfach

+
+
+Rezepteigenschaften +Vegetarisch, Schnell, Griechisch, Vorspeise, Beilage, Dressing - Vinaigrette, Salate, Gemüse, Käseprodukte, Kräuter, Milch + Milchprodukte +
+
+

Pro Portion

+Energie: 448 kcal, +Kohlenhydrate: 9 g, +Eiweiß: 16 g, +Fett: 38 g +
+
+
+
+ +
+
Anzeige
+ +
+
+
+

Zutaten

+ +Für + +
4
+ +Portionen + +500 +

+g +g +rote und gelbe Paprikaschoten +

+300 +

+g +g +Tomaten +(mittelgroß) +

+400 +

+g +g +Salatgurke +

+200 +

+g +g +rote Zwiebeln +

+0.5 +

+Bund +Bund +Minze +

+100 +

+g +g +schwarze Oliven +

+300 +

+g +g +Schafskäse +

+5 +

+El +El +Weißweinessig +

+8 +

+El +El +Olivenöl +

+ + +Salz +

+ + +Pfeffer +

+
Rezept bei Chefkoch speichern + +
+
Anzeige
+ +
+
+
+

+Zubereitung +

    +
  1. +
    +Paprikaschoten halbieren, entkernen und in 2 cm große Würfel schneiden. Tomaten sechsteln. Salatgurke längs vierteln und quer in 2 cm große Stücke schneiden. Zwiebeln in 1 cm dicke Scheiben schneiden. Minze in feine Streifen schneiden. Oliven halbieren, Schafskäse in 2 cm große Würfel schneiden.
    2. +
  2. +
    +Essig mit 10 El kaltem Wasser, Öl, Salz und Pfeffer in einer Schüssel verrühren. Paprikaschoten, Tomaten, Gurke, Zwiebeln, Minze, Schafskäse und Oliven mit dem Dressing mischen und kurz durchziehen lassen. Dazu passt Fladenbrot.
    3. +
+ +
+
Anzeige
+ +
+
+ +

+Griechischer Salat vom feinsten +

+Ein Griechischer Salat enthält einfache und gute Zutaten, das Ergebnis ist einfach fantastisch: Zu knackiger Gurke kommen fruchtige Tomaten, frische Paprika, schwarze und grüne Oliven, cremiger Schafskäse und rote Zwiebeln. Das Ganze wird mit hochwertigem Olivenöl, Salz und Pfeffer abgeschmeckt. Fertig!

+
+
Anzeige
+ +
+
+

+Je nach Region und persönlichen Vorlieben wird Griechischer Salat auch mit grünem Blattsalat, Knoblauch, Kräutern wie Oregano und Petersilie, Eiern und Weißkohl erweitert und das Dressing mit frischem Zitronensaft und Zucker abgeschmeckt.

+
+
+
+

+Griechischer Salat - auch Bauernsalat oder Choriatiki genannt - eignet sich hervorragend als Vorspeise für ein griechisches Menü oder als Beilagensalat zu Lammkoteletts und punktet bei jeder Grillparty!

+Grichischer Salat: Rezepte +

+Wenn Sie auf den Geschmack gekommen sind, sollten Sie sich unbedingt unsere Rezept-Ideen der griechischen Küche anschauen. Traumhaft!

+
+
Anzeige
+ +
+
+
+
+Griechischer Salat mit Lammkoteletts +
+
+

+Griechischer Salat mit Lammkoteletts

+ +
+
+ +
+25 Min. +930 kcal +
+
Urlaubsgefühle mit würzigem Salat, Lamm und einem Brotaufstrich aus Erbsen und Dicken Bohnen. Unvergleichlich gut!
+
+
+
+
+
+
+
+
+
+Griechischer Bauernsalat +
+
+

+Griechischer Bauernsalat

+ +
+
+ +
+20 Min. +432 kcal +
+
Frischer Bauernsalat: Gurken, Tomaten, Feta, rote Zwiebeln und schwarze Oliven, verfeinert mit einem Dressing aus Olivenöl und Zitronensaft. Perfekt als leichte Mahlzeit oder Beilage.
+
+
+

+Hier sollten Sie auch mal reingucken +

+
+Tzatziki Rezepte: klassisch & raffiniert +
+
+

+So gelingt Tzatziki

+
+
Sahnejoghurt, Salatgurke, Knoblauch und Olivenöl: Mit diesen Zutaten zaubern Sie cremig-frischen Tzatziki. Wir haben tolle Rezepte für Sie!
+
+
+
+
+Caesar Salad mit grünem Blattsalat, knusprigen Croûtons, Parmesanhobeln und cremigem Dressing in einem weißen Teller. +
+
+

+Caesar Salad – klassisch & einfach

+ +
+
+ +
+20 Min. +683 kcal +
+
Caesar Salad begeistert mit knackigem Römersalat, krossen Croûtons und einem raffinierten Dressing. Hier gibt es den zeitlosen Klassiker zum Nachmachen.
+
+
+ +
+
Anzeige
+ +
+
+
+
+
+
+

Übrigens: Um Ihrem griechischen Salat einen besonderen Pfiff zu geben, können Sie mit verschiedenen Schafskäse-Sorten experimentieren. Neben dem klassischen Feta können auch andere gereifte Schafskäse für interessante Geschmacksvariationen sorgen. Es darf aber natürlich auch Käse aus Kuhmilch sein, zum Beispiel Hirtenkäse. Achten Sie auf die Herkunft und Qualität des Käses, um den bestmöglichen Geschmack zu erzielen.

+
+
+
+
+ +
+
+
+ + +
+
Anzeige
+ +
+
+ +
+
Anzeige
+ +
+
+ + + +
+ +
+ +
+
+
+ +
+VG-Wort Pixel + + + \ No newline at end of file diff --git a/tests/fixtures/web/reddit-login-wall.html b/tests/fixtures/web/reddit-login-wall.html new file mode 100644 index 00000000..4a45726e --- /dev/null +++ b/tests/fixtures/web/reddit-login-wall.html @@ -0,0 +1,322 @@ + + + + + Welcome to Reddit + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Skip to main content + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + +
+
+ +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/fixtures/web/shop-listing-ok.html b/tests/fixtures/web/shop-listing-ok.html new file mode 100644 index 00000000..827f4ef3 --- /dev/null +++ b/tests/fixtures/web/shop-listing-ok.html @@ -0,0 +1,4237 @@ +Geizhals Preisvergleich Deutschland + + + + + + + + + + + + + + + + + +Zum Hauptinhalt +
+ +
+
+
+ +
+ +
+
+ +
+ + + + +
+
+
+ +Zum Hauptinhalt + +
+
+
+ + +
+
+ +
+
+ + + + + +
+ +
+ +

Geizhals.de

+
+
+ +
+
+

Was bewegt dich?

+

Entdecke Sportschuhe beliebter Marken zum Bestpreis.

+ +Alle Sneakers & Sportschuhe +
+
+ + + + +
+
+ + + +
+ +Anzeige +
+
+
+
    +
  1. + + +Handys ohne Vertrag + +
    2. +
  2. + + +Monitore + +
    3. +
  3. + + +Notebooks + +
    4. +
  4. + + +PCIe + +
    5. +
  5. + + +Mainboards + +
    6. +
  6. + + +Speicher + +
    7. +
  7. + + +PC-Gehäuse + +
    8. +
  8. + + +Kopfhörer & Headsets + +
    9. +
+
+
    +
  1. +

    + +Hardware + +

    +
      +
    1. Grafikkarten
      2. +
    2. Monitore
      3. +
    3. Festplatten & SSDs
      4. +
    4. Notebooks
      5. +
    5. Prozessoren (CPUs)
      6. +
    6. Mainboards
      7. +
    7. Eingabegeräte
      8. +
    8. Gehäuse
      9. +
    9. Tablets
      10. +
    10. Arbeitsspeicher (RAM)
      11. +
    11. Netzwerk
      12. +
    12. Luftkühlung
      13. +
    13. Netzteile & USV
      14. +
    14. Systeme
      15. +
    15. Wasserkühlung
      16. +
    +
    2. +
  2. +

    + +Telefon + +

    +
      +
    1. Handy & Smartphones
      2. +
    2. Smartwatches
      3. +
    +
    3. +
  3. +

    + +Haushalt + +

    +
      +
    1. Kühlen & Heizen
      2. +
    2. Staubsaugen & Reinigen
      3. +
    3. Küchenkleingeräte
      4. +
    4. Kühlen & Gefrieren
      5. +
    5. Kochen & Backen
      6. +
    6. Waschen & Trocknen
      7. +
    7. Kaffee & Tee
      8. +
    8. Besteck & Geschirr
      9. +
    9. Geschirrspülen
      10. +
    +
    4. +
  4. +

    + +Sport & Freizeit + +

    +
      +
    1. Outdoor
      2. +
    2. Sportschuhe
      3. +
    3. Bekleidung
      4. +
    4. Fahrradzubehör
      5. +
    5. Fahrräder
      6. +
    6. Fahrradkomponenten
      7. +
    7. Wassersport
      8. +
    8. Rollsport
      9. +
    9. Sportuhren
      10. +
    +
    5. +
  5. +

    + +Baumarkt & Garten + +

    +
      +
    1. Maschinen
      2. +
    2. Gebäudeautomation & Sicherheit
      3. +
    3. Gartenmaschinen
      4. +
    4. Stromerzeugung & -speicherung
      5. +
    5. Werkstattausstattung
      6. +
    6. Werkzeugzubehör & Verbrauchsmaterial
      7. +
    7. Tierbedarf
      8. +
    8. Elektroinstallation
      9. +
    9. Griller
      10. +
    +
    6. +
  6. +

    + +Video, Foto & TV + +

    +
      +
    1. Fernseher
      2. +
    2. Fotografie
      3. +
    3. Foto-/Videozubehör
      4. +
    +
    7. +
  7. +

    + +Audio & HiFi + +

    +
      +
    1. Kopfhörer & Headsets
      2. +
    2. HiFi-Komponenten
      3. +
    3. Professional Audio
      4. +
    +
    8. +
  8. +

    + +Drogerie + +

    +
      +
    1. Medikation & Nahrungsergänzung
      2. +
    2. Rasur & Haarentfernung
      3. +
    3. Parfümerie
      4. +
    4. Mund- und Zahnpflege
      5. +
    5. Gesundheit
      6. +
    6. Gesichts- & Körperpflege
      7. +
    7. Haarstyler
      8. +
    8. Haarpflege & -styling
      9. +
    9. Sonne & Bräunung
      10. +
    +
    9. +
  9. +

    + +Auto & Motorrad + +

    +
      +
    1. Autoreifen & Felgen
      2. +
    2. E-Ladesysteme
      3. +
    3. Motorrad
      4. +
    +
    10. +
  10. +

    + +Büro & Schule + +

    +
      +
    1. Drucker & Scanner
      2. +
    2. Büromöbel
      3. +
    3. Bürogeräte
      4. +
    +
    11. +
  11. +

    + +Spiele & Konsolen + +

    +
      +
    1. PlayStation 5 (PS5)
      2. +
    2. Nintendo Switch 2
      3. +
    3. Xbox Series X & Series S
      4. +
    +
    12. +
  12. +

    + +Spielzeug & Modellbau + +

    +
      +
    1. Bau- & Konstruktionsspiele
      2. +
    2. Unterhaltung
      3. +
    3. RC-Modellbau
      4. +
    +
    13. +
  13. +

    +Software & Filme +

    +
      +
    1. Office
      2. +
    2. Betriebssysteme
      3. +
    3. Blu-ray
      4. +
    +
    14. +
  14. +

    + +Vergleichsrechner + +

    +
      +
    1. Strom
      2. +
    2. Gas
      3. +
    3. Handytarife
      4. +
    4. DSL
      5. +
    +
    15. +
+
+
+ +
+
+
+
+ +
+
+

Deals

+
+
+
+
+ +
+ +Alle Deals + + +
+
+
+
+ + + + + + + + +
+
+
+
+
+

Ratgeber & Blog

+
+
+
+
+ +
+

+Kontaktgriller & Waffeleisen +

+

Lust auf duftende Waffeln, knusprige Paninis, ein kleines Steak mit Gemüse? Kontaktgrills gehören zu den Küchengeräten, bei denen du für wenig Geld ziemlich viel bekommst. Ein gutes Gerät kann dir Waffeln, Sandwiches, Paninis, Steaks, Gemüse oder sogar kleine Kuchen zubereiten. Gleichzeitig unterscheiden sich die Modelle deutlich darin, was sie tatsächlich können. Unser Ratgeber hilft dir, das optimale Gerät zu finden.

+
+Ratgeber | +
+
+
+
+ +
+

+Netzteile +

+

Das Netzteil wird da schnell als reines Zubehörteil behandelt und erst am Ende der Kaufliste berücksichtigt. Wir erklären dir, warum es sich durchaus lohnt, auch beim Netzteil gut zu überlegen und zeigen dir Schritt für Schritt, worauf es bei Wattzahl, Effizienzklasse, Formfaktor und Anschlüssen wirklich ankommt. Tipps für unterschiedliche Einsatzszenarien vom Büro-PC bis zur High-End-Workstation inklusive.

+
+Ratgeber | +
+
+
+
+ +
+

+Gasgriller +

+

Das Wetter passt, die Stimmung ist gut und ganz nebenbei brutzelt der Grill – mit einem Gasgrill ist Grillen auch für Unerfahrene keine Wissenschaft mehr. Gasgrills bieten eine komfortable und schnelle Möglichkeit, auch nach der Arbeit oder bei spontanem Besuch Köstlichkeiten zuzubereiten.

+
+Ratgeber | +
+
+
+
+
+
+ +
+

+Back to School, aber günstig: So spart man zum Schulbeginn +

+

Teurer Schulstart? Muss nicht sein: Schreibtische, Schultaschen, Bastelbedarf und Co sind online oft günstiger als im stationären Handel. Für alle Eltern, die schon bei den Besorgungen für das neue Schuljahr sind oder bald damit beginnen, haben wir die wichtigsten Spartipps zusammengefasst. Und damit auch die Kleinen etwas davon haben, verlosen wir außerdem eine Schultüte, gefüllt mit bunten Überraschungen. +

+
+Blog | +
+
+
+
+ +
+

+Glasklare Sache? Kärcher RCW 4 Fensterputzroboter im Test +

+

Der Kärcher RCW 4 Fensterputzroboter verspricht Sauberkeit ohne viel Aufwand und ist im Handumdrehen einsatzbereit. Das Gerät bietet vier automatische Reinigungsmodi und wird mittels Fernbedienung gesteuert. Im Praxistest zeigt sich aber, dass der RCW 4 nur dann gute Ergebnisse liefert, wenn die Ausgangslage stimmt. +

+
+Blog | +
+
+
+
+ +
+

+Nachhaltige Kopfhörer mit Top-Sound? – Fairphone Fairbuds XL (2025) im Test +

+

Die Fairphone Fairbuds XL (2025) sind hochwertige Allround-Kopfhörer mit einem außergewöhnlichen Nachhaltigkeitskonzept: Fast alle Teile lassen sich rasch und unkompliziert austauschen (z. B. Ohrpolster, Akku, Lautsprecher) und bestehen immerhin zu 50 % aus fair gehandelten und recycelten Materialien. Die Fairbuds XL (2025) überzeugen aber auch mit ihrer Performance und Alltagstauglichkeit. +

+
+Blog | +
+
+
+
+
+
+

+Preise vergleichen. +
+Energie sparen! +

+ +Verivox Logo + +
+
+ + +
+
+ + + + + + + + + + + +
Finde den besten Stromtarif für Dich:
+
+ + +
+
+ +
+ + + +
+
+
+ +
+ +
+
+
+ +
+
+
+
+
+ + + + + + + + + + +
Finde den besten Gastarif für Dich:
+
+ + +
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+
+
+
+ +
+
+
+
+
+

Top-10 Produkte

+
+
+
+ + + + + + + + + + +
+
+
+ +
+
+
+ +
+
+
+
+ + + + + +
+ + + diff --git a/tests/framework/test_web_fetch.py b/tests/framework/test_web_fetch.py new file mode 100644 index 00000000..cac8df7d --- /dev/null +++ b/tests/framework/test_web_fetch.py @@ -0,0 +1,298 @@ +"""The ladder's promise: cheap first, expensive only on proven failure. + +Two things are being pinned here, and the second matters more than the +first. + +The obvious one is ordering — structured data beats extraction, and a +browser is reached only when a browser could actually help. + +The load-bearing one is that *no tier skips the gate*. The bug that +started this work was an extractor whose success condition was "the +string is not empty". A 19-second stealth fetch that returns a +challenge page is exactly as wrong as an instant one, and costs more, +so tier 3 gets judged on the same terms as tier 2. + +Transports are plain callables here — the ladder takes one rather than +importing a client, so these tests drive the real ladder over real +fixture HTML with no network and no mocking of anything internal. +""" + +from __future__ import annotations + +from stack.web.fetch import Response, fetch_url + + +def serving(html: str, *, url: str = "https://example.com/article", + status: int = 200, content_type: str = "text/html"): + """A transport that answers every request with one page.""" + async def _transport(_url: str, _headers: dict) -> Response: + return Response(url=url, status=status, html=html, content_type=content_type) + return _transport + + +def unreachable(): + async def _transport(_url: str, _headers: dict): + raise OSError("connection refused") + return _transport + + +def recording(html: str, **kwargs): + """A transport that also records what it was asked for, so a test + can assert on the URL the ladder actually requested.""" + calls: list[str] = [] + + async def _transport(url: str, headers: dict) -> Response: + calls.append(url) + return Response(url=kwargs.get("url", url), status=kwargs.get("status", 200), + html=html, content_type="text/html") + return _transport, calls + + +ARTICLE = """Why local LLMs matter +

Why local LLMs matter

+

Running models on your own hardware changes the privacy calculus entirely. +Your prompts never leave the machine, the model never phones home, and you can +iterate without worrying about quotas or per-token billing at any point.

+

For a family this means one server in a closet replaces three separate cloud +subscriptions. The arithmetic works out somewhere around the fourth month, give +or take whatever the power company decides to charge this year.

+
""" + + +class TestTheHappyPath: + async def test_an_article_becomes_content_with_a_title(self): + outcome = await fetch_url("https://example.com/article", transport=serving(ARTICLE)) + + assert outcome.ok + assert outcome.verdict.name == "ok" + assert outcome.content is not None + assert "privacy calculus" in outcome.content.text + assert outcome.content.title_hint == "Why local LLMs matter" + + async def test_the_source_uri_is_where_we_landed(self): + """Redirects are normal and the vault should point at the page + that actually exists, not the link somebody pasted.""" + outcome = await fetch_url( + "https://example.com/old", + transport=serving(ARTICLE, url="https://example.com/new"), + ) + assert outcome.content is not None + assert outcome.content.source_uri == "https://example.com/new" + + async def test_tier_zero_runs_before_any_request(self): + """The transport must be asked for the canonical URL, not the + one handed in — otherwise the rewrite is decorative.""" + transport, calls = recording(ARTICLE) + await fetch_url("https://www.reddit.com/r/x/?utm_source=share", transport=transport) + + assert calls == ["https://old.reddit.com/r/x/"] + + +class TestStructuredDataWins: + async def test_json_ld_short_circuits_extraction(self, web_fixture): + """When a site publishes its own Recipe object there is nothing + to be gained by guessing at the rendered page.""" + outcome = await fetch_url( + "https://www.essen-und-trinken.de/rezepte/48816-rzpt-griechischer-salat", + transport=serving(web_fixture("recipe-jsonld")), + ) + + assert outcome.ok + assert outcome.tier == "1" + assert outcome.content is not None + assert "## Ingredients" in outcome.content.text + + async def test_a_page_without_markup_falls_through_to_extraction(self): + outcome = await fetch_url("https://example.com/article", transport=serving(ARTICLE)) + assert outcome.tier == "2" + + +class TestBlockedPagesProduceAReasonNotContent: + """Every one of these used to produce a vault entry.""" + + async def test_a_challenge_yields_no_content(self, web_fixture): + outcome = await fetch_url( + "https://www.decathlon.de/", + transport=serving(web_fixture("cloudflare-challenge"), status=403), + ) + + assert not outcome.ok + assert outcome.content is None + assert outcome.verdict.name == "challenge" + + async def test_a_login_wall_yields_no_content(self, web_fixture): + """The 200 status and the 320 KB body are what made this file + as an article. The landing URL is the honest signal.""" + outcome = await fetch_url( + "https://www.reddit.com/r/selfhosted/", + transport=serving( + web_fixture("reddit-login-wall"), + url="https://old.reddit.com/login/?reason=lor2", + ), + ) + + assert not outcome.ok + assert outcome.verdict.name == "login" + + async def test_the_reason_names_the_site_where_we_know_it(self, web_fixture): + """A profile's note beats generic advice: the family is told + what is actually true about reddit.""" + outcome = await fetch_url( + "https://www.reddit.com/r/selfhosted/", + transport=serving( + web_fixture("reddit-login-wall"), + url="https://old.reddit.com/login/?reason=lor2", + ), + ) + assert "signed in" in outcome.verdict.detail + + async def test_a_non_html_url_is_declined_with_its_type(self): + """PDFs reach the archivist by a different road (upload to + Paperless). This path is web pages only, and says so.""" + outcome = await fetch_url( + "https://example.com/report.pdf", + transport=serving("%PDF-1.7", content_type="application/pdf"), + ) + + assert not outcome.ok + assert "application/pdf" in outcome.verdict.detail + + async def test_an_unreachable_host_is_a_verdict_not_an_exception(self): + """The ladder's contract is that it never raises: a caller + rendering a chat reply has nothing to do with a socket error.""" + outcome = await fetch_url("https://example.invalid/x", transport=unreachable()) + + assert not outcome.ok + assert outcome.verdict.detail + + +class TestEscalation: + """Tier 3 is the only expensive rung, so when it runs is the whole + cost model.""" + + async def test_a_challenge_escalates_and_the_browser_result_is_used(self, web_fixture): + outcome = await fetch_url( + "https://www.decathlon.de/", + transport=serving(web_fixture("cloudflare-challenge"), status=403), + stealth=serving(ARTICLE, url="https://www.decathlon.de/"), + ) + + assert outcome.ok + assert outcome.tier == "3" + + async def test_a_login_wall_does_not_escalate(self, web_fixture): + """A browser is served the same login wall, slower. Escalating + anything a browser cannot fix turns tier 3 into a tax.""" + calls = [] + + async def _stealth(url: str, headers: dict): + calls.append(url) + return Response(url=url, status=200, html=ARTICLE) + + await fetch_url( + "https://www.reddit.com/r/x/", + transport=serving(web_fixture("reddit-login-wall"), + url="https://old.reddit.com/login/?reason=lor2"), + stealth=_stealth, + ) + + assert calls == [], "a login wall must not reach the browser tier" + + async def test_an_empty_page_does_not_escalate(self): + calls = [] + + async def _stealth(url: str, headers: dict): + calls.append(url) + return Response(url=url, status=200, html=ARTICLE) + + await fetch_url( + "https://example.com/x", + transport=serving(""), + stealth=_stealth, + ) + + assert calls == [] + + async def test_the_browser_tier_is_judged_too(self, web_fixture): + """A stealth fetch that is also served a challenge is exactly + as wrong as a cheap one, and cost 19 seconds. It gets the same + gate.""" + challenge = web_fixture("cloudflare-challenge") + outcome = await fetch_url( + "https://www.decathlon.de/", + transport=serving(challenge, status=403), + stealth=serving(challenge, status=403), + ) + + assert not outcome.ok + assert outcome.content is None + + async def test_escalation_happens_at_most_once(self, web_fixture): + """There is no loop back to tier 2. If the browser is blocked + too, the answer is that we cannot read this page.""" + challenge = web_fixture("cloudflare-challenge") + calls = [] + + async def _stealth(url: str, headers: dict): + calls.append(url) + return Response(url=url, status=403, html=challenge) + + await fetch_url( + "https://www.decathlon.de/", + transport=serving(challenge, status=403), + stealth=_stealth, + ) + + assert len(calls) == 1 + + async def test_without_the_web_stacklet_a_challenge_is_terminal(self, web_fixture): + """`stealth=None` is what "the web stacklet is not installed" + looks like. It must degrade to an honest reason, not an error.""" + outcome = await fetch_url( + "https://www.decathlon.de/", + transport=serving(web_fixture("cloudflare-challenge"), status=403), + stealth=None, + ) + + assert outcome.verdict.name == "challenge" + assert outcome.verdict.detail + + async def test_a_broken_browser_tier_falls_back_to_the_cheap_verdict(self, web_fixture): + """If the stealth service is down, the family still gets told + the site was blocked rather than nothing at all.""" + outcome = await fetch_url( + "https://www.decathlon.de/", + transport=serving(web_fixture("cloudflare-challenge"), status=403), + stealth=unreachable(), + ) + + assert outcome.verdict.name == "challenge" + + +class TestShellPages: + """A page can be HTTP 200, well-formed, and have nothing to read.""" + + async def test_a_maps_place_is_answered_from_the_url(self, web_fixture): + """Google Maps renders in the browser. Rather than a link card, + the place name in the path makes a real entry the family can + find again.""" + outcome = await fetch_url( + "https://www.google.com/maps/place/Brandenburger+Tor/", + transport=serving(web_fixture("google-maps-shell")), + ) + + assert outcome.ok + assert outcome.tier == "0" + assert outcome.content is not None + assert outcome.content.title_hint == "Brandenburger Tor" + + async def test_an_ordinary_shell_page_is_declined(self): + """Without a profile saying the URL carries the subject, an + empty document is just empty.""" + outcome = await fetch_url( + "https://example.com/app", + transport=serving("App"), + ) + + assert not outcome.ok + assert outcome.verdict.name == "empty" diff --git a/tests/framework/test_web_profiles.py b/tests/framework/test_web_profiles.py new file mode 100644 index 00000000..b4751923 --- /dev/null +++ b/tests/framework/test_web_profiles.py @@ -0,0 +1,122 @@ +"""Tier 0: the free step, and the one that changes the most outcomes. + +Canonicalization runs before any request. It costs nothing, and it does +two jobs — it stops the vault accumulating four URLs for one page, and +it keeps the person who shared a link out of the link. A campaign tag +says "Marge clicked this from a newsletter", and that ends up in chat +and in the wiki alongside the entry. +""" + +from __future__ import annotations + +import pytest + +from stack.web.profiles import canonicalize, profile_for, strip_tracking, title_from_url + + +class TestTrackingParametersAreDropped: + """Campaign and referrer tags identify the sharer, not the page.""" + + @pytest.mark.parametrize("param", [ + "utm_source=newsletter", "utm_medium=email", "fbclid=abc123", + "gclid=xyz", "igshid=99", "mc_eid=deadbeef", "ref_src=twsrc", + ]) + def test_known_trackers_go(self, param): + cleaned = strip_tracking(f"https://example.com/article?{param}") + assert cleaned == "https://example.com/article" + + def test_load_bearing_parameters_stay(self): + """A query string is usually the page. Stripping an article id + or a page number would turn a good link into a 404, so anything + unrecognised is kept.""" + url = "https://example.com/search?q=immich&page=3&id=4711" + assert strip_tracking(url) == url + + def test_trackers_are_removed_from_among_real_parameters(self): + cleaned = strip_tracking("https://example.com/p?id=7&utm_source=x&page=2") + assert "id=7" in cleaned and "page=2" in cleaned + assert "utm_source" not in cleaned + + def test_a_url_with_no_query_is_untouched(self): + url = "https://example.com/article" + assert strip_tracking(url) == url + + +class TestHostRewrites: + """Reddit is the only rewrite that ships. It is kept even though it + no longer recovers the post, because of what it does to the *reason* + — see the profile's own comment.""" + + @pytest.mark.parametrize("given", [ + "https://www.reddit.com/r/selfhosted/comments/abc/title/", + "https://reddit.com/r/selfhosted/comments/abc/title/", + ]) + def test_reddit_is_rewritten_to_the_old_frontend(self, given): + assert canonicalize(given).startswith("https://old.reddit.com/r/selfhosted/") + + def test_a_path_survives_the_rewrite(self): + out = canonicalize("https://www.reddit.com/r/selfhosted/comments/abc/title/") + assert out.endswith("/r/selfhosted/comments/abc/title/") + + def test_already_canonical_urls_are_left_alone(self): + url = "https://old.reddit.com/r/selfhosted/" + assert canonicalize(url) == url + + def test_unprofiled_hosts_are_not_rewritten(self): + url = "https://example.com/article" + assert canonicalize(url) == url + + def test_canonicalize_strips_trackers_too(self): + out = canonicalize("https://www.reddit.com/r/x/?utm_source=share") + assert "utm_source" not in out + assert "old.reddit.com" in out + + +class TestProfileLookup: + def test_subdomains_resolve_to_the_parent_profile(self): + assert profile_for("https://old.reddit.com/r/x/").name == "reddit" + assert profile_for("https://www.reddit.com/r/x/").name == "reddit" + + def test_an_unknown_host_gets_the_default(self): + assert profile_for("https://example.com/").name == "default" + + def test_a_lookalike_host_does_not_match(self): + """Suffix matching must be on label boundaries: `notreddit.com` + is somebody else's site.""" + assert profile_for("https://notreddit.com/r/x/").name == "default" + + def test_reddit_asks_for_recall_and_comments(self): + """A Reddit page's content is the discussion under it. Measured + at the time: precision returned 821 characters of sidebar, + recall returned the 5513-character post.""" + profile = profile_for("https://old.reddit.com/r/x/") + assert profile.favor_recall + assert profile.include_comments + + def test_a_blocked_site_carries_its_own_explanation(self): + """Generic advice is worse than naming the reason where we + already know it.""" + assert "signed in" in (profile_for("https://old.reddit.com/r/x/").blocked_note or "") + + +class TestTitleFromPath: + """Some pages have no text because they are an application. A + Google Maps place is the worked example: HTTP 200, a full document, + and the only prose is the site-wide meta description. The place + name is in the path we were handed.""" + + def test_a_maps_place_name_is_recovered_from_the_url(self): + title = title_from_url("https://www.google.com/maps/place/Brandenburger+Tor/") + assert title == "Brandenburger Tor" + + def test_percent_escapes_are_decoded(self): + title = title_from_url("https://www.google.com/maps/place/Caf%C3%A9+Einstein/") + assert title == "Café Einstein" + + def test_an_ordinary_article_url_yields_nothing(self): + """A slug makes a poor title when the document has a real one, + so this never guesses outside a profile that asked for it.""" + assert title_from_url("https://example.com/2026/why-local-llms-matter") is None + + def test_a_google_url_that_is_not_a_place_yields_nothing(self): + assert title_from_url("https://www.google.com/search?q=famstack") is None diff --git a/tests/framework/test_web_quality.py b/tests/framework/test_web_quality.py new file mode 100644 index 00000000..4b87aefa --- /dev/null +++ b/tests/framework/test_web_quality.py @@ -0,0 +1,221 @@ +"""What the quality gate promises: no blocked page reaches the vault. + +The archivist used to treat "trafilatura returned some text" as success. +A Cloudflare interstitial, a login wall and a JavaScript shell all +return text, so all three were filed as if they were articles. This +module pins the opposite promise: every fetched page is classified +before anything downstream sees it, and the classification is a *reason* +rather than a boolean, because the reason is what the family reads in +chat and what we grep for in the logs. + +The fixtures under `tests/fixtures/web/` are real captures, not +hand-written HTML. That matters: a fixture written next to the detector +would encode the same assumption as the detector and both would agree +while reality disagreed. Each was fetched from the live site, then had +inline script and style *bodies* stripped (see the fixtures README) -- +every tag, attribute, meta element and ld+json block the gate reads is +exactly what the site served. +""" + +from __future__ import annotations + +import pytest + +from stack.web.quality import Page, assess + + +# ── Blocked pages are named, not merely rejected ────────────────────── + +class TestBlockedPagesAreClassifiedByReason: + """Each block shape gets its own verdict. The caller renders a + different chat reply per reason, so collapsing them to "failed" + would be a regression in what the family is told.""" + + def test_cloudflare_interstitial_is_a_challenge(self, web_fixture): + """decathlon.de answers an anonymous fetch with Cloudflare's + "Just a moment..." page at HTTP 403. This is the page that a + browser tier could plausibly get past, so it must be named + `challenge` and not lumped in with a hard refusal.""" + verdict = assess(Page( + url="https://www.decathlon.de/", + status=403, + html=web_fixture("cloudflare-challenge"), + )) + + assert verdict.name == "challenge" + assert not verdict.ok + + def test_login_wall_is_detected_from_the_final_url(self, web_fixture): + """old.reddit.com 302s an anonymous reader to `/login/?reason=lor2` + and serves 320 KB of JavaScript shell titled "Welcome to Reddit". + + Nothing in the body says "you must log in", so a detector that + only reads HTML is blind to it -- and the 200 status plus the + large body is exactly what made this file as a real article. The + landing URL is the honest signal, which is why the gate is given + the URL it ended on rather than the one it was asked for.""" + verdict = assess(Page( + url="https://old.reddit.com/login/?reason=lor2&dest=https%3A%2F%2Fold.reddit.com%2Fr%2Fselfhosted%2F", + status=200, + html=web_fixture("reddit-login-wall"), + text="Welcome to Reddit\n\nThe heart of the internet", + )) + + assert verdict.name == "login" + + def test_javascript_shell_with_no_article_is_empty(self, web_fixture): + """A Google Maps place URL returns HTTP 200 and a full HTML + document whose only prose is the site-wide meta description. + There is no page to read, so the verdict is `empty` -- the + caller turns that into a link card rather than an entry + summarising "Find local businesses, view maps".""" + verdict = assess(Page( + url="https://www.google.com/maps/place/Brandenburger+Tor/", + status=200, + html=web_fixture("google-maps-shell"), + text="", + )) + + assert verdict.name == "empty" + + def test_consent_redirect_is_named_consent(self): + """Google bounces EU traffic to `consent.google.com` before it + will serve anything. The host is the contract here -- it is a + documented, stable redirect target, unlike the wording of the + banner, which is localised and changes.""" + verdict = assess(Page( + url="https://consent.google.com/m?continue=https://www.google.com/search", + status=200, + html="Bevor du zu Google weitergehst", + )) + + assert verdict.name == "consent" + + def test_payment_required_status_is_a_paywall(self): + """HTTP 402 is rare but unambiguous. Naming it separately keeps + "we could fetch this with a browser" (challenge) apart from + "no amount of fetching will help" (paywall).""" + verdict = assess(Page( + url="https://example.com/article", + status=402, + html="Subscribe to continue", + )) + + assert verdict.name == "paywall" + + +# ── Good pages must survive the gate ────────────────────────────────── + +class TestRealPagesPass: + """A gate that rejects everything is not a fix. These are live + captures of pages we *want* filed, including ones carrying the + cookie banners and bot-protection vocabulary that a careless + detector would trip over.""" + + def test_article_with_a_body_is_ok(self, web_fixture, extracted): + """essen-und-trinken.de serves a recipe page to a plain fetch. + Once trafilatura has a body, the verdict is `ok`.""" + html = web_fixture("recipe-jsonld") + verdict = assess(Page( + url="https://www.essen-und-trinken.de/rezepte/48816-rzpt-griechischer-salat", + status=200, + html=html, + text=extracted(html), + )) + + assert verdict.name == "ok" + assert verdict.ok + + def test_shop_page_with_a_cookie_banner_is_still_ok(self, web_fixture, extracted): + """geizhals.de serves its real listing to an anonymous fetch and + carries a cookie consent banner in the same document. The banner + must not be read as a consent *wall*: the content is right + there. This is the false-positive guard for consent detection.""" + html = web_fixture("shop-listing-ok") + verdict = assess(Page( + url="https://geizhals.de/", + status=200, + html=html, + text=extracted(html), + )) + + assert verdict.name == "ok" + + def test_prose_about_bot_protection_is_not_a_challenge(self): + """An article *about* Cloudflare contains every word the + challenge page does. Detection keys on structure -- the exact + ``, Cloudflare's own DOM ids, its challenge host -- so + writing about the thing does not trip the detector for it.""" + verdict = assess(Page( + url="https://example.com/blog/cloudflare", + status=200, + html="<html><head><title>How Cloudflare's Just a moment page works", + text=( + "Cloudflare's interstitial shows the text 'Just a moment...' " + "while it runs a challenge. Enable JavaScript and cookies to " + "continue is the fallback message shown to clients that cannot " + "execute the challenge script. This post explains what the " + "browser is actually doing during those few seconds and why " + "the check exists at all for high-traffic origins." + ), + )) + + assert verdict.name == "ok" + + +# ── The empty floor ─────────────────────────────────────────────────── + +class TestEmptyFloor: + """`ok` requires enough prose to be worth a vault entry. The floor + exists because the failure it prevents is silent: a sidebar, a + cookie notice or a nav rail extracts cleanly and reads like content + to everything downstream.""" + + def test_no_text_is_empty(self): + verdict = assess(Page(url="https://example.com", status=200, html="", text="")) + assert verdict.name == "empty" + + def test_whitespace_only_text_is_empty(self): + verdict = assess(Page(url="https://example.com", status=200, html="", text=" \n\n ")) + assert verdict.name == "empty" + + def test_a_nav_rail_sized_body_is_empty(self): + """trafilatura on a blocked Reddit page returned 821 characters + of subreddit sidebar. Short extractions are the signature of + having scraped furniture instead of an article.""" + verdict = assess(Page( + url="https://example.com", + status=200, + html="", + text="Home | About | Archive | Subscribe | Contact", + )) + assert verdict.name == "empty" + + def test_the_floor_is_caller_tunable(self): + """A site profile may legitimately produce short bodies. The + floor is an argument so a profile can lower it rather than + forcing the caller to bypass the gate entirely.""" + short = "Two short sentences. That is the whole page." + assert assess(Page(url="https://e.com", status=200, html="", text=short)).name == "empty" + assert assess(Page(url="https://e.com", status=200, html="", text=short), min_chars=10).name == "ok" + + +# ── Verdicts carry a reason ─────────────────────────────────────────── + +class TestVerdictCarriesDetail: + """The reason is a value because it has two readers: the chat reply + the family sees, and whoever greps the logs asking why a link did + not file.""" + + @pytest.mark.parametrize("name", ["challenge", "login", "consent", "paywall", "empty"]) + def test_every_failure_explains_itself(self, name, web_fixture): + pages = { + "challenge": Page(url="https://d.de/", status=403, html=web_fixture("cloudflare-challenge")), + "login": Page(url="https://old.reddit.com/login/?reason=lor2", status=200, html="", text="x" * 400), + "consent": Page(url="https://consent.google.com/m", status=200, html=""), + "paywall": Page(url="https://e.com/a", status=402, html=""), + "empty": Page(url="https://e.com/a", status=200, html="", text=""), + } + verdict = assess(pages[name]) + assert verdict.name == name + assert verdict.detail, "a failure verdict with no detail tells nobody anything" diff --git a/tests/framework/test_web_structured.py b/tests/framework/test_web_structured.py new file mode 100644 index 00000000..7a39b834 --- /dev/null +++ b/tests/framework/test_web_structured.py @@ -0,0 +1,192 @@ +"""What JSON-LD buys us: a recipe nobody had to guess at. + +Recipe sites publish schema.org markup for Google's benefit, and it is +strictly better than anything recoverable from the rendered page — the +ingredients are a list with quantities attached, the steps are ordered, +and no model was involved. This module pins that we read it, and that +we decline to read it when it is describing something other than the +page in front of us. + +The recipe fixture is a live capture of essen-und-trinken.de, ld+json +untouched. Asserting against markup the site actually publishes is the +point: a hand-written fixture would agree with the parser by +construction and prove only that the two were written together. +""" + +from __future__ import annotations + +from stack.web.structured import read_structured + + +class TestRecipeIsReadWithoutAModel: + """The measured case: a German recipe site, a plain fetch, a + complete Recipe object.""" + + def test_real_recipe_page_yields_ingredients_and_steps(self, web_fixture): + content = read_structured( + web_fixture("recipe-jsonld"), + url="https://www.essen-und-trinken.de/rezepte/48816-rzpt-griechischer-salat", + ) + + assert content is not None + assert content.mime == "text/markdown" + assert "## Ingredients" in content.text + assert "## Instructions" in content.text + + def test_the_title_comes_from_the_markup_not_the_slug(self, web_fixture): + content = read_structured(web_fixture("recipe-jsonld")) + + assert content is not None + assert "Griechischer Salat" in (content.title_hint or "") + + def test_every_published_ingredient_survives(self, web_fixture): + """The site publishes 11 ingredients. Dropping one silently + would be the worst kind of bug here — the entry still looks + right, and the family only finds out at the stove.""" + content = read_structured(web_fixture("recipe-jsonld")) + + assert content is not None + bullets = [ln for ln in content.text.splitlines() if ln.startswith("- ")] + assert len(bullets) == 11 + + def test_servings_and_total_time_are_rendered_readably(self, web_fixture): + """`PT35M` is not something to show a family.""" + content = read_structured(web_fixture("recipe-jsonld")) + + assert content is not None + assert "**Servings:** 4" in content.text + assert "**Total time:** 35 min" in content.text + + +class TestListingPagesAreDeclined: + """A round-up page embeds a Recipe object per thumbnail. Accepting + one would file "our 30 best apple cakes" under the ingredients of + whichever cake happened to be first in the markup.""" + + def test_a_recipe_mention_without_ingredients_is_not_the_page(self): + listing = """ + + """ + assert read_structured(listing) is None + + def test_a_recipe_with_ingredients_but_no_steps_is_not_the_page(self): + partial = """ + + """ + assert read_structured(partial) is None + + +class TestMarkupShapesInTheWild: + """JSON-LD nests three ways and sites mix them. Each shape here is + one seen in real markup, not a hypothetical.""" + + def test_graph_wrapped_payloads_are_found(self): + graphed = """ + + """ + content = read_structured(graphed) + assert content is not None + assert content.title_hint == "Suppe" + assert "1. Erhitzen." in content.text + + def test_type_as_a_list_still_matches(self): + """Sites commonly declare `"@type": ["Recipe", "NewsArticle"]`.""" + multi = """ + + """ + content = read_structured(multi) + assert content is not None + assert "1. Kneten." in content.text + assert "2. Backen." in content.text + + def test_sections_are_flattened_into_one_numbered_list(self): + """`HowToSection` groups steps ("for the dough", "for the + topping"). The vault entry is prose, so the grouping is dropped + and the order is kept.""" + sectioned = """ + + """ + content = read_structured(sectioned) + assert content is not None + assert "1. Mehl sieben." in content.text + assert "3. Äpfel schneiden." in content.text + + def test_one_broken_block_does_not_cost_a_good_one(self): + """Malformed ld+json is common. A trailing comma in the + breadcrumb block must not hide the recipe in the next one.""" + mixed = """ + + + + """ + content = read_structured(mixed) + assert content is not None + assert content.title_hint == "Salat" + + +class TestArticles: + """`articleBody` is optional and usually absent, which is why + general extraction still exists. When present it is the cleanest + body available — the publisher's own idea of the article.""" + + def test_article_body_is_used_when_published(self): + article = """ + + """ + content = read_structured(article) + assert content is not None + assert content.title_hint == "Local LLMs" + assert "Running models on your own hardware" in content.text + + def test_an_article_stub_with_no_body_falls_through(self): + """Almost every news page carries a headline-only NewsArticle + object. Returning it would replace the real article with its + own teaser.""" + stub = """ + + """ + assert read_structured(stub) is None + + +class TestNoMarkup: + def test_a_page_without_json_ld_falls_through(self): + assert read_structured("

Just prose.

") is None + + def test_empty_input_is_handled(self): + assert read_structured("") is None diff --git a/tests/stacklets/test_archivist_routing.py b/tests/stacklets/test_archivist_routing.py index 7cd0de5c..9163cf10 100644 --- a/tests/stacklets/test_archivist_routing.py +++ b/tests/stacklets/test_archivist_routing.py @@ -607,6 +607,21 @@ async def test_capture_success_checks(self, tmp_path, monkeypatch): o = SimpleNamespace( status="captured", source_title_hint="t", classification={}, display_link="http://x", transcript=None, envelope=None, scope=None, + blocked_reason=None, + ) + await bot._reply_for_capture("!r:server", o, "$tgt") + assert reacts == [self.CHECK] + + async def test_blocked_capture_still_checks_because_it_filed(self, tmp_path, monkeypatch): + """A link card is a successful capture. The 🔒 line explains why + there is no summary, but the entry exists, so the glyph is ✅ -- + ❌ would say nothing was kept, which is the old lie.""" + bot, reacts = self._bot(tmp_path) + monkeypatch.setattr("archivist.render_capture_reply", lambda *a, **k: "x") + o = SimpleNamespace( + status="captured", source_title_hint="t", classification={}, + display_link="https://www.decathlon.de/", transcript=None, + envelope=None, scope=None, blocked_reason="challenge", ) await bot._reply_for_capture("!r:server", o, "$tgt") assert reacts == [self.CHECK] diff --git a/tests/stacklets/test_capture_pipeline.py b/tests/stacklets/test_capture_pipeline.py index 082ebe2f..7b2a1b2b 100644 --- a/tests/stacklets/test_capture_pipeline.py +++ b/tests/stacklets/test_capture_pipeline.py @@ -15,10 +15,13 @@ import pytest _REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_REPO_ROOT / "lib")) sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "docs" / "bot")) from capture_pipeline import CapturePipeline # noqa: E402 from vault_entry import capture_frontmatter, render_capture # noqa: E402 +from stack.web import FetchOutcome # noqa: E402 +from stack.web.quality import Verdict # noqa: E402 def _source(*, text="article body", source_uri=None, title_hint="A Title"): @@ -26,12 +29,32 @@ def _source(*, text="article body", source_uri=None, title_hint="A Title"): class FakeExtractor: - def __init__(self, result): + """Stands in for the real extractor at the pipeline's seam. + + The URL path asks for `fetch`, because it needs the gate's verdict + to explain a refusal; the text path still asks for `extract`. A + None result means the gate refused the page, and `verdict` says + which refusal, so a test can pin what the pipeline was told rather + than just that something went wrong. + """ + + def __init__(self, result, verdict="challenge"): self._result = result + self._verdict = verdict async def extract(self, _arg): return self._result + async def fetch(self, url): + if self._result is None: + return FetchOutcome( + verdict=Verdict(self._verdict, f"the site returned a {self._verdict}"), + url=url, + ) + return FetchOutcome( + verdict=Verdict("ok", "extracted"), content=self._result, url=url, + ) + class FakeClassifier: def __init__(self, payload=None, raises=None): @@ -120,14 +143,17 @@ async def acknowledge(self): self.acknowledged += 1 -def _pipeline(*, mirror, classifier=None, capture_keep_body=False, - llm=None, text_extractor=None): +_UNSET = object() + + +def _pipeline(*, mirror=_UNSET, classifier=None, capture_keep_body=False, + llm=None, text_extractor=None, url_extractor=None): return CapturePipeline( - url_extractor=FakeExtractor(_source(source_uri="http://src")), + url_extractor=url_extractor or FakeExtractor(_source(source_uri="http://src")), text_extractor=text_extractor or FakeExtractor(_source(source_uri="http://embedded")), classifier=classifier or FakeClassifier(), - mirror=mirror, + mirror=FakeMirror() if mirror is _UNSET else mirror, capture_tags=FakeTags(), paperless=FakePaperless(), bot_name="archivist-bot", @@ -157,26 +183,70 @@ async def test_acknowledges_then_captures(self): assert out.display_link == "http://example.com" @pytest.mark.asyncio - async def test_extract_failure(self): - pipe = CapturePipeline( - url_extractor=FakeExtractor(None), # extraction fails - text_extractor=FakeExtractor(None), - classifier=FakeClassifier(), - mirror=FakeMirror(), - capture_tags=FakeTags(), - paperless=FakePaperless(), - bot_name="b", classify_max_chars=100, - capture_keep_body=False, capture_tag_prompt_size=50, - ) + async def test_a_blocked_page_files_a_link_card_instead_of_nothing(self): + """A page we cannot read still files. + + Dropping it was the old behaviour and it lost the two things + worth keeping -- the link, and whatever the sender wrote around + it. A shop that checks for bots is exactly when "gear list for + the camping trip" is the useful half of the message. + """ + mirror = FakeMirror() + pipe = _pipeline(mirror=mirror, url_extractor=FakeExtractor(None)) notifier = FakeNotifier() - out = await pipe.capture_url(url="http://x", sender_mxid="@homer:s", notifier=notifier) - assert out.status == "extract_failed" - # URL-shaped failure -> the reply layer renders the link error - # message (`Couldn't read that link...`). - assert out.failure_reason == "url" - # The 👀 acknowledgement still fired before the failed extract. + + out = await pipe.capture_url( + url="https://www.decathlon.de/", sender_mxid="@homer:s", + notifier=notifier, user_hint="Gear list for the camping trip", + ) + + assert out.status == "captured" + assert len(mirror.captures) == 1 + # The 👀 acknowledgement still fires before the work. assert notifier.acknowledged == 1 + @pytest.mark.asyncio + async def test_the_link_card_carries_the_gate_reason(self): + """The verdict rides out on the outcome so the reply layer can + name the obstacle. "Reddit wants you signed in" is actionable; + "couldn't read that link" is not.""" + pipe = _pipeline(url_extractor=FakeExtractor(None, verdict="login")) + + out = await pipe.capture_url( + url="https://old.reddit.com/r/x/", sender_mxid="@homer:s", + notifier=FakeNotifier(), + ) + + assert out.blocked_reason == "login" + + @pytest.mark.asyncio + async def test_a_readable_page_is_not_marked_blocked(self): + """`blocked_reason` is the flag the reply layer keys on, so a + clean capture must leave it unset or every entry grows a + spurious "the site blocked us" line.""" + out = await _pipeline().capture_url( + url="http://example.com", sender_mxid="@homer:s", notifier=FakeNotifier(), + ) + + assert out.status == "captured" + assert out.blocked_reason is None + + @pytest.mark.asyncio + async def test_the_sender_words_become_the_link_card_body(self): + """The classifier's input is the sender's own text plus the + link -- and deliberately not "this page was blocked", which + would come back as an entry titled after the failure.""" + mirror = FakeMirror() + pipe = _pipeline(mirror=mirror, url_extractor=FakeExtractor(None)) + + await pipe.capture_url( + url="https://www.decathlon.de/", sender_mxid="@homer:s", + notifier=FakeNotifier(), user_hint="Gear list for the camping trip", + ) + + filed = mirror.captures[0] + assert "blocked" not in str(filed).lower() + @pytest.mark.asyncio async def test_no_mirror(self): pipe = _pipeline(mirror=None) diff --git a/tools/web/capture-fixture.py b/tools/web/capture-fixture.py new file mode 100755 index 00000000..85886d5d --- /dev/null +++ b/tools/web/capture-fixture.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Capture a live page as a test fixture for the web quality gate. + +Adding a site profile should cost one fixture and one test. This is the +fixture half: it fetches a URL the way the archivist does, strips the +bulk that no detector reads, and writes the result into +`tests/fixtures/web/`. + +Why a tool rather than "save the page": fixtures have to stay real. The +gate's whole claim is that it was tested against what sites actually +serve, and a hand-written fixture agrees with the detector that reads +it by construction. So this keeps every tag, attribute, `` and +`ld+json` block byte-for-byte, and drops only inline script and style +*bodies* -- which are most of the megabyte and none of the meaning. + +Usage: + + uv run python tools/web/capture-fixture.py + +(`uv run` because the framework needs Python 3.11+ for tomllib, and +the system python on macOS is usually older.) + +Then add a row to `tests/fixtures/web/README.md` saying where it came +from and what the site answered, and a test asserting the verdict. +""" + +from __future__ import annotations + +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "lib")) + +from stack.web.fetch import BROWSER_HEADERS # noqa: E402 + +FIXTURES = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "web" + + +def shrink(html: str) -> str: + """Drop the bytes no detector reads, keep everything one might.""" + def _script(match: re.Match) -> str: + attrs = match.group(1) + # ld+json is data, not code — it is the whole point of tier 1. + if "ld+json" in attrs.lower(): + return match.group(0) + return f"" + + html = re.sub(r"]*)>.*?", _script, html, flags=re.S | re.I) + html = re.sub(r"]*)>.*?", r"", html, flags=re.S | re.I) + html = re.sub(r"data:[a-z/+-]+;base64,[A-Za-z0-9+/=]+", "data:stripped", html) + html = re.sub(r'(\ssrcset=")[^"]{200,}(")', r"\1stripped\2", html) + html = re.sub(r'(\sd=")[^"]{200,}(")', r"\1stripped\2", html) + return html + + +def main(argv: list[str]) -> int: + if len(argv) != 3: + print(__doc__.strip().split("Usage:")[1].strip(), file=sys.stderr) + return 2 + url, name = argv[1], argv[2] + + request = urllib.request.Request(url, headers=dict(BROWSER_HEADERS)) + try: + with urllib.request.urlopen(request, timeout=30) as response: + status, landed = response.status, response.url + body = response.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as err: + # A 403 challenge page *is* the fixture we usually want. + status, landed = err.code, err.url + body = err.read().decode("utf-8", errors="replace") + except OSError as err: + print(f"could not fetch {url}: {err}", file=sys.stderr) + return 1 + + shrunk = shrink(body) + FIXTURES.mkdir(parents=True, exist_ok=True) + (FIXTURES / f"{name}.html").write_text(shrunk, encoding="utf-8") + + title = re.search(r"]*>([^<]*)", shrunk, re.I | re.S) + print(f"wrote tests/fixtures/web/{name}.html ({len(shrunk)} bytes, from {len(body)})") + print(f" HTTP {status}") + print(f" landed on {landed}") + print(f" title {title.group(1).strip()!r}" if title else " no ") + if landed != url: + print(" note: the redirect target is the gate's signal — record it in the README") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv))