From 58882ef10fbd614423f167d7ff07baaf536f85de Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 16:50:45 -0400 Subject: [PATCH] feat(console): surface raw-document caching, sitemap conditional fetching and explain cadence; v0.14.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Catch-up with plugin v0.77.0 (PRs #165, #177, #181, #182). Four signals the plugin now emits or returns had no console surface, and each one is invisible by construction rather than merely missing. RAW-DOCUMENT CACHE (plugin #181). New Traffic panel reading `prerender_ops` `raw_cache`, and it leads with the REFUSALS: a route that is enabled and storing nothing produces the same miss rate, the same origin proxies and the same absence of errors as a route nobody enabled, so `stored` alone would be blind to the whole failure mode. `has-cookie` (the origin is personalizing a route assumed to be shared) and `oversize` (`maxBytes` below the route's real document size) are called out above the breakdown rather than left as bars. `raw` AS A SERVE VERDICT. The colours were already added; this makes them coherent everywhere else: - the non-hit taxonomy gains a `raw` family — without it the verdict fell through to "an outcome this console does not know about"; - the Cache-served tile now counts the VERDICT (`isCacheServed`) rather than the source `cache`. The two agreed until v0.76.0; `raw` is a cache serve with its own source, so the tile would have fallen as the feature started working while gross offload rose. The raw share is named in the subtitle, never folded in; - the staleness panel's past-due denominator moves the other way, to source `cache` only: `page_age` is emitted only on that branch, so counting raw serves there would have shrunk the past-due share and fired the "verdicts disagree" note against a fleet that is genuinely behind. SITEMAPS (plugin #165, #177). `lastRefreshed` is relabelled "ingested": a 304 deliberately writes nothing, so it is when this document's ENTRIES last changed, not when it was checked — and "refreshed" made conditional fetching working read as a sitemap hours stale. When it was checked comes from the run row and is shown beside it. The per-root walk now reports its 304s and its fast-path creates (a SUBSET of created, stated with its denominator), and a new panel sums the `sitemap_*` counters across roots and nodes over 24h — `sitemap_not_modified` is the only evidence anywhere that conditional fetching works at all, and a flat zero across a day of walks is flagged. EXPLAIN CADENCE (plugin #182). The explainer renders the new `cadence` block: effective interval, all four inputs, the demand floor, and `clampedBy` first. `floor` across a route means the ladder has no dynamic range there. The Target card's `renderInterval` is relabelled as the ceiling it is. THE GUARD'S sitemap_* BLIND SPOT IS CLOSED. `metrics.sitemapRun` is on DYNAMIC_SERIES_SLOT, and that exemption cost exactly what the probe one did: two series shipped unread with a green suite on both sides. The new test reads the EMIT SITES rather than the catalog, deliberately — the catalog enumerates six `sitemap_*` names and the plugin emits fifteen, so a catalog-based guard would have passed while blind to the two series it exists for. Not closed, and named in the guard rather than left silent: the `sitemap_departure_*` family (ten series) has no panel. Departures are a separate decision surface with dry-run-vs-armed semantics of their own and want their own panel, not seven more tiles on the walk card. Co-Authored-By: Claude Opus 5 (1M context) --- package-lock.json | 2 +- packages/console/README.md | 43 +++- packages/console/package.json | 2 +- packages/console/src/admin/charts.js | 26 +- packages/console/src/admin/views/inspect.js | 127 +++++++++- packages/console/src/admin/views/sitemaps.js | 202 ++++++++++++++- packages/console/src/admin/views/traffic.js | 246 ++++++++++++++++++- packages/console/test/adminAssets.test.js | 79 +++++- packages/console/test/inspectView.test.js | 177 +++++++++++++ packages/console/test/sitemapsView.test.js | 127 +++++++++- packages/console/test/trafficView.test.js | 91 +++++++ 11 files changed, 1082 insertions(+), 40 deletions(-) create mode 100644 packages/console/test/inspectView.test.js diff --git a/package-lock.json b/package-lock.json index 90a0617..863a590 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8556,7 +8556,7 @@ }, "packages/console": { "name": "@harperfast/prerender-console", - "version": "0.13.1", + "version": "0.14.0", "license": "Apache-2.0", "dependencies": { "undici": "^7.18.2" diff --git a/packages/console/README.md b/packages/console/README.md index 90d9e09..f937349 100644 --- a/packages/console/README.md +++ b/packages/console/README.md @@ -158,7 +158,8 @@ API contract; the short version: **Overview** (scale, serve health, backlog shap floor, schedule repair, and the discovered-target purge), **Traffic** (offload/hit-rate charts from one bounded analytics scan per node, freshness reported relative to each route's own render cadence, the non-hit verdicts broken out by what would fix them — coverage stated net of URLs the origin does -not have — the discovery gate, and a client-side bot filter), **Sitemaps**, **Page cache**, +not have — the discovery gate, the raw-document cache's refusals, and a client-side bot filter), +**Sitemaps** (per-root ingest and check state, plus 24h walk counters), **Page cache**, **Queue** (render/claim health and the backlog), **Nodes**, **Invalidations** (preview-first record/clear), **Change probe**, **URL explainer**, **Metrics** (the live catalog), **Config**. @@ -254,6 +255,43 @@ way a row survived a pass — deferred, spared, unreadable, failed — is subtra reports what the pass never reached; a missing term there turns "we spared 40% on purpose" into "~40% was never reached". +**The raw-document cache reports what it REFUSED**, not what it stored (plugin v0.76.0, console +v0.14.0). `render.raw` keeps the origin document a miss already fetched, for URLs outside the +render rotation — the facet and parameter combinations a crawler invents, which own no target and +therefore miss on every single request. The failure mode is silence: a route that is enabled and +filling nothing produces the same miss rate, the same origin proxies and the same absence of errors +as a route nobody enabled, so the panel leads with the reason each candidate was turned away. +Two of those reasons are findings rather than traffic and are called out above the breakdown — +`has-cookie` means the origin is personalizing a route that was enabled on the assumption it is +shared (the refusal is right, the assumption is not), and `oversize` means `render.raw.maxBytes` +sits below the route's real document size, so the feature is on and structurally cannot fill. +A raw serve is a **cache serve** and counts toward offload, but it is never a hit and never a +freshness number: nothing rendered it, it has no cadence to be measured against, and the plugin +emits no `page_age` for it. The Cache-served tile names the raw share instead of folding it in. + +**Ingested is not checked** (plugin v0.69.0, console v0.14.0). A sitemap walk now sends +`If-Modified-Since`, and a `304` deliberately writes nothing — the stored row and its validator are +still current. That makes `Sitemap.lastRefreshed` the time that document's _entries_ were last +ingested, which on a nightly-rebuilt corpus is hours old by design; when it was last _looked at_ +lives on the run row. Both are shown under their own names, because printing the first under the +second's label turns conditional fetching working into an operator chasing a sitemap that is not +stale. Beside them, a walk-activity panel sums the per-run counters across roots and nodes over 24h +(the same range key the Change probe view uses, so the two share one cached scan): `not modified` is +the only evidence anywhere that conditional fetching is working at all — a walk that re-parses every +document succeeds exactly like one that skipped — and a flat zero across a day of walks is flagged. +`rendered soon` is the share of new targets whose first render was pulled inside +`sitemap.newTargets.window` rather than waiting out a full interval of jitter; it is a _subset_ of +created, stated with its denominator, and the gap is the per-run cap sending a bulk ingest back to +the old behaviour. + +**A cadence is not its ceiling** (plugin v0.77.0, console v0.14.0). The URL explainer's Target card +shows `renderInterval`, which is the interval the demand ladder schedules _inside_ — with the ladder +armed it is not the cadence for most of a corpus, and nothing said so. The explainer now renders the +plugin's own resolution: the effective interval, all four inputs (route, stored, default, rung), the +demand floor, and `clampedBy` — the clamp that actually bound. `floor` on one URL is information; +`floor` across a route means the ladder has no dynamic range there and the promotion machinery is +running for nothing. `ceiling` means the rung is inert. + Two more changed shape when configuration became editable: - **Nodes is new**, and it exists because "is this node healthy" had four homes: liveness and the @@ -271,7 +309,8 @@ Two more changed shape when configuration became editable: Each domain view owns the options that govern the data it shows — `sitemap.*` under Sitemaps, `queue`/`render`/`scan` under Queue, `page`/`cacheKey` under Page cache, `analytics`/`crawlStats` -under Traffic, `invalidation` under Invalidations, `changeProbe` under Change probe — while Config +under Traffic, `invalidation` under Invalidations, `changeProbe` under Change probe (`render.raw.*` +rides the `render` group under Queue) — while Config remains exhaustive, so a setting can be found either by where it acts or by name. ## Development diff --git a/packages/console/package.json b/packages/console/package.json index 2c5dcce..ab7ecfd 100644 --- a/packages/console/package.json +++ b/packages/console/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-console", - "version": "0.13.1", + "version": "0.14.0", "type": "module", "description": "Standalone Harper component serving the prerender management console UI, proxying to a prerender deployment's /prerender_admin API", "license": "Apache-2.0", diff --git a/packages/console/src/admin/charts.js b/packages/console/src/admin/charts.js index 6a695c1..1b45564 100644 --- a/packages/console/src/admin/charts.js +++ b/packages/console/src/admin/charts.js @@ -66,14 +66,27 @@ export const CACHE_STATUS_COLORS = { * `hit`, `swr` and `verified` are the page itself; `peer-rescue` is the owner's copy of it. What * is NOT here: `miss`/`stale` (origin), `blob-*` (the local body failed AND no rescue landed, so * the request went to origin), `invalidated` (refused), `skip`/`bypass` (never consulted). + * + * `raw` IS IN THIS SET (plugin v0.76.0): a stored origin document answered the request, so the + * origin was spared exactly as it was by a snapshot. Leaving it out would not read as missing — + * it would read as a smaller cache-served share, which is the same failure `verified` caused for + * one release. + * + * BUT IT IS NOT AN AGE POPULATION. `page_age` / `route_page_age` are emitted only when the serve + * SOURCE is `cache` (`recordServeOutcome`), and a raw serve's source is `raw` — nothing rendered + * it, so it has no cadence to be measured against. Anything dividing by "cache serves" to talk + * about freshness must therefore count the source, not this set; see the staleness panel. */ -// `raw` IS IN THIS SET, and leaving it out would not read as missing — it would read as a smaller -// hit rate, exactly as `verified` did for one plugin release. A raw serve answered from storage and -// cost the origin nothing, which is what this set means. export const CACHE_SERVED = new Set(['hit', 'swr', 'verified', 'peer-rescue', 'raw']); export const isCacheServed = (status) => CACHE_SERVED.has(status); -/** Where the bytes came from (bot_serve.path). `origin` is the one offload counts against. */ +/** + * Where the bytes came from (bot_serve.path). `origin` is the one offload counts against. + * + * `raw` is its own source upstream rather than a flavour of `cache`, and it stays that way here: + * "answered from storage" and "a render covers this URL" are different questions, and a raw + * document answers only the first. + */ export const SOURCE_COLORS = { cache: OK, rendered: INFO, raw: '#7fd4e8', origin: WARN }; /** What became of a posted render result (render outcome.method). */ @@ -594,7 +607,10 @@ const SITEMAP_FETCHES = 'sitemap_sitemaps'; * probes `probe_probed` — one origin call per attempt, failures included (a refused probe was * still a request). Sweep and canary both emit it. A probe hits a small endpoint, not a * page render, so it is cheaper than the other three; it is still a request. - * sitemaps `sitemap_sitemaps` — one fetch per sitemap a refresh run processed. + * sitemaps `sitemap_sitemaps` — one fetch per sitemap a refresh run processed. Conditional + * fetching (plugin v0.69.0) does NOT remove these: the counter is attempts, and a + * document the origin answered `304` to still cost a request. What it removes is the + * re-parse and the prune scan on THIS side, which no origin-load term ever counted. * * `renders` is read with sumCount (one emit = one result); the two pass counters with sumValues * (one emit per pass carrying the pass's count — sumCount there would count passes). diff --git a/packages/console/src/admin/views/inspect.js b/packages/console/src/admin/views/inspect.js index 4f82a6d..334e055 100644 --- a/packages/console/src/admin/views/inspect.js +++ b/packages/console/src/admin/views/inspect.js @@ -26,6 +26,11 @@ * - The same rule covers "below the claim floor" and "leased", which are answers about the * OWNER's node-local shared buffer. The owner computes both and this view consumes them * verbatim; it never compares a row against the querying node's own floor. + * - A CADENCE IS NOT ITS CEILING. The Target's `renderInterval` is the interval the demand + * ladder schedules INSIDE, not the one this URL runs on, and the two differ for most of a + * corpus once the ladder is armed. `cadenceCard` renders the plugin's own resolution of it + * (`explain`'s `cadence` block, plugin v0.77.0) — every input, and the clamp that decided — + * rather than leaving a reader to infer a cadence from a ceiling. * * THE BROWSE HALF'S QUERY SHAPE, AND THE HONESTY RULE THAT FOLLOWS FROM IT. `PrerenderedPage` * has only its primary key. A prefix search is a primary-key range and cheap; anything else — @@ -234,6 +239,7 @@ function explanation(ctx, data) { verdictPills(data, page), ], }), + cadenceCard(data), card('Stored rows', { body: [ el('h3', { cls: 'subhead', text: 'PrerenderedPage' }), @@ -301,7 +307,12 @@ function explanation(ctx, data) { : '—', ], ['Scheduler node', mono(target.schedulerNode ?? '—')], - ['Render interval', target.renderInterval ? duration(Number(target.renderInterval)) : 'default'], + // The stored CEILING, not the cadence: the demand ladder schedules inside it. The + // Render cadence card above resolves the two, which is why this row names what it is. + [ + 'Render interval (ceiling)', + target.renderInterval ? duration(Number(target.renderInterval)) : 'default', + ], ['State', target.state === 'suppressed' ? pill('suppressed', 'warn') : pill('active', 'ok')], data.rows.suppression && ['Suppressed', suppressionSummary(data.rows.suppression)], ]) @@ -313,6 +324,120 @@ function explanation(ctx, data) { ]; } +/** + * How often this URL actually re-renders, and which input decided it. + * + * WHY THIS IS NOT ONE NUMBER. The cadence a page runs on is the demand ladder's stored rung, raised + * to the route's `demandFloor` and then capped by the base interval — route > stored > default. So + * there are four inputs and two clamps, and every one of them is a value an operator can set + * somewhere else and then fail to see the effect of. The `Render interval` row on the Target card + * below is the CEILING, not the cadence; reading it as the cadence is the specific mistake the + * plugin grew `explainCadence` (v0.77.0) to make impossible. + * + * READ `clampedBy` FIRST. It names the clamp that actually bound: + * + * floor the ladder wanted faster and `demandFloor` refused. One URL is information; `floor` + * across a route means the ladder has NO dynamic range there — it is pinned at the + * floor whatever the traffic does, and the promotion machinery is running for nothing. + * ceiling the ladder wanted slower than the route grants, so the route's own interval won. The + * page renders at its configured cadence; the rung is inert. + * null the rung applied as computed, or there is no rung yet. + * + * The ceiling is tested FIRST upstream, deliberately: a floor above the route's interval reports + * `ceiling`, because that is the clamp that produced the answer — and a `demandFloor` larger than + * the `renderInterval` it modifies is precisely the misconfiguration this view is opened to find. + */ +const CLAMP = { + floor: [ + 'warn', + 'clamped by demandFloor', + 'The ladder’s rung was raised to the route’s demand floor. Across a route this means the ladder has ' + + 'no range to work in: it cannot go faster than the floor, so promotion is running for nothing.', + ], + ceiling: [ + '', + 'clamped by the route ceiling', + 'The rung was slower than the route’s own interval, so the route won. The ladder never schedules ' + + 'slower than the cadence a route already grants — the rung is inert here.', + ], +}; + +/** + * An interval as text, or null when there isn't one. + * + * `duration()` takes `Math.abs(ms)`, so it formats `null` as "0s" and `undefined` as "NaNs" — both + * of which read as a real cadence. Every figure on this card comes from a payload that may be + * older than this console, so each one goes through here rather than trusting the field to exist. + * `Number()` because `storedInterval` rides a schema Long and arrives as a string on some payloads. + */ +const intervalText = (value) => { + const ms = Number(value); + return Number.isFinite(ms) && ms > 0 ? duration(ms) : null; +}; + +function cadenceCard(data) { + const cadence = data.cadence; + // Null for a URL with no target, and that is a real answer rather than a missing one: cadence is + // a property of being in the rotation. An older plugin sends no `cadence` at all, which is the + // same absence — say which, rather than drawing a card of dashes. + if (!cadence) return null; + + const clamp = CLAMP[cadence.clampedBy]; + const effective = intervalText(cadence.effectiveInterval); + const rung = intervalText(cadence.demandInterval); + const floor = intervalText(cadence.demandFloor); + const row = (text, note) => el('span', null, [mono(text), muted(` ${note}`)]); + + return card('Render cadence', { + head: [ + clamp ? pill(clamp[1], clamp[0]) : rung ? pill('ladder rung applied', 'ok') : pill('base interval', ''), + spacer(), + effective && el('span', { cls: 'mono', style: { fontSize: '13px', color: 'var(--fg-0)' }, text: effective }), + ], + body: [ + clamp && el('div', { cls: `note ${clamp[0]}`.trim(), style: { marginBottom: '10px' } }, [clamp[2]]), + kv([ + [ + 'Effective interval', + effective ? row(effective, 'what the scheduler actually files') : muted('— not reported'), + ], + [ + 'Base (ceiling)', + intervalText(cadence.baseInterval) + ? row(intervalText(cadence.baseInterval), `from ${cadence.baseFrom ?? 'unknown'}`) + : muted('— not reported'), + ], + [ + 'Route interval', + intervalText(cadence.routeInterval) + ? mono(intervalText(cadence.routeInterval)) + : muted('— the route sets none'), + ], + [ + 'Stored on the target', + intervalText(cadence.storedInterval) + ? mono(intervalText(cadence.storedInterval)) + : muted('— sitemap changefreq or an explicit write'), + ], + [ + 'Default interval', + intervalText(cadence.defaultInterval) ? mono(intervalText(cadence.defaultInterval)) : muted('—'), + ], + [ + 'Demand rung', + rung ? row(rung, 'the ladder’s stored decision') : muted('— the ladder has not evaluated this target'), + ], + ['Demand floor', floor ? row(floor, 'the fastest rung this route may reach') : muted('— the route sets none')], + ]), + el('p', { cls: 'muted', style: { margin: '12px 0 0', fontSize: '12px' } }, [ + 'Resolved exactly as the scheduler resolves it: the rung raised to the floor, then capped by the ', + 'base — so the Target card’s render interval below is the ceiling this is clamped into, never the ', + 'cadence on its own.', + ]), + ], + }); +} + function verdictPills(data, page) { return el('div', { cls: 'toolbar', style: { marginTop: '14px' } }, [ // A timed-out read must never render as a confident verdict: an unread page row is diff --git a/packages/console/src/admin/views/sitemaps.js b/packages/console/src/admin/views/sitemaps.js index 28bca45..785fa33 100644 --- a/packages/console/src/admin/views/sitemaps.js +++ b/packages/console/src/admin/views/sitemaps.js @@ -7,17 +7,43 @@ * The detail fetches ONE page of entries at a time, sliced server-side; per-entry state comes * from bounded point reads on just that page. Text filtering below is within the fetched page * and is labelled as such. + * + * INGESTED IS NOT CHECKED, and conflating the two misreports a healthy corpus as a stale one. + * Since plugin v0.69.0 a walk sends `If-Modified-Since`, and a `304` writes NOTHING — the stored + * row, its validator and its `lastRefreshed` are all still current, so they are deliberately left + * untouched. That makes `Sitemap.lastRefreshed` the time this document's ENTRIES were last + * INGESTED, which on a corpus that rebuilds nightly is hours old by design and says nothing about + * when it was last looked at. When it was last looked at lives on the run row (`SitemapRefresh`), + * per root. Both are shown, named for what they are: "ingested" on the document, "checked" from + * the run. A console that prints one under the other's label turns the feature working into an + * operator chasing a sitemap that is not stale. + * + * THE ANALYTICS WINDOW IS 24h AND SHARED. A refresh pass is daily, so an hour-wide window would + * almost never contain one; 24h is also the key the Change probe view uses, so whichever loaded + * first inside `management.analytics.cacheTtl` serves the other from the worker's cache. The walk + * counters are the only cluster-wide view of pass outcomes — the run row above is one root on one + * node — and `sitemap_not_modified` in particular is the ONLY evidence that conditional fetching + * is working at all. */ -import { ago, card, el, ICONS, kv, link, meter, muted, num, pct, pill, spacer, table } from '../ui.js'; +import { ago, card, el, ICONS, kv, link, meter, muted, num, pct, pill, spacer, stat, table } from '../ui.js'; +import { emptyNote, fmtCount, pick, scanFooter, scopeLabel, sumValues, windowEmpty } from '../charts.js'; import { appliedNote, editTray, loadConfig, settingsCard } from './_configEdit.js'; export const meta = { id: 'sitemaps', label: 'Sitemaps', crumb: 'sitemaps', icon: ICONS.sitemaps }; const PAGE_SIZE = 50; +/** The window the walk counters are read over — a day, because a pass is daily. See the header. */ +const WALK_RANGE_MS = 24 * 3_600_000; + export async function load(ctx) { - const [res] = await Promise.all([ctx.get('sitemaps'), loadConfig(ctx)]); + const [res, analyticsRes] = await Promise.all([ + ctx.get('sitemaps'), + ctx.get('analytics', { range: WALK_RANGE_MS }), + loadConfig(ctx), + ]); + ctx.data.analytics = analyticsRes.ok ? analyticsRes.body : null; if (!res.ok) { ctx.data.list = null; ctx.data.error = res.body?.error ?? `Could not load sitemaps (${res.status})`; @@ -92,11 +118,24 @@ export function render(ctx) { detail(ctx), ]), ]), + walkActivity(ctx), settings(ctx), editTray(ctx), ]; } +/** + * When this root was last WALKED, from its run row — as distinct from when its document was last + * ingested. Null for a child sitemap, which has no run row of its own: `SitemapRefresh` holds one + * row per root plus the `all` marker, so a child's only timestamp is its ingest. + */ +const checkedAt = (refresh) => { + const at = refresh?.finishedAt ?? refresh?.lastRefreshed ?? null; + if (!at) return null; + const ms = new Date(at).getTime(); + return Number.isFinite(ms) ? ms : null; +}; + function rootList(ctx, roots) { return el( 'div', @@ -133,8 +172,16 @@ function rootList(ctx, roots) { failed ? pill('✗', 'bad') : running ? pill('…', 'info') : pill('✓', 'ok'), ]), el('div', { cls: 'mono muted', style: { fontSize: '11px', marginTop: '4px' } }, [ - `${num(sitemap.entryCount)} entries · ${sitemap.lastRefreshed ? ago(new Date(sitemap.lastRefreshed).getTime()) : 'never refreshed'}`, + `${num(sitemap.entryCount)} entries · ingested ${ + sitemap.lastRefreshed ? ago(new Date(sitemap.lastRefreshed).getTime()) : 'never' + }`, ]), + // The run row, not the document row — see the header. A 304 leaves the document's own + // timestamp alone, so this is the only line that says the sitemap was looked at. + checkedAt(sitemap.refresh) && + el('div', { cls: 'mono muted', style: { fontSize: '11px' } }, [ + `checked ${ago(checkedAt(sitemap.refresh))}`, + ]), ] ); }) @@ -190,10 +237,31 @@ function detail(ctx) { refresh && el('div', { style: { marginTop: '14px' } }, [ kv([ - ['Last walk', refresh.finishedAt ? ago(new Date(refresh.finishedAt).getTime()) : '—'], + ['Last walk', checkedAt(refresh) ? ago(checkedAt(refresh)) : '—'], refresh.created !== undefined && [ 'Created / updated / removed', - `${num(refresh.created)} / ${num(refresh.updated)} / ${num(refresh.removed)}`, + el('span', null, [ + `${num(refresh.created)} / ${num(refresh.updated)} / ${num(refresh.removed)}`, + // A SUBSET of `created`, never a fourth number: these are the new targets whose FIRST + // render was pulled into `sitemap.newTargets.window` instead of waiting out a full + // interval of jitter. `created - createdSoon` is the bulk-population overflow that + // fell back to the old behaviour, which is why the denominator is stated. + refresh.createdSoon !== undefined && refresh.created + ? muted(` ${num(refresh.createdSoon)} of the creates rendered soon`) + : null, + ]), + ], + // THE ONLY PROOF CONDITIONAL FETCHING IS WORKING, per root. A steady zero where the + // walk sends If-Modified-Since means the origin is not honouring it and every pass is + // re-parsing and re-scanning documents that did not change. + refresh.notModified !== undefined && [ + 'Not modified (304)', + el('span', null, [ + `${num(refresh.notModified)} of ${num(refresh.sitemapsProcessed)} documents`, + refresh.sitemapsProcessed + ? muted(` ${pct(refresh.notModified, refresh.sitemapsProcessed)} skipped the re-parse`) + : null, + ]), ], refresh.duplicates ? ['Duplicates (overlapping sitemaps)', num(refresh.duplicates)] : null, ]), @@ -205,8 +273,10 @@ function detail(ctx) { } function statsRow(detail) { - const { sitemap, targetCount } = detail; + const { sitemap, targetCount, refresh } = detail; const entryCount = sitemap.entryCount ?? 0; + const ingested = sitemap.lastRefreshed ? ago(new Date(sitemap.lastRefreshed).getTime()) : 'never'; + const checked = checkedAt(refresh); // AN INDEX HAS NO TARGETS OF ITS OWN, structurally: a walk attributes each Target to the // sitemap that actually listed the URL, which is always a child. So "Targets 0 / Coverage 0%" @@ -216,9 +286,12 @@ function statsRow(detail) { return el('div', { style: STATS_GRID }, [ statCell('Child sitemaps', num(entryCount), muted('listed by this index')), statCell( - 'Last walked', - sitemap.lastRefreshed ? ago(new Date(sitemap.lastRefreshed).getTime()) : 'never', - muted('this document, not its children') + 'Child list ingested', + ingested, + // NOT "last walked". A 304 on an index says its CHILD LIST is unchanged, and the walk + // still descends into every child — so this document can be hours old while the corpus + // behind it was rebuilt minutes ago. + muted(checked ? `this document, not its children · checked ${ago(checked)}` : 'this document, not its children') ), statCell('Entries', '—', muted('an index lists sitemaps, not URLs — open one below')), ]); @@ -238,6 +311,15 @@ function statsRow(detail) { targetCount === null || !entryCount ? '—' : pct(Math.min(targetCount.count, entryCount), entryCount), muted('entries with a render target') ), + // A CHILD HAS NO RUN ROW, so this is usually the only timestamp it carries — and it is an + // INGEST, not a check. Under conditional fetching a child that has not changed is fetched on + // every pass and written on none of them, so an hours-old figure here is the normal steady + // state rather than a walk that stopped reaching it. + statCell( + 'Entries ingested', + ingested, + muted(checked ? `checked ${ago(checked)}` : 'when this document last changed — not when it was checked') + ), ]); } @@ -382,6 +464,108 @@ function shortPath(url) { } } +/** + * Every walk that finished in the last day, across every root and every node. + * + * WHY IT IS NOT THE RUN ROW ABOVE. `SitemapRefresh` holds the LAST run per root, on whichever node + * claimed it. These counters are one emit per finished run, so they sum passes — which is the only + * way to see a root that ran three times, or a node whose walks are failing while another node's + * succeed. They are also the only home for the two numbers a rollout is judged on: + * + * not modified documents the origin answered `304` to, so their entries were never re-parsed + * and their prune scan never ran. On a healthy corpus this is most of every pass + * between rebuilds. A steady ZERO where conditional fetching is enabled means the + * origin is not honouring `If-Modified-Since` and every pass is doing full work — + * and nothing else anywhere reports that, because the walk still succeeds. + * rendered soon new targets whose first render was pulled into `sitemap.newTargets.window` + * instead of waiting out a full interval of jitter. A SUBSET of created, so the + * gap between the two is the per-run cap sending the overflow back to the old + * behaviour — which is what a bulk first ingest is supposed to look like. + * + * VALUE SEMANTICS: each row is one emit per RUN carrying that run's count, so the sum is + * Σ(mean × count) — `sumValues`, never `sumCount`, which would count runs. + */ +function walkActivity(ctx) { + const data = ctx.data.analytics; + if (!data || data.available === false) return null; + + const combos = pick(data, 'prerender_ops', (s) => typeof s.path === 'string' && s.path.startsWith('sitemap_')); + const totalOf = (series) => sumValues(combos.filter((s) => s.path === `sitemap_${series}`)); + const documents = totalOf('sitemaps'); + const notModified = totalOf('not_modified'); + const created = totalOf('created'); + const createdSoon = totalOf('created_soon'); + const updated = totalOf('updated'); + const skipped = totalOf('skipped'); + const removed = totalOf('removed'); + const failed = totalOf('failed'); + + const title = `Walk activity — ${scopeLabel(data)}, last 24h`; + if (windowEmpty(data) || !combos.length) { + return card(title, { + body: [ + emptyNote('sitemap walk', data), + el('p', { cls: 'muted chart-note' }, [ + 'These counters are emitted once per FINISHED walk. A refresh pass is daily, so an empty panel ', + 'most often means no walk completed inside the window — press Refresh all above, or check that ', + el('code', { text: 'sitemap.node' }), + ' names a node that is still in the cluster.', + ]), + ], + foot: [scanFooter(data)], + }); + } + + // Zero 304s across a day of walks is the conditional-fetch rollout not working. It is not an + // error anywhere — the walks succeed, the corpus is correct — it just costs a full re-parse and + // a prune scan per document, forever, which is the entire saving the feature was for. + const conditionalDead = documents > 0 && notModified === 0; + + return card(title, { + head: [failed > 0 ? pill(`${fmtCount(failed)} failed`, 'bad') : null, spacer()], + body: [ + conditionalDead && + el('div', { cls: 'note warn' }, [ + el('strong', { text: 'No document was answered 304 in this window. ' }), + 'Every walk re-parsed every sitemap and re-ran its prune scan. Conditional fetching sends ', + el('code', { text: 'If-Modified-Since' }), + ' from the stored validator, so a flat zero means either the origin ignores it or nothing has ', + 'a stored validator yet — the first pass after an upgrade is legitimately all-zero here, a ', + 'week of them is not.', + ]), + el('div', { cls: 'stats' }, [ + stat('Documents fetched', fmtCount(documents), 'attempts across every finished walk, failures included'), + stat( + 'Not modified', + documents ? pct(notModified, documents) : '—', + `${fmtCount(notModified)} skipped the re-parse and the prune scan`, + { warn: conditionalDead } + ), + stat( + 'Targets created', + fmtCount(created), + created + ? `${pct(createdSoon, created)} rendered soon rather than waiting out the jitter` + : 'nothing new was declared' + ), + stat('Re-attributed', fmtCount(updated), 'moved between sitemaps — the page did not change'), + stat('Unchanged', fmtCount(skipped), 'already correct, no write'), + stat('Unlinked', fmtCount(removed), 'left the sitemap that declared them'), + stat('Failed', fmtCount(failed), 'child sitemaps a walk could not read', { warn: failed > 0 }), + ]), + el('p', { cls: 'muted chart-note' }, [ + 'One emit per finished walk, summed across roots and nodes — so these are passes, not the state ', + 'of any one sitemap; the panel above is that. “Rendered soon” is a SUBSET of created, capped per ', + 'run by ', + el('code', { text: 'sitemap.newTargets.maxPerRun' }), + ', and the overflow falls back to full-interval jitter, which is what a bulk first ingest is ', + 'meant to look like. A 304 is still an origin request; what it saves is this side’s work.', + ]), + ], + foot: [scanFooter(data)], + }); +} + /** * Ingestion settings, below the state they produce. * diff --git a/packages/console/src/admin/views/traffic.js b/packages/console/src/admin/views/traffic.js index 2c42fbb..ea82485 100644 --- a/packages/console/src/admin/views/traffic.js +++ b/packages/console/src/admin/views/traffic.js @@ -40,14 +40,23 @@ * and one origin_fetch row — except under a bot filter, where origin_fetch carries no bot name; * there the tile says it is not netted instead of scaling one population by the other's share. * - * NOT EVERY NON-HIT IS A MISS. "miss" is one of nine freshness verdicts and the only one that + * NOT EVERY NON-HIT IS A MISS. "miss" is one of ten freshness verdicts and the only one that * means what the word implies — nothing cached under the key. The others are a page served past * its cadence (`swr`), one past the SWR window entirely (`stale`), a body that could not be read * although the key is cached and scheduled (`blob-missing` / `blob-timeout`, rescued or not), a - * serve a bulk invalidation cost us (`invalidated`), and requests where the cache was never - * consulted at all (`skip`, `bypass`). They have four different fixes — corpus coverage, render - * cadence, blob integrity, and nothing-to-fix — so they get their own panel rather than one bar - * labelled "miss", with what each one cost at the origin beside it. + * serve a bulk invalidation cost us (`invalidated`), a miss answered from a stored ORIGIN document + * rather than a snapshot (`raw`), and requests where the cache was never consulted at all (`skip`, + * `bypass`). They have five different fixes — corpus coverage, render cadence, blob integrity, + * nothing-to-fix, and a setting — so they get their own panel rather than one bar labelled "miss", + * with what each one cost at the origin beside it. + * + * `raw` IS A CACHE SERVE AND NEVER A HIT. `render.raw` keeps the document a miss already fetched, + * for URLs that own no target and never will, so the next crawler is answered from storage: it + * spares the origin exactly as a snapshot does, and it counts in Cache-served and in offload. But + * no browser ran on it, it has no cadence to be measured against, and it emits no `page_age` — so + * it is deliberately absent from every freshness number on this view. Its own panel reports what + * the store REFUSED, because an enabled route filling nothing is otherwise indistinguishable from + * one nobody enabled. * * THE BOT FILTER IS CLIENT-SIDE, ALWAYS. Selecting bots re-renders from the payload already in * hand; it never refetches, because the load discipline below is the whole reason this view can @@ -184,7 +193,10 @@ export function render(ctx) { el('div', { cls: 'cols' }, [latency(data, filter), statusCodes(data, filter)]), crawlers(data, scope), routes(ctx, data, cadences, filter), + // The two panels about URLs the rotation does not cover, together: the gate is what keeps them + // out of it, and the raw cache is what answers them anyway. discoveryGate(ctx, data, filter), + rawCache(ctx, data, filter), breadth(ctx, filter), el('div', { cls: 'scan-foot' }, [scanFooter(data)]), knobs, @@ -396,7 +408,14 @@ function kpis(data, scope) { const { serves, requests, filter } = scope; const total = sumCount(serves); const originServes = sumCount(serves.filter((s) => s.path === 'origin')); - const cacheServes = sumCount(serves.filter((s) => s.path === 'cache')); + // BY VERDICT, NOT BY SOURCE. Until plugin v0.76.0 the two agreed — every cache-served verdict + // carried source `cache`, `peer-rescue` included — so this counted `path === 'cache'` and got the + // right answer. `raw` broke that: it is a cache serve with its OWN source, so the source test + // would have dropped it and this tile would have fallen as the feature started working, while + // gross offload rose. `isCacheServed` is the enumeration charts.js says every such sum must + // share, and this is now one of them. + const rawServes = sumCount(serves.filter((s) => s.path === 'raw')); + const cacheServes = sumCount(serves.filter((s) => isCacheServed(s.method))); const freshHits = sumCount(serves.filter((s) => s.method === 'hit')); const coverage = coverageSplit({ serves, costs: originCostByReason(data), filter }); const arrived = sumCount(requests); @@ -465,7 +484,16 @@ function kpis(data, scope) { // more requests than the crawlers would have on their own — the finding, not a display bug. { warn: Number.isFinite(load.net) && load.net < 0.5 } ), - stat('Cache-served', pct(cacheServes, total), 'stored snapshot answered'), + stat( + 'Cache-served', + pct(cacheServes, total), + // Named apart the moment there are any, because the two are different claims about the + // deployment: a snapshot means a render covers that URL, a raw document means one never + // will and the origin was spared anyway. + rawServes > 0 + ? `answered from storage · ${pct(rawServes, total)} raw documents, not snapshots` + : 'stored snapshot answered' + ), stat('Fresh hits', pct(freshHits, total), 'inside the configured cadence'), stat( 'Coverage miss', @@ -512,9 +540,11 @@ function freshness(data, { serves, filter }) { ? stackedBars(data, keys, stacks, (k) => colorFor(CACHE_STATUS_COLORS, k)) : emptyNote('bot_serve', data), el('p', { cls: 'muted chart-note' }, [ - 'hit + swr + verified + peer-rescue is cache-served. A rising miss share is a coverage problem; a ', - 'rising swr share is the fleet not keeping the configured cadence; blob-* should sit at zero. What ', - 'each verdict costs, and which of them are the same problem, is the panel below.', + 'hit + swr + verified + peer-rescue + raw is cache-served. A rising miss share is a coverage ', + 'problem; a rising swr share is the fleet not keeping the configured cadence; blob-* should sit at ', + 'zero. `raw` is a miss answered from a stored ORIGIN document rather than a snapshot — it spares ', + 'the origin, but nothing rendered it, so it never counts as a hit. What each verdict costs, and ', + 'which of them are the same problem, is the panel below.', ]), filter && muted(`Filtered to ${[...filter].join(', ')}.`), ], @@ -573,10 +603,16 @@ function staleness(ctx, data, scope) { // differ whenever a target's cadence comes from its stored value (a sitemap `changefreq`) // instead — a case no metric exposes. When the two disagree, the verdicts win and the divisor // is what is wrong, so say that rather than let a config gap read as a fleet failure. - const cacheServed = sumCount(serves.filter((x) => isCacheServed(x.method))); + // + // THE DENOMINATOR IS THE SERVES THAT PRODUCED THE DISTRIBUTION, which is the serves whose SOURCE + // was `cache` — `page_age`/`route_page_age` are emitted only on that branch. It is deliberately + // not `isCacheServed`, which since plugin v0.76.0 also contains `raw`: a raw document contributes + // no age sample, so counting it here would shrink the past-due share on exactly the deployments + // that serve a lot of raw and fire this note against a fleet that is genuinely behind. + const agedServes = sumCount(serves.filter((x) => x.path === 'cache')); const pastDue = sumCount(serves.filter((x) => x.method === 'swr' || x.method === 'stale')); const contradicted = - normalizable && Number.isFinite(ratioP95) && ratioP95 > 1 && cacheServed > 0 && pastDue / cacheServed < 0.01; + normalizable && Number.isFinite(ratioP95) && ratioP95 > 1 && agedServes > 0 && pastDue / agedServes < 0.01; return card(mode === 'ratio' ? 'Staleness at serve (÷ cadence, ≈)' : 'Page age at serve (≈)', { head: [ @@ -630,7 +666,7 @@ function staleness(ctx, data, scope) { contradicted && el('div', { cls: 'note' }, [ el('strong', { text: 'The freshness verdicts disagree with this ratio, and they are the authority. ' }), - `Only ${pct(pastDue, cacheServed)} of cache serves were past due (swr + stale), which is decided per `, + `Only ${pct(pastDue, agedServes)} of snapshot serves were past due (swr + stale), which is decided per `, 'request against that page’s own expiry — so these pages are fresh and the divisor is short. That ', 'happens when a target’s cadence comes from its stored interval (a sitemap ', el('code', { text: 'changefreq' }), @@ -687,6 +723,15 @@ const FAMILIES = [ // each other they size what per-page verification is buying; apart, neither means much. hint: 'a bulk invalidation touched the serve — refused, or rescued on evidence', }, + { + key: 'raw', + label: 'Raw document', + // Not a fresh hit and not a problem: the request was a miss, and a stored ORIGIN document + // answered it instead of a live proxy. Its own family because every other one names something + // to fix — this one names a miss that cost the origin nothing, on a URL nothing will ever + // render. Folding it into cadence or coverage would report the feature working as a fault. + hint: 'a miss answered from a stored origin document — never rendered, so it has no cadence', + }, { key: 'not-cacheable', label: 'Not cacheable', @@ -717,6 +762,10 @@ const NOT_HIT = { 'invalidated', 'an invalidation would have refused it; the change probe proved its claims current — still a cache serve', ], + // A cache serve too, and counted as one — but never as a hit: a hit means a page this system + // rendered was inside its cadence, and nothing rendered this. `render.raw` stores the document a + // miss already fetched so the next crawler asking for the same URL is answered from storage. + 'raw': ['raw', 'a stored origin document answered it — no browser ran on it, and it owns no render target'], 'skip': ['not-cacheable', 'the cache was deliberately not consulted (renderNow / Cache-Control)'], 'bypass': ['not-cacheable', 'not a cacheable request at all (non-GET/HEAD)'], }; @@ -1535,6 +1584,177 @@ function discoveryGate(ctx, data, filter) { }); } +/** + * The raw-document cache: what it stored, and — the point of the panel — what it REFUSED. + * + * READ THE REFUSALS. An enabled route that stores nothing looks exactly like a route nobody + * enabled: same miss rate, same origin proxies, no error anywhere. The only thing that + * distinguishes them is the reason each candidate was turned away, which is why the plugin emits + * one `raw_cache` row per store ATTEMPT rather than counting successes. A panel that charted + * `stored` alone would be blind to the whole failure mode it exists for. + * + * TWO OUTCOMES ARE FINDINGS RATHER THAN FACTS OF LIFE, and they are flagged: + * + * has-cookie the origin tried to set a cookie, i.e. it is personalizing a route that was + * assumed to be shared. Storing that document would replay one visitor's page to + * every crawler, so the refusal is correct — but the ASSUMPTION is wrong, and + * nothing else in this console would ever say so. + * oversize `render.raw.maxBytes` is below the route's real document size, so the feature is + * enabled and structurally cannot fill. It is a settings fix, not a fault. + * + * The rest are the shape of the traffic: `not-200` and `content-type` are documents that were never + * eligible, `no-store` is the origin declining a shared cache (correctly honoured), `capture-busy` + * is the per-worker concurrency cap shedding a capture rather than the heap, and `empty` is a 200 + * with no body — the one that used to be stored and replayed as a zero-byte document. + * + * WHAT THE STORE RATE IS NOT: a hit rate. Fills and serves are different populations over the same + * window — a document stored now is read by the NEXT crawler, possibly after this window — so the + * serves are shown beside the fills and never divided by them. + */ +// Outcome names as constants at the lookup sites, for the same reason the discovery gate's are: +// the route-contract scanner in adminAssets.test.js reads a quoted name inside a Map lookup as a +// fetch of an admin route by that name. +const RAW_STORED = 'stored'; +const RAW_HAS_COOKIE = 'has-cookie'; +const RAW_OVERSIZE = 'oversize'; + +/** Every refusal the plugin can report, and what an operator should do about it. */ +const RAW_REFUSALS = { + 'not-200': ['the origin did not answer 200 — only a 200 is ever eligible', ''], + 'staging': ['fetched through the staging origin, so the bytes are not production', ''], + 'has-cookie': ['the origin set a cookie: this route is personalized, not shared', 'bad'], + 'content-type': ['not in render.raw.contentTypes', ''], + 'no-store': ['the origin sent private / no-store — it is declining a shared cache', ''], + 'no-body': ['the response body was not a stream, so there was nothing to capture', ''], + 'oversize': ['larger than render.raw.maxBytes — served, and the capture abandoned', 'warn'], + 'capture-failed': ['the origin body errored or truncated mid-capture', 'warn'], + 'write-failed': ['the store itself threw — the document was served, nothing was kept', 'bad'], + 'empty': ['a 200 with a zero-byte body — refused rather than replayed as an empty document', 'warn'], + 'capture-busy': ['render.raw.maxConcurrentCaptures was full; the next request stores it', ''], +}; + +function rawCache(ctx, data, filter) { + const attempts = pick(data, 'prerender_ops', (s) => s.path === 'raw_cache'); + const options = optionIndex(configState(ctx).payload); + const enabled = options.get('render.raw.enabled')?.effective === true; + const rawRoutes = (options.get('ingress.routes')?.effective ?? []).filter( + (entry) => entry && typeof entry === 'object' && entry.rawCache === true + ); + // All bots: `raw_cache` carries no bot dimension (its slots are outcome and nothing else), and + // the serve counts are narrowed by the filter like every other bot_serve read on this view. + const rawServes = sumCount(pick(data, 'bot_serve', (s) => s.path === 'raw' && keepBot(filter, s.type))); + + const byOutcome = new Map(); + for (const s of attempts) byOutcome.set(s.method ?? 'unknown', (byOutcome.get(s.method ?? 'unknown') ?? 0) + s.count); + const stored = byOutcome.get(RAW_STORED) ?? 0; + const total = [...byOutcome.values()].reduce((acc, n) => acc + n, 0); + const refused = total - stored; + + // Off and never switched on: describe the capability instead of drawing an empty chart, which + // would read as a subsystem that is on and failing. Same exit as the discovery gate's. + if (!enabled && !total && !rawServes) { + return card('Raw-document cache', { + head: [spacer(), pill('off', '')], + body: [ + el('p', { cls: 'muted chart-note' }, [ + 'A URL outside the render rotation — a facet or parameter combination a crawler invented — ', + 'misses on every single request, and every miss is an origin fetch for a document the origin ', + 'served minutes ago to a different crawler. With ', + el('code', { text: 'render.raw.enabled' }), + ' and ', + el('code', { text: 'rawCache' }), + ' on a route, that document is kept and the next crawler is answered from storage: no render ', + 'capacity, no second origin request. It is NOT a prerendered snapshot — verify the route’s ', + 'server-rendered HTML already carries its whole SEO surface before enabling it there. Both ', + 'switches are under Rendering on ', + link('Config →', () => ctx.go('config')), + '.', + ]), + ], + }); + } + + const cookieRefusals = byOutcome.get(RAW_HAS_COOKIE) ?? 0; + const oversize = byOutcome.get(RAW_OVERSIZE) ?? 0; + const ranked = [...byOutcome.entries()].filter(([outcome]) => outcome !== RAW_STORED).sort((a, b) => b[1] - a[1]); + const maxBytes = options.get('render.raw.maxBytes')?.effective; + + return card(`Raw-document cache — ${scopeLabel(data)}`, { + head: [ + enabled ? null : pill('master switch off', 'warn'), + rawRoutes.length + ? pill(`${rawRoutes.length} route${rawRoutes.length === 1 ? '' : 's'} opted in`, 'info') + : pill('no route opted in', 'warn'), + spacer(), + ], + body: [ + // The two refusals that are findings rather than traffic, each above the breakdown so it + // is not something an operator has to spot in a bar list. + cookieRefusals > 0 && + el('div', { cls: 'note bad' }, [ + el('strong', { text: `${num(cookieRefusals)} document(s) refused for setting a cookie. ` }), + 'The origin is personalizing a route that was enabled on the assumption it is shared. The ', + 'refusal is correct — a stored personalized document would be replayed to every crawler that ', + 'asks — but the assumption is not, and nothing else here reports it. Check what that route ', + 'sets, or take it off ', + el('code', { text: 'rawCache' }), + '.', + ]), + oversize > 0 && + el('div', { cls: 'note warn' }, [ + el('strong', { text: `${num(oversize)} document(s) were larger than render.raw.maxBytes` }), + maxBytes ? ` (${fmtCount(maxBytes)} bytes)` : '', + '. Those were served and not stored, so the route is enabled and cannot fill. The cap bounds ', + 'COMPRESSED bytes as the origin sent them, and budget roughly 2× it per in-flight capture.', + ]), + el('div', { cls: 'stats' }, [ + stat( + 'Stored', + fmtCount(stored), + total ? `${pct(stored, total)} of ${fmtCount(total)} attempts` : 'no attempts' + ), + stat('Refused', fmtCount(refused), 'the reasons are below — this is the number to read', { + warn: total > 0 && stored === 0, + }), + // Deliberately NOT stored ÷ serves: a document stored in this window is read by the next + // crawler, which may be in the next one. Different populations, shown side by side. + stat('Raw serves', fmtCount(rawServes), `misses answered from storage${filter ? ' · filtered' : ''}`), + ]), + total === 0 + ? el('p', { cls: 'muted chart-note' }, [ + 'No store attempt in this window. A capture is only ever attempted on a MISS on an opted-in ', + 'route, so this is either a route mix that is not missing or a window too narrow to contain ', + 'one — not a fault.', + ]) + : stored === total + ? el('div', { cls: 'note ok' }, ['Every eligible document in this window was stored.']) + : // A TABLE, not a bar list, and for the same reason the non-hit verdicts are one: the + // reason has to be readable beside the count rather than behind a hover, and a bar + // list's sub-label is a nowrap cell sized for a number. + table( + ['refusal', { text: 'documents', right: true }, { text: 'share', right: true }], + ranked.map(([outcome, count]) => { + const [means, severity] = RAW_REFUSALS[outcome] ?? ['an outcome this console does not know about', '']; + return el('tr', null, [ + el('td', null, [pill(outcome, severity), el('div', { cls: 'muted', text: means })]), + el('td', { cls: 'right mono', text: num(count) }), + el('td', { cls: 'right mono', text: pct(count, total) }), + ]); + }) + ), + el('p', { cls: 'muted chart-note' }, [ + 'One row per store ATTEMPT, which is what makes a route that is enabled and filling nothing ', + 'distinguishable from one that is switched off — the two are identical in every other number on ', + 'this page. A raw document only ever replaces an origin PROXY: it is read on a true miss and ', + 'never in place of a stale or invalidated snapshot, because those mean a render is coming and ', + 'the live origin is the better answer. Stored documents expire on ', + el('code', { text: 'render.raw.expiry' }), + ' and refill on demand, one request at a time.', + ]), + ], + }); +} + /** Distinct URLs per bot per day — how much of the corpus crawlers actually walk. */ function breadth(ctx, filter) { const state = ctx.data.breadth; diff --git a/packages/console/test/adminAssets.test.js b/packages/console/test/adminAssets.test.js index 0ca2a3f..9bec885 100644 --- a/packages/console/test/adminAssets.test.js +++ b/packages/console/test/adminAssets.test.js @@ -295,11 +295,11 @@ test('a metric the plugin emits is charted by the console, or waived with a reas * above says so and it has happened — which is why this checks the family the OTHER test is blind * to instead of replacing it. Between them, a new probe series has to be charted or waived. * - * WHAT THIS STILL DOES NOT COVER: the other dynamic families (`sitemap_*`, `demand_*`, - * `queue_health`'s gauges, `invalidation_*`). Those are legitimately unread from analytics — the - * queue gauges are read from the overview endpoint instead, and the demand series have no panel — - * so guarding them means a waiver list stating a decision per series, which belongs with whoever - * makes those decisions rather than in a catch-up change. + * WHAT THIS STILL DOES NOT COVER: the remaining dynamic families (`demand_*`, `queue_health`'s + * gauges, `invalidation_*`). Those are legitimately unread from analytics — the queue gauges are + * read from the overview endpoint instead, and the demand series have no panel — so guarding them + * means a waiver list stating a decision per series, which belongs with whoever makes those + * decisions rather than in a catch-up change. `sitemap_*` has its own test below. */ test('every probe series the catalog declares is read by the console, or waived with a reason', async () => { const { METRICS } = await import('../../plugin/src/metrics.js'); @@ -324,3 +324,72 @@ test('every probe series the catalog declares is read by the console, or waived ); } }); + +/** + * The same contract again, for the `sitemap_*` family — and this one reads the EMIT SITES, not the + * catalog. + * + * `metrics.sitemapRun` is on `DYNAMIC_SERIES_SLOT`, so the literal scan above cannot see any of + * these, and that exemption cost exactly what the probe one did: plugin v0.69.0 added + * `sitemap_not_modified` and v0.74.0 `sitemap_created_soon`, the console read neither, and every + * test on both sides stayed green. `sitemap_not_modified` is the ONLY evidence anywhere that + * conditional sitemap fetching is working — a walk that re-parses everything succeeds exactly like + * one that skipped. + * + * GROUND TRUTH IS `resources/Sitemap.js`, deliberately NOT the catalog's `values` list as the probe + * test uses. The two have drifted here and the drift is the point: at the time of writing the + * catalog enumerates six `sitemap_*` names and the plugin emits fifteen, so a catalog-based guard + * would pass while blind to the two series this test exists for. The emit site is what makes a + * series real. + */ +test('every sitemap series the plugin emits is read by the console, or waived with a reason', () => { + const source = readFileSync(fileURLToPath(new URL('../../plugin/src/resources/Sitemap.js', import.meta.url)), 'utf8'); + // To the END of the statement, not to the first `)`: the departure emitter's own argument + // contains parentheses, and a lazier match would hand this test a truncated string it could + // neither recognize nor report usefully. Every call site is one line. + const calls = [...source.matchAll(/metrics\.sitemapRun\([^,]+,\s*(.+)\);$/gm)]; + assert.ok(calls.length > 5, 'expected to find the sitemap emit sites at all — has Sitemap.js been restructured?'); + + // A series name built at the call site cannot be read from here, exactly as with the emitters + // the test above exempts — so each one is named, with what it is, rather than passing silently. + const BUILT_AT_THE_CALL_SITE = new Map([ + [ + "`departure_${name.replace(/-/g, '_')}`", + 'the post-walk sitemap-departure family (departure_render / _expire / _reattached / …) — ten ' + + 'series with dry-run-vs-armed semantics of their own. No console panel reads them: the ' + + 'Sitemaps view charts walk OUTCOMES, and departures are a separate decision surface that ' + + 'needs its own panel rather than seven more tiles on this one. Tracked, not forgotten.', + ], + ]); + + const emitted = []; + for (const [, arg] of calls) { + const literal = arg.trim().match(/^(['"])([a-z_0-9]+)\1$/); + if (literal) { + emitted.push(`sitemap_${literal[2]}`); + continue; + } + assert.ok( + BUILT_AT_THE_CALL_SITE.has(arg.trim()), + `metrics.sitemapRun is called with a series name this test cannot read (\`${arg.trim()}\`). Name it in ` + + 'BUILT_AT_THE_CALL_SITE with what it is and whether the console reads it, or pass a literal' + ); + } + assert.ok(emitted.length > 5, 'expected literal sitemap series names'); + + // The walk panel holds the SUFFIX — `totalOf('not_modified')` builds `sitemap_not_modified` — + // so the full name never appears in the client. Both spellings count, as in the probe test. + const client = [...clientSources.values()].join('\n'); + const isRead = (name) => client.includes(`'${name}'`) || client.includes(`'${name.slice('sitemap_'.length)}'`); + + const NOT_CHARTED = new Map(); + + for (const name of new Set(emitted)) { + if (NOT_CHARTED.has(name)) continue; + assert.ok( + isRead(name), + `the plugin emits prerender_ops.${name} and no console view reads it — chart it on the Sitemaps ` + + "view, or add it to this test's NOT_CHARTED with the reason" + ); + } +}); diff --git a/packages/console/test/inspectView.test.js b/packages/console/test/inspectView.test.js new file mode 100644 index 0000000..4a8e9a4 --- /dev/null +++ b/packages/console/test/inspectView.test.js @@ -0,0 +1,177 @@ +/** + * The Inspect view's explainer, executed — specifically the RENDER CADENCE card. + * + * The card exists because "how often does this URL re-render" had no honest answer in this console + * before plugin v0.77.0. The Target's `renderInterval` was the only interval on screen, and it is + * the CEILING the demand ladder schedules inside — for most of a corpus with the ladder armed it is + * not the cadence, and there was nothing to say so. `explain` now resolves it exactly as the + * scheduler does and reports the clamp that bound; this pins that the console shows the resolution + * rather than the ceiling, and that a clamp is legible without arithmetic. + */ + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { installDom } from './domShim.js'; + +installDom(); + +const { el } = await import('../src/admin/ui.js'); +const { load, render } = await import('../src/admin/views/inspect.js'); + +const HOUR = 3_600_000; + +const BROWSE = { rows: [], total: { recordCount: 0 }, nextCursor: null }; + +const CONFIG = { + schema: { + children: { + page: { children: { ttl: { kind: 'option' } } }, + cacheKey: { children: { includeDevice: { kind: 'option' } } }, + management: { children: { pageSize: { kind: 'option' } } }, + }, + }, + layers: [{ path: 'page.ttl', effective: 24 * HOUR }], +}; + +/** The explain body, shaped as PrerenderAdmin.explain returns it. */ +const explain = (cadence) => ({ + resolved: { + cacheKey: 'www.example.com/catalog/x.jsp?CN=a|desktop', + canonicalUrl: 'https://www.example.com/catalog/x.jsp?CN=a', + deviceType: 'desktop', + deviceTypeFellBack: false, + }, + ingress: { mode: 'forwarded', routeClass: 'prerender', route: null }, + allowlist: { used: ['CN'], source: 'route' }, + underGlobalAllowlist: { differs: false, allowlist: ['CN'] }, + eligibility: { prerendered: true, domainAllowed: true, excludedByPattern: null }, + verdict: { recurring: true, suppressed: false }, + residency: { scheduleReadIsAuthoritative: true, scheduleAuthoritative: true, queriedNode: 'node-a' }, + degraded: null, + cadence, + rows: { + renderTarget: { url: 'https://www.example.com/catalog/x.jsp?CN=a', renderInterval: 24 * HOUR, state: 'active' }, + renderSchedule: null, + prerenderedPage: null, + suppression: null, + }, +}); + +function makeCtx(result) { + const views = {}; + const scratch = (id) => (views[id] ??= {}); + const ctx = { + scratch, + busy: false, + get data() { + return scratch('inspect'); + }, + async get(route) { + if (route === 'pages') return { ok: true, body: BROWSE }; + if (route === 'config') return { ok: true, body: CONFIG }; + return { ok: true, body: {} }; + }, + async post() { + return { ok: true, body: {} }; + }, + render() {}, + reload() {}, + go() {}, + }; + ctx.data.result = result; + return ctx; +} + +const textOf = (ctx) => el('div', null, render(ctx)).textContent; + +const ready = async (cadence) => { + const ctx = makeCtx({ ok: true, body: explain(cadence) }); + await load(ctx); + return ctx; +}; + +test('the cadence card shows the resolution, and the target row is labelled as the ceiling', async () => { + const ctx = await ready({ + effectiveInterval: 6 * HOUR, + baseFrom: 'route', + baseInterval: 24 * HOUR, + routeInterval: 24 * HOUR, + storedInterval: null, + defaultInterval: 24 * HOUR, + demandInterval: 6 * HOUR, + demandFloor: null, + clampedBy: null, + }); + const text = textOf(ctx); + assert.match(text, /Render cadence/); + assert.match(text, /Effective interval/); + assert.match(text, /what the scheduler actually files/); + assert.match(text, /ladder rung applied/); + // The Target card's interval is the ceiling this is clamped into — reading it as the cadence is + // the mistake the card exists to prevent, so it says which it is. + assert.match(text, /Render interval \(ceiling\)/); +}); + +test('a floor clamp says the ladder has no range here, which one number could never say', async () => { + const ctx = await ready({ + effectiveInterval: 12 * HOUR, + baseFrom: 'route', + baseInterval: 24 * HOUR, + routeInterval: 24 * HOUR, + storedInterval: null, + defaultInterval: 24 * HOUR, + demandInterval: 6 * HOUR, + demandFloor: 12 * HOUR, + clampedBy: 'floor', + }); + const text = textOf(ctx); + assert.match(text, /clamped by demandFloor/); + assert.match(text, /no range to work in/); + // Both sides of the clamp are on screen: the rung the ladder wanted, and the floor that refused. + assert.match(text, /Demand rung/); + assert.match(text, /Demand floor/); +}); + +test('a ceiling clamp reports the rung as inert rather than as the cadence', async () => { + const ctx = await ready({ + effectiveInterval: 2 * HOUR, + baseFrom: 'route', + baseInterval: 2 * HOUR, + routeInterval: 2 * HOUR, + storedInterval: null, + defaultInterval: 24 * HOUR, + demandInterval: 6 * HOUR, + demandFloor: null, + clampedBy: 'ceiling', + }); + const text = textOf(ctx); + assert.match(text, /clamped by the route ceiling/); + assert.match(text, /the rung is inert here/); +}); + +test('a URL with no target gets no cadence card at all — absence is the answer, not a row of dashes', async () => { + const ctx = await ready(null); + assert.doesNotMatch(textOf(ctx), /Render cadence/); +}); + +test('an older plugin that sends no intervals never formats a missing one as a real cadence', async () => { + // `duration()` takes Math.abs, so an unguarded null formats as "0s" and undefined as "NaNs" — + // both of which read as a configured cadence rather than as an absent field. + const ctx = await ready({ + effectiveInterval: 24 * HOUR, + baseFrom: 'default', + baseInterval: 24 * HOUR, + routeInterval: null, + storedInterval: null, + defaultInterval: 24 * HOUR, + demandInterval: null, + demandFloor: null, + clampedBy: null, + }); + const text = textOf(ctx); + assert.doesNotMatch(text, /NaN/); + assert.doesNotMatch(text, /\b0s\b/); + assert.match(text, /the ladder has not evaluated this target/); + assert.match(text, /base interval/); +}); diff --git a/packages/console/test/sitemapsView.test.js b/packages/console/test/sitemapsView.test.js index ba8785d..f86a5e2 100644 --- a/packages/console/test/sitemapsView.test.js +++ b/packages/console/test/sitemapsView.test.js @@ -26,14 +26,75 @@ const CHILD = 'https://example.com/sitemap-products-1.xml'; const LIST = { node: 'node-a', lastFullPass: Date.now() - 3_600_000, - sitemaps: [{ url: ROOT, entryCount: 2, lastRefreshed: Date.now() - 7_200_000, refresh: { state: 'idle' } }], + sitemaps: [ + { + url: ROOT, + // INGESTED two hours ago, CHECKED five minutes ago. That gap is the normal steady state + // under conditional fetching, and a console that prints the first under the second's label + // sends an operator chasing a sitemap that is not stale. + entryCount: 2, + lastRefreshed: Date.now() - 7_200_000, + refresh: { state: 'idle', finishedAt: Date.now() - 300_000 }, + }, + ], +}; + +const HOUR = 3_600_000; +const BUCKETS = 4; + +/** One analytics combo, as `util/analyticsRead.js` emits it — value series, so mean x count. */ +const combo = (path, count, value) => ({ + metric: 'prerender_ops', + path, + method: null, + type: null, + count, + total: 0, + mean: value, + median: value, + p95: value, + counts: new Array(BUCKETS).fill(count / BUCKETS), + means: new Array(BUCKETS).fill(value), +}); + +/** A day of walks: 30 documents fetched, 24 of them unchanged, 10 creates of which 6 went fast. */ +const ANALYTICS = { + available: true, + scope: 'node', + node: 'node-a', + rangeMs: 24 * HOUR, + startMs: 0, + endMs: 24 * HOUR, + bucketMs: 6 * HOUR, + bucketCount: BUCKETS, + cacheAgeMs: 0, + scan: { ms: 9, scanned: 400, kept: 20, cap: 20_000 }, + series: [ + combo('sitemap_sitemaps', 2, 15), + combo('sitemap_not_modified', 2, 12), + combo('sitemap_created', 2, 5), + combo('sitemap_created_soon', 2, 3), + combo('sitemap_updated', 2, 1), + combo('sitemap_skipped', 2, 400), + combo('sitemap_removed', 2, 2), + combo('sitemap_failed', 2, 0), + ], }; /** The index detail: entries are child sitemaps, and the server still sends page-shaped fields. */ const INDEX_DETAIL = { node: 'node-a', sitemap: { url: ROOT, isIndex: true, entryCount: 2, lastRefreshed: Date.now() - 7_200_000, parentUrl: null }, - refresh: { state: 'idle', finishedAt: Date.now() - 7_200_000 }, + refresh: { + state: 'idle', + finishedAt: Date.now() - 300_000, + created: 10, + createdSoon: 6, + updated: 1, + removed: 2, + notModified: 24, + sitemapsProcessed: 30, + }, // Structurally zero for an index: a walk attributes every Target to the child that listed it. targetCount: { count: 0, cap: 1000, truncated: false }, entries: [ @@ -54,7 +115,7 @@ const CHILD_DETAIL = { limit: 50, }; -function makeCtx() { +function makeCtx({ analytics = ANALYTICS } = {}) { const views = {}; const scratch = (id) => (views[id] ??= {}); const calls = { posts: [], reloads: 0 }; @@ -67,6 +128,7 @@ function makeCtx() { }, async get(route) { if (route === 'sitemaps') return { ok: true, body: LIST }; + if (route === 'analytics') return { ok: true, body: analytics }; return { ok: true, body: null }; }, async post(route, body) { @@ -154,3 +216,62 @@ test('a selection that no longer resolves falls back to a root instead of a dead assert.equal(ctx.data.selected, ROOT); assert.equal(ctx.data.detail.sitemap.url, ROOT); }); + +// ---- ingested vs checked, and the walk counters ----------------------------- + +test('a sitemap reports when it was INGESTED and, separately, when it was checked', async () => { + const ctx = await ready(); + const text = textOf(ctx); + // Under conditional fetching a 304 writes nothing, so the document's own timestamp is the last + // time its ENTRIES changed. Labelling that "refreshed" reads as a walk that has not reached + // this sitemap in hours, which is exactly the feature working. + assert.match(text, /ingested/); + assert.match(text, /checked/); + assert.doesNotMatch(text, /never refreshed/, 'the old label conflated the two timestamps'); +}); + +test('the last walk reports its 304s and its fast-path creates, not just created/updated/removed', async () => { + const ctx = await ready(); + const text = textOf(ctx); + // sitemap_not_modified non-zero is the ONLY evidence conditional fetching works: a walk that + // re-parses every document succeeds exactly like one that skipped 24 of 30. + assert.match(text, /Not modified \(304\)/); + assert.match(text, /24 of 30 documents/); + // A SUBSET of created, stated with its denominator so it can never be read as a fourth count. + assert.match(text, /6 of the creates rendered soon/); +}); + +test('the walk panel sums passes across roots and nodes, and names the fast-path share', async () => { + const ctx = await ready(); + const text = textOf(ctx); + assert.match(text, /Walk activity/); + // 2 emits x 15 documents = 30 fetched, 24 not modified: 80%. + assert.match(text, /Documents fetched/); + assert.match(text, /80%/); + // 10 created, 6 of them soon. + assert.match(text, /60% rendered soon/); +}); + +test('zero 304s across a day of walks is reported as the rollout not working', async () => { + const analytics = { + ...ANALYTICS, + series: ANALYTICS.series.map((s) => + s.path === 'sitemap_not_modified' ? { ...s, mean: 0, means: s.means.map(() => 0) } : s + ), + }; + const ctx = makeCtx({ analytics }); + await load(ctx); + const text = textOf(ctx); + // Nothing errors and no walk fails — the only symptom is a full re-parse and prune scan per + // document, forever, which is the entire saving the feature was for. + assert.match(text, /No document was answered 304 in this window/); + assert.match(text, /If-Modified-Since/); +}); + +test('a window with no finished walk says so rather than reading as a dead scheduler', async () => { + const ctx = makeCtx({ analytics: { ...ANALYTICS, series: [] } }); + await load(ctx); + const text = textOf(ctx); + assert.match(text, /Walk activity/); + assert.match(text, /no walk completed inside the window|emitted once per FINISHED walk/i); +}); diff --git a/packages/console/test/trafficView.test.js b/packages/console/test/trafficView.test.js index f9110d0..a9e0ad1 100644 --- a/packages/console/test/trafficView.test.js +++ b/packages/console/test/trafficView.test.js @@ -837,3 +837,94 @@ test('a configured gate that refused nothing says so, rather than reading as unc assert.match(text, /1 route gated/); assert.match(text, /refused nothing in this window/); }); + +// ---- the raw-document cache ------------------------------------------------- + +/** + * A window where `render.raw` is on, mostly refusing. The refusals are the panel's whole reason to + * exist: an enabled route that stores nothing produces the same miss rate, the same origin proxies + * and the same absence of errors as a route nobody enabled. + */ +const RAW_ANALYTICS = { + ...ANALYTICS, + series: [ + combo('bot_serve', 'cache', 'hit', 'googlebot', 400), + combo('bot_serve', 'raw', 'raw', 'googlebot', 300), + combo('bot_serve', 'origin', 'miss', 'googlebot', 300), + combo('bot_request', 'www.example.com', 'googlebot', 'desktop', 1000), + combo('page_age', 'googlebot', 'desktop', null, 400, 3 * HOUR, 9 * HOUR), + // raw_cache: method = outcome, one emit per store ATTEMPT. + combo('prerender_ops', 'raw_cache', 'stored', null, 120), + combo('prerender_ops', 'raw_cache', 'has-cookie', null, 80), + combo('prerender_ops', 'raw_cache', 'oversize', null, 40), + combo('prerender_ops', 'raw_cache', 'not-200', null, 10), + ], +}; + +const RAW_CONFIG = { + ...CONFIG, + layers: [ + ...CONFIG.layers, + { path: 'render.raw.enabled', effective: true }, + { path: 'render.raw.maxBytes', effective: 1_048_576 }, + ].map((layer) => + layer.path === 'ingress.routes' + ? { + path: 'ingress.routes', + effective: layer.effective.map((r) => (r.path === '/catalog/' ? { ...r, rawCache: true } : r)), + } + : layer + ), +}; + +const rawReady = async () => { + const ctx = makeCtx({ analytics: RAW_ANALYTICS, config: RAW_CONFIG }); + await load(ctx); + return ctx; +}; + +test('a raw serve is cache-served and counts toward offload, but is never a fresh hit', async () => { + assert.ok(isCacheServed('raw'), 'a stored origin document answered it — the origin was spared'); + const ctx = await rawReady(); + const text = textOf(ctx); + // 700 of 1000 serves came from storage (400 hit + 300 raw); only 400 were fresh hits. Reading + // the SOURCE instead of the verdict would have made cache-served 40% and falling as the + // feature started working. + assert.match(text, /Cache-served/); + assert.match(text, /70%/, 'hit + raw'); + assert.match(text, /raw documents, not snapshots/, 'the split is named, never silently folded in'); + // And a raw serve has no cadence to be late against, so it must not read as a coverage or + // cadence problem in the non-hit taxonomy. + const rows = notHitRows([combo('bot_serve', 'raw', 'raw', 'googlebot', 300)]); + assert.equal(rows[0].family, 'raw'); +}); + +test('the raw-cache panel leads with the refusals, because a silent route looks like a disabled one', async () => { + const ctx = await rawReady(); + const text = textOf(ctx); + assert.match(text, /Raw-document cache/); + // 120 of 250 attempts stored; the other 130 are the number an operator has to see. + assert.match(text, /Refused/); + assert.match(text, /has-cookie/); + assert.match(text, /oversize/); + assert.match(text, /not-200/); +}); + +test('a personalized origin and an undersized cap are called out, not left in a bar list', async () => { + const ctx = await rawReady(); + const text = textOf(ctx); + // has-cookie is the one outcome worth an alert: the route was enabled on the assumption it is + // shared, and the origin says otherwise. + assert.match(text, /refused for setting a cookie/); + // oversize means the feature is on and structurally cannot fill — a settings fix. + assert.match(text, /larger than render\.raw\.maxBytes/); +}); + +test('a deployment with the raw cache off is told what it is for, not shown an empty chart', async () => { + const ctx = makeCtx(); + await load(ctx); + const text = textOf(ctx); + assert.match(text, /Raw-document cache/); + assert.match(text, /render\.raw\.enabled/, 'the capability is described, with the switch that turns it on'); + assert.doesNotMatch(text, /Refused/, 'no chart of zeroes for a subsystem nobody enabled'); +});