From d91c46b5f390933e80f410abfd83aa6805f12d8a Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 17:40:54 -0400 Subject: [PATCH 1/6] =?UTF-8?q?feat(browser):=20readiness=20contracts=20?= =?UTF-8?q?=E2=80=94=20decide=20a=20render=20is=20complete=20by=20asking?= =?UTF-8?q?=20the=20page;=20v1.30.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every settle signal today is a timer, and a timer cannot be wrong out loud. A render that missed a widget, or that serialized pre-hydration markup, reports 200, non-empty and indexable, and nothing downstream can tell. Measured under 6x CPU throttling: 8 of 15 renders finish incomplete and all 15 report outcome=ok, one storing 426KB instead of 958KB with 11 of 27 islands unhydrated and zero product links — while the hand-written review gate was satisfied. A contract states per page type what a COMPLETE render contains. The renderer holds until that is true AND the DOM has gone quiet, then stops, and posts the per-clause result back so an incomplete render is a fact rather than a silence. Six assertion forms, each because something else could not express it: presence; anyOf (an empty-but-legitimate listing page is structurally identical to one whose grid has not arrived); absent (skeletons); shed (frameworks drop a marker on hydrate — the check that catches pre-hydration markup); every+contains ("every rail is filled" survives a template change where "at least 3 rails" does not); and nonEmptyText. Any clause may carry onlyIf, keying it on what the page's own JSON-LD declares or on what the DOM holds — the difference between guessing whether reviews should exist and asking the document. Three things the measurements forced: - The quiet window is not optional and is where the speed comes from. The same policy with an EMPTY contract saves the same time to within 30ms on 7 of 8 pages. Stopping the instant a contract holds saves more and loses the rails (99% of product links on one page, 100% on an empty facet): rails have no server-rendered placeholder, so no clause can assert one is coming. - No verdict before the parser finishes, and a shed clause requires its elements to exist. Both were violated: a contract declared a page complete at 4ms and stored zero review nodes, because absence-shaped clauses are true on an empty document and it is quiet for the same reason. - Contracts rot when templates change, so a clause that has never been true — once every other clause holds and the DOM is quiet — stands aside after unmetGraceMs and is reported unsatisfied. Cost then degrades to roughly today's behaviour instead of waiting out the timeout on every render forever; without that valve one unsatisfiable clause measured +385% wall. Also lands per-URL learned expectations: observations are compared against what this URL last produced, and three consecutive shortfalls re-baseline rather than alarming forever, so a rail removed site-wide converges instead of failing that URL permanently. Verified on five live page types (two product, listing, empty facet, home) with a paired A/B/A parity gate against a measured per-page churn floor: -31% to -54% wall, -14% to -35% CPU, content identical on every one. An unsatisfied contract always falls through to the normal settle, so a badly written contract can cost time but never content. No-op when unconfigured. Co-Authored-By: Claude Opus 5 (1M context) --- package-lock.json | 2 +- packages/browser/README.md | 71 +++ packages/browser/package.json | 2 +- packages/browser/src/RenderJob.ts | 16 + packages/browser/src/config.ts | 9 + packages/browser/src/domMonitor.ts | 149 +++++ packages/browser/src/readiness.ts | 653 +++++++++++++++++++++ packages/browser/src/renderer.ts | 135 ++++- packages/browser/test/expectations.test.ts | 90 +++ packages/browser/test/readiness.test.ts | 262 +++++++++ 10 files changed, 1383 insertions(+), 6 deletions(-) create mode 100644 packages/browser/src/domMonitor.ts create mode 100644 packages/browser/src/readiness.ts create mode 100644 packages/browser/test/expectations.test.ts create mode 100644 packages/browser/test/readiness.test.ts diff --git a/package-lock.json b/package-lock.json index d4d6cb4..9b23a55 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8516,7 +8516,7 @@ }, "packages/browser": { "name": "@harperfast/prerender-browser", - "version": "1.29.0", + "version": "1.30.0", "license": "Apache-2.0", "dependencies": { "mqtt": "^5.10.4", diff --git a/packages/browser/README.md b/packages/browser/README.md index 0906003..04fe130 100644 --- a/packages/browser/README.md +++ b/packages/browser/README.md @@ -291,6 +291,29 @@ include what you change: "pathPattern": "^/product/", // only product pages have this widget }, ], + // optional: per-page-type statements of what a COMPLETE render contains. When one governs a + // render it REPLACES the timer-based settle — see "Readiness contracts" below. Absent → no-op. + "readiness": { + "onSatisfied": "quiet", // stop once the contract holds AND the DOM has been still for quietMs + "quietMs": 250, + "unmetGraceMs": 1000, // stop waiting on a clause that has never been true once everything else holds + "contracts": [ + { + "name": "product", + "pathPattern": "^/product/", + "require": [ + { "name": "price", "selector": "[data-price]", "nonEmptyText": true, "textMatches": "\\$\\s?\\d" }, + { "name": "hydrated", "selector": "astro-island", "shed": "ssr" }, + { "name": "no-skeletons", "absent": ".skeleton" }, + { "name": "grid-or-empty", "anyOf": [{ "selector": ".tile", "minCount": 1 }, { "selector": ".no-results" }] }, + { "name": "rails-filled", "every": ".rail", "contains": ".slide" }, + // only required when the page's own JSON-LD says the content should exist + { "name": "reviews", "selector": ".review", "onlyIf": { "jsonLdNumber": "aggregateRating.ratingCount" } }, + ], + "observe": [{ "name": "product-links", "selector": "a[href^=/product/]" }], // reported, never waited on + }, + ], + }, "postProcess": { "stripScripts": true, // remove executable ` + ) + ); + // Islands that shed their marker on "hydration", plus one that never does. + case '/islands': + return res.end( + page( + '', + `` + ) + ); + case '/no-islands': + return res.end(page('

nothing to hydrate

')); + // A page whose structured data declares reviews, and one that declares none. + case '/declares-reviews': + return res.end( + page( + '
', + ` + ` + ) + ); + case '/declares-none': + return res.end( + page('
', '') + ); + // Containers that are all filled, and a page with none at all. + case '/rails': + return res.end(page('
1
')); + case '/no-rails': + return res.end(page('

none

')); + // A grid with products, and one that is legitimately empty. + case '/grid': + return res.end(page('
')); + case '/grid-empty': + return res.end(page('

nothing

')); + default: + return res.end(page('

ok

')); + } + }); + await new Promise((resolve) => origin.listen(0, '127.0.0.1', resolve)); + base = `http://127.0.0.1:${(origin.address() as AddressInfo).port}`; +}); + +after(() => origin.close()); + +const render = async (path: string, contract: Record, extra: Record = {}) => { + const result = await renderOnce({ + url: `${base}${path}`, + captureNonIndexable: true, + config: { + navigation: { networkIdleMs: 50, networkIdleTimeoutMs: 200, domStableMs: 0, domStableTimeoutMs: 500 }, + scroll: { enabled: false }, + readiness: { onSatisfied: 'quiet', quietMs: 100, contracts: [contract], ...extra }, + } as never, + }); + return result; +}; + +test('a contract holds the render until the content it names arrives', async () => { + const result = await render('/late', { + name: 'late', + require: [{ name: 'items', selector: '.item', minCount: 2 }], + timeoutMs: 5000, + }); + assert.equal(result.job.readiness?.satisfied, true); + assert.match(result.html ?? '', /class="item"/); + // It cannot have been satisfied before the content existed. + assert.ok((result.job.readiness?.firstSatisfiedMs ?? 0) >= 200, 'must not report success before the content landed'); +}); + +test('a clause is not satisfiable by a document that has not parsed', async () => { + // `absent` is the shape that fails this way: nothing matches an empty document, so the clause is + // trivially true before the page exists — and the page is quiet for the same reason. + const result = await render('/late', { + name: 'absent-only', + require: [ + { name: 'no-skeletons', absent: '.skeleton' }, + { name: 'items', selector: '.item', minCount: 2 }, + ], + timeoutMs: 5000, + }); + assert.equal(result.job.readiness?.satisfied, true); + assert.match(result.html ?? '', /class="item"/, 'the late content still had to arrive'); +}); + +test('"every island hydrated" requires islands to exist, and honours maxRemaining', async () => { + // One island never sheds its marker, so the strict form can never hold. + const strict = await render('/islands', { + name: 'strict', + require: [{ name: 'hydrated', selector: 'my-island', shed: 'ssr' }], + timeoutMs: 1200, + }); + assert.equal(strict.job.readiness?.satisfied, false); + assert.equal(strict.job.readiness?.require.find((r) => r.name === 'hydrated')?.count, 1); + + const tolerant = await render('/islands', { + name: 'tolerant', + require: [{ name: 'hydrated', selector: 'my-island', shed: 'ssr', maxRemaining: 1 }], + timeoutMs: 5000, + }); + assert.equal(tolerant.job.readiness?.satisfied, true); + + // A page with NO islands: vacuously "all hydrated", which is what an unparsed document looks like. + const none = await render('/no-islands', { + name: 'none', + require: [{ name: 'hydrated', selector: 'my-island', shed: 'ssr' }], + timeoutMs: 800, + }); + assert.equal(none.job.readiness?.satisfied, false, 'no islands must not read as fully hydrated'); + + const allowed = await render('/no-islands', { + name: 'allowed', + require: [{ name: 'hydrated', selector: 'my-island', shed: 'ssr', allowNone: true }], + timeoutMs: 5000, + }); + assert.equal(allowed.job.readiness?.satisfied, true); +}); + +test('a guarded clause applies only when the page declares the content', async () => { + const declared = await render('/declares-reviews', { + name: 'declared', + require: [ + { + name: 'reviews', + selector: '.review', + minCount: 1, + onlyIf: { jsonLdNumber: 'aggregateRating.ratingCount', atLeast: 1 }, + }, + ], + timeoutMs: 5000, + }); + assert.equal(declared.job.readiness?.satisfied, true); + assert.equal(declared.job.readiness?.require[0].skipped, undefined, 'the page declared 12, so it was checked'); + assert.ok((declared.job.readiness?.firstSatisfiedMs ?? 0) >= 200, 'and it waited for them'); + + // The same contract on a page whose structured data carries no such field: the page is saying + // there are none. Measured on a live product page, failing this instead cost +385% wall. + const none = await render('/declares-none', { + name: 'undeclared', + require: [ + { + name: 'reviews', + selector: '.review', + minCount: 1, + onlyIf: { jsonLdNumber: 'aggregateRating.ratingCount', atLeast: 1 }, + }, + ], + timeoutMs: 5000, + }); + assert.equal(none.job.readiness?.satisfied, true); + assert.equal(none.job.readiness?.require[0].skipped, true, 'skipped, and reported as skipped rather than passed'); +}); + +test('a DOM-guarded clause stands aside when the page has none of the thing', async () => { + const empty = await render('/grid-empty', { + name: 'empty-grid', + require: [ + { name: 'grid-or-zero', anyOf: [{ selector: '.tile', minCount: 1 }, { selector: '.no-results' }] }, + { name: 'tiles-imaged', every: '.tile', contains: 'img', onlyIf: { present: '.tile', atLeast: 1 } }, + ], + timeoutMs: 5000, + }); + assert.equal(empty.job.readiness?.satisfied, true); + assert.equal(empty.job.readiness?.require.find((r) => r.name === 'tiles-imaged')?.skipped, true); + + const populated = await render('/grid', { + name: 'grid', + require: [{ name: 'tiles-imaged', every: '.tile', contains: 'img', onlyIf: { present: '.tile', atLeast: 1 } }], + timeoutMs: 5000, + }); + assert.equal(populated.job.readiness?.satisfied, true); + assert.equal(populated.job.readiness?.require[0].skipped, undefined, 'a populated grid IS checked'); +}); + +test('"every container is filled" is not satisfied by having no containers', async () => { + const none = await render('/no-rails', { + name: 'rails', + require: [{ name: 'rails', every: '.rail', contains: '.slide' }], + timeoutMs: 800, + }); + assert.equal(none.job.readiness?.satisfied, false, 'zero of zero filled is true and means nothing'); + + const filled = await render('/rails', { + name: 'rails', + require: [{ name: 'rails', every: '.rail', contains: '.slide' }], + timeoutMs: 5000, + }); + assert.equal(filled.job.readiness?.satisfied, true); +}); + +test('a clause that can never hold stands aside instead of burning the whole timeout', async () => { + // THE ROT VALVE. A renamed class makes a clause permanently false; without this the contract waits + // out `timeoutMs` on every render of the page type, for as long as nobody notices. + const started = Date.now(); + const result = await render( + '/no-islands', + { + name: 'rotted', + require: [ + { name: 'fine', selector: 'p', minCount: 1 }, + { name: 'renamed-away', selector: '.this-class-no-longer-exists', minCount: 1 }, + ], + timeoutMs: 20000, + }, + { unmetGraceMs: 300 } + ); + const waited = Date.now() - started; + + assert.equal(result.job.readiness?.satisfied, false, 'and it is reported as unsatisfied'); + assert.equal(result.job.readiness?.require.find((r) => r.name === 'renamed-away')?.ok, false); + assert.ok(waited < 10000, `gave up in ${waited}ms rather than waiting out the 20s timeout`); + assert.ok(result.html, 'the render still produced content — a rotted contract costs time, never content'); +}); + +test('the render still serializes when a contract is never satisfied', async () => { + const result = await render('/no-islands', { + name: 'impossible', + require: [{ name: 'nope', selector: '#absent', minCount: 1 }], + timeoutMs: 700, + }); + assert.equal(result.job.readiness?.satisfied, false); + assert.match(result.html ?? '', /nothing to hydrate/); +}); From e55ac1fd5758f6c240e40e67bc8de643ae00c110 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 17:52:22 -0400 Subject: [PATCH 2/6] fix(browser): key the resolved-config cache by the config, not by its overrides array MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `resolveConfigForJob` cached resolved configs keyed by (device + matching override names) and cleared that cache only when the IDENTITY of `config.overrides` changed. Two configs that share an overrides array — which is what you get by spreading one config and changing a field — therefore resolved to whichever one was seen first, for every URL matching an override, for the rest of the process. A URL matching NO override returns before the cache, so it was unaffected. That is what makes the bug hard: it presents as "this setting works on the home page and nowhere else", and it cost two wasted measurement runs before it was found. Production has been safe only because a config reload reparses and so happens to produce a new array; nothing enforces that. A WeakMap keyed on the base config cannot fail that way — a different config is a different key by construction, and an old config's entries are collected with it. Also posts the readiness verdict back with the render result (`VariantMetadata. readiness`): the contract name, satisfied, the names of unmet and skipped clauses, and the observation counts the consumer stores as this URL's expectation. Without it on the wire the consumer cannot make an unsatisfiable clause visible, which is the trap contracts are supposed to avoid rather than reproduce. Co-Authored-By: Claude Opus 5 (1M context) --- packages/browser/src/RenderJob.ts | 49 +++++++++++++++++++ packages/browser/src/config.ts | 26 +++++++--- packages/browser/test/configOverrides.test.ts | 24 +++++++++ 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/packages/browser/src/RenderJob.ts b/packages/browser/src/RenderJob.ts index 461970d..b64da1e 100644 --- a/packages/browser/src/RenderJob.ts +++ b/packages/browser/src/RenderJob.ts @@ -315,6 +315,23 @@ export default class RenderJob { * Builds the encoded body too (the expensive gzip) so a caller assembling several variants pays * it once per variant and retries re-send the same bytes. */ + /** The contract's verdict in its wire form, or undefined when no contract governed this render. */ + private readinessReport(): ReadinessReport | undefined { + const r = this.readiness; + if (!r) return undefined; + return { + contract: r.contract, + satisfied: r.satisfied, + unmet: r.require.filter((c) => !c.ok).map((c) => c.name), + skipped: r.require.filter((c) => c.skipped).map((c) => c.name), + waitedMs: r.waitedMs, + firstSatisfiedMs: r.firstSatisfiedMs, + learned: r.learned ?? {}, + shortfalls: r.shortfalls ?? [], + rebaselined: r.rebaselined ?? false, + }; + } + async resultMetadata(): Promise<{ metadata: VariantMetadata; contentBuffer: Buffer | null }> { const attemptError = this.error; const metadata: VariantMetadata = { @@ -325,6 +342,7 @@ export default class RenderJob { redirectedTo: this.redirectedTo, isIndexable: this.isIndexable, structuredOffers: this.structuredOffers, + readiness: this.readinessReport(), outcome: this.outcome, // Present only when true, so the flat legacy envelope is byte-identical for every render // that did not reuse a document (and an older plugin never sees the key at all). @@ -385,6 +403,9 @@ export default class RenderJob { redirectedTo: this.redirectedTo, isIndexable: this.isIndexable, structuredOffers: this.structuredOffers, + // Carried even on a failed result build: a render that could not be reported is exactly + // when knowing whether its contract held is most useful. + readiness: this.readinessReport(), outcome: 'error', // Carried through so the plugin still sees where this variant's document came from, even // though its body never made it onto the wire. @@ -456,6 +477,32 @@ export default class RenderJob { } /** One variant's share of a posted result — see `RenderJob.resultMetadata`. */ +/** + * What a readiness contract said, as it travels back to the consumer. + * + * Deliberately narrower than the in-process `ReadinessResult`: `unmet` carries only the names of + * clauses that did not hold, because that is what a metric needs to make an unsatisfiable clause + * visible — a gate that quietly fails on every render is the trap this feature exists to avoid, and + * it is only avoided if the failure reaches something that counts it. `learned` is the observation + * counts the consumer stores as this URL's expectation for next time. + */ +export type ReadinessReport = { + contract: string; + satisfied: boolean; + /** Names of the clauses that did not hold. Empty when satisfied. */ + unmet: string[]; + /** Names of clauses a guard decided did not apply — never conflate with "checked and passed". */ + skipped: string[]; + waitedMs: number; + firstSatisfiedMs: number | null; + /** Observation counts, for the consumer to store as this URL's expectation. */ + learned: Record; + /** Observations that fell far below what this URL last produced. */ + shortfalls: Array<{ name: string; expected: number; got: number; ratio: number }>; + /** The shortfalls repeated enough times to be the page's new shape; re-learn from this render. */ + rebaselined: boolean; +}; + export type VariantMetadata = { deviceType: string; statusCode: number | undefined; @@ -464,6 +511,8 @@ export type VariantMetadata = { redirectedTo: string | undefined; isIndexable: boolean | undefined; structuredOffers: Array | null | undefined; + /** Present only when a contract governed this render, so an older consumer never sees the key. */ + readiness?: ReadinessReport; outcome: JobOutcome; documentReused: true | undefined; documentPrefetched: true | undefined; diff --git a/packages/browser/src/config.ts b/packages/browser/src/config.ts index cfed052..5952a05 100644 --- a/packages/browser/src/config.ts +++ b/packages/browser/src/config.ts @@ -914,8 +914,19 @@ export type ResolvedConfig = { // matching override names), not by URL: every product page resolves the same config, so the cache // holds one entry per distinct combination rather than one per URL. Bounded by construction — the // number of combinations is a property of the config, not of the corpus. -const resolvedCache = new Map(); -let resolvedCacheFor: ConfigOverride[] | undefined; +// Keyed by the BASE CONFIG OBJECT, not by its overrides array. +// +// The previous key was the identity of `config.overrides`, which is wrong whenever two configs share +// an overrides array — exactly what happens when one config is derived from another by spreading it +// and changing a field. Every URL matching an override then resolved to the FIRST config's cached +// result, silently and for the rest of the process; a URL matching none was unaffected, because that +// path returns before the cache. It cost two wasted measurement runs before it was found, and it +// presents as "this feature works on the home page and nowhere else", which is not a shape anyone +// debugs quickly. +// +// A WeakMap on the config itself cannot have that failure: a different base config is a different +// key by construction, and an old config's entries are collected with it. +const resolvedCache = new WeakMap>(); /** * The effective config for one render. Matches `config.overrides` against this job's URL path and @@ -933,9 +944,10 @@ export const resolveConfigForJob = ( // The cache is keyed by name-signature, so it must be dropped when the config itself is replaced // (a live config reload). Identity of the overrides array is the cheapest correct witness. - if (resolvedCacheFor !== overrides) { - resolvedCache.clear(); - resolvedCacheFor = overrides; + let cache = resolvedCache.get(config); + if (!cache) { + cache = new Map(); + resolvedCache.set(config, cache); } let path = ''; @@ -954,14 +966,14 @@ export const resolveConfigForJob = ( if (!applied.length) return { config, applied }; const key = `${deviceType}\u0000${applied.join('\u0000')}`; - let resolved = resolvedCache.get(key); + let resolved = cache.get(key); if (!resolved) { const names = new Set(applied); resolved = overrides.reduce( (acc, override) => (names.has(override.name) ? deepMerge(acc, override.config) : acc), config ); - resolvedCache.set(key, resolved); + cache.set(key, resolved); } return { config: resolved, applied }; }; diff --git a/packages/browser/test/configOverrides.test.ts b/packages/browser/test/configOverrides.test.ts index 0dc0b07..98115ed 100644 --- a/packages/browser/test/configOverrides.test.ts +++ b/packages/browser/test/configOverrides.test.ts @@ -169,3 +169,27 @@ test('devices and pathPattern shapes are validated', () => { /config must be an object/ ); }); + +test('two configs sharing an overrides array resolve independently', () => { + // THE REGRESSION. The resolved-config cache used to be keyed by the identity of + // `config.overrides` and cleared only when that array changed — so a config derived from another + // by spreading it and changing a field resolved to the FIRST config's cached result, for every + // URL that matched an override, for the rest of the process. A URL matching NO override was + // unaffected, because that path returns before the cache, which is what made it present as "this + // setting works on the home page and nowhere else". + const overrides: ConfigOverride[] = [ + { name: 'product', pathPattern: '^/product/', config: { navigation: { networkIdleMs: 111 } } }, + ]; + const first = mergeConfig({ overrides, scroll: { topSettleMs: 300 } }); + const second = mergeConfig({ overrides, scroll: { topSettleMs: 900 } }); + + const a = resolveConfigForJob(first, { url: 'https://example.com/product/x', deviceType: 'mobile' }); + const b = resolveConfigForJob(second, { url: 'https://example.com/product/x', deviceType: 'mobile' }); + + assert.deepEqual(a.applied, ['product']); + assert.deepEqual(b.applied, ['product']); + assert.equal(a.config.navigation.networkIdleMs, 111, 'the override still applies'); + assert.equal(b.config.navigation.networkIdleMs, 111); + assert.equal(a.config.scroll.topSettleMs, 300); + assert.equal(b.config.scroll.topSettleMs, 900, 'the second config must not resolve to the first'); +}); From 32c58dd254a6e04af763975af8858db9083ebc6d Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 17:58:58 -0400 Subject: [PATCH 3/6] feat(plugin): make a render's readiness verdict visible, and learn what each URL produces; v0.79.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser now posts what its readiness contract said. Without something on this side reading it, contracts would reproduce the exact trap they exist to avoid: a gate that fails on every render and costs a timeout, with nothing anywhere saying so. Two consumers. 1. `render_readiness`, a new metric. `verdict` is the share of renders that finished COMPLETE — the only signal that separates a render missing a widget from a good one, since both are 200, non-empty and indexable (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported outcome=ok). `unmet` names the CLAUSE, so a contract that has rotted against a template change reads as one clause failing across a page type rather than as an unexplained slowdown. `satisfied_ms` is what `timeoutMs` should be tuned from: a p95 approaching the timeout means the contract is being abandoned under load and the optimisation is quietly gone. 2. `RenderExpectation`, per-URL, node-local, written on the result path beside the probe claim. A contract bounds what it NAMES, and recommendation rails cannot usefully be named — they have no server-rendered placeholder, so "every rail that exists is filled" is true of a page that ended up with one rail instead of three. Measured: a render that satisfied its contract stored 610 product links where that URL normally stores 674. The page's own history is the only oracle that covers it. It converges rather than alarming forever, which is the whole difficulty: a rail removed site-wide must not fail that URL on every future render. A shortfall is a vote, not a verdict — three in a row and the expectation is re-learned. A SUSPECTED shortfall deliberately does not re-learn, because learning from a render we believe is short would ratchet the expectation down to whatever the page just failed to produce, which is how a real regression would erase its own evidence. The comparison runs where the data already is, on the result. Handing the expectation to the renderer at claim time would cost a cross-database point read per job or a denormalization onto RenderSchedule — residency-pinned and rewritten on every render, the two costs its own schema comments exist to avoid. The browser still supports being given expectations for the in-render path; nothing needs it for detection. Both are best-effort on the render path, like recordPageClaim: a regression signal must never cost a render. Version note: main is at 0.78.0 and open PR #178 bumps to 0.75.0, which would downgrade it. This takes 0.79.0. Co-Authored-By: Claude Opus 5 (1M context) --- package-lock.json | 2 +- packages/browser/README.md | 15 ++ packages/browser/src/renderer.ts | 9 +- packages/plugin/METRICS.md | 69 ++++----- packages/plugin/package.json | 2 +- packages/plugin/src/metrics.js | 61 ++++++++ packages/plugin/src/resources/RenderQueue.js | 31 ++++ packages/plugin/src/schemas/schema.graphql | 31 ++++ .../plugin/src/util/readinessExpectation.js | 137 ++++++++++++++++++ packages/plugin/test/metrics.test.js | 41 ++++++ .../plugin/test/readinessExpectation.test.js | 68 +++++++++ 11 files changed, 429 insertions(+), 37 deletions(-) create mode 100644 packages/plugin/src/util/readinessExpectation.js create mode 100644 packages/plugin/test/readinessExpectation.test.js diff --git a/package-lock.json b/package-lock.json index 9b23a55..956da0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.78.0", + "version": "0.79.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/browser/README.md b/packages/browser/README.md index 04fe130..877d48e 100644 --- a/packages/browser/README.md +++ b/packages/browser/README.md @@ -439,6 +439,21 @@ should exist and the rendered DOM is held to it. waiting out the timeout on every render forever; without that valve one unsatisfiable clause measured +385% wall. +- **Cap `timeoutMs` low.** A contract that does not satisfy pays its whole timeout **and then the + full fallback settle on top**, and it costs CPU rather than only wall, because the page keeps + executing while the poll runs. Measured on a concurrency ladder: with a 15s timeout, one render in + twelve under contention took 10.8s and CPU/render rose 36%; at 3s the straggler was 4.8s and CPU + rose 12%, with the median render unchanged either way. Erring low is the safe direction — giving up + early falls back to exactly what the renderer does without a contract, so a too-low timeout costs + the optimisation and never the content. Set it from the `firstSatisfiedMs` distribution the results + carry rather than from a guess; if p95 approaches the timeout, the contract is being abandoned + under load and the win is quietly gone. + +Note this is a different failure from the rot valve above, and needs its own control: `unmetGraceMs` +fires when a clause has never been true **and the page has gone quiet**, which is what a template +change looks like. A page that is merely slow is still mutating, so the valve does not fire and the +timeout is what bounds the wait. + An unsatisfied contract always falls through to the normal settle, so a badly written contract can cost a render time but never content. `job.readiness` carries `satisfied`, per-clause `ok`/`count`/ `firstTrueMs`, and any `observe` counts. diff --git a/packages/browser/src/renderer.ts b/packages/browser/src/renderer.ts index f2e3267..ee8b8bf 100644 --- a/packages/browser/src/renderer.ts +++ b/packages/browser/src/renderer.ts @@ -739,10 +739,12 @@ const renderer: Renderer = async (page, job) => { // is lazy, then hold until the page says it is complete. Everything below is what runs when no // contract governs this render, or when one is not satisfied. let contractSatisfied = false; + let contractScrolled = false; if (contract) { if (config.scroll.enabled) { await page.evaluate(scrollToBottom, config.scroll.stepMs); await scrollToTop(); + contractScrolled = true; } contractSatisfied = await awaitContract(contract); } @@ -751,10 +753,13 @@ const renderer: Renderer = async (page, job) => { if (contractStopped) { // The contract asserted the page is complete and quiet, which is a stronger statement than any // wait below would establish. - } else if (config.scroll.enabled && config.scroll.settleUntilStable) { + } else if (config.scroll.enabled && !contractScrolled && config.scroll.settleUntilStable) { await scrollSettle(); } else { - if (config.scroll.enabled) { + // A contract that timed out already scrolled this page, and then waited out its whole timeout + // on top — so the fallback repeating the pass is redundant work on the one path that is by + // definition already paying twice. + if (config.scroll.enabled && !contractScrolled) { // Scroll to the bottom to trigger lazy-loaded content, then back to the top // (e.g. so a scroll-aware navbar renders in its default state). await page.evaluate(scrollToBottom, config.scroll.stepMs); diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index f87617b..2bf039b 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -121,39 +121,42 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi One-line summaries; `src/metrics.js` carries the full description of every dimension value and the reasoning behind it. -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set), `raw_cache` (outcome: `stored`, or the refusal — `not-200`, `staging`, `has-cookie`, `content-type`, `no-store`, `no-body`, `oversize`, `capture-failed`, `write-failed`, `empty` (a 200 with no body — never stored), `capture-busy` (`maxConcurrentCaptures` reached; served, not stored). **Read the refusals, not the successes**: an enabled route that is filling nothing looks exactly like a disabled one unless the reason is recorded. `oversize` climbing means `render.raw.maxBytes` is under the route's real document size; `has-cookie` climbing means the origin is personalizing a route that was assumed shared, which is the one outcome worth an alert). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_not_modified` — documents the origin answered 304 to, whose entries were never re-parsed and whose prune scan never ran. **A steady ZERO where `sitemap.conditional` is enabled means the origin is not honouring `If-Modified-Since`** and every pass is doing full work, so the refresh frequency should come back down; plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ------------------ | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied/rebaselined) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set), `raw_cache` (outcome: `stored`, or the refusal — `not-200`, `staging`, `has-cookie`, `content-type`, `no-store`, `no-body`, `oversize`, `capture-failed`, `write-failed`, `empty` (a 200 with no body — never stored), `capture-busy` (`maxConcurrentCaptures` reached; served, not stored). **Read the refusals, not the successes**: an enabled route that is filling nothing looks exactly like a disabled one unless the reason is recorded. `oversize` climbing means `render.raw.maxBytes` is under the route's real document size; `has-cookie` climbing means the origin is personalizing a route that was assumed shared, which is the one outcome worth an alert). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied/rebaselined) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied/rebaselined) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_not_modified` — documents the origin answered 304 to, whose entries were never re-parsed and whose prune scan never ran. **A steady ZERO where `sitemap.conditional` is enabled means the origin is not honouring `If-Modified-Since`** and every pass is doing full work, so the refresh frequency should come back down; plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 29d4e0d..f0935c0 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.78.0", + "version": "0.79.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/metrics.js b/packages/plugin/src/metrics.js index c2f9d9c..22f7f7f 100644 --- a/packages/plugin/src/metrics.js +++ b/packages/plugin/src/metrics.js @@ -348,6 +348,52 @@ export const METRICS = Object.freeze({ }, }), + render_readiness: metric('render_readiness', { + kind: 'value', + emittedBy: 'resources/RenderQueue.js', + cadence: 'per device variant of a posted result that a readiness contract governed; nothing when none did', + summary: 'What each page type\u2019s completeness contract said about the renders it governed.', + usefulFor: + 'THE POINT OF CONTRACTS IS THAT AN INCOMPLETE RENDER BECOMES VISIBLE, and this is where it becomes ' + + 'visible. `verdict` is the share of renders that finished complete — a rising `unsatisfied` share is ' + + 'the fleet telling you it is storing pages that are missing something, which no other signal here ' + + 'can say (those renders are 200, non-empty and indexable). `unmet` names the CLAUSE, so a contract ' + + 'that has rotted against a template change shows up as one clause failing across every render of ' + + 'its page type instead of as a silent slowdown. `satisfied_ms` is how `timeoutMs` should be tuned: ' + + 'if its p95 approaches the configured timeout the contract is being abandoned under load and the ' + + 'optimisation is quietly gone.', + caveats: + '`unmet` emits once per failing clause, so it does NOT sum to renders — read it against the ' + + '`unsatisfied` count in `verdict`. A clause that a guard decided did not apply is not unmet and is ' + + 'not counted; that distinction is the difference between "checked and failed" and "nothing to ' + + 'check". `shortfall` is a comparison against what this URL last produced, so it is silent on a ' + + 'first render and after a rebaseline.', + gatedBy: 'a readiness contract matching the rendered URL (config.readiness)', + dimensions: { + path: { + name: 'series', + values: ['verdict', 'unmet', 'shortfall', 'satisfied_ms'], + description: + 'verdict = counter of how the contract ended. unmet = counter, one per clause that did not ' + + 'hold. shortfall = counter, one per observation that fell far below this URL\u2019s history. ' + + 'satisfied_ms = distribution of how long the contract took to first hold.', + }, + method: { + name: 'contract', + description: 'The contract\u2019s configured name — i.e. the page type, as the config defines it.', + }, + type: { + name: 'verdict (verdict) / clause (unmet) / observation (shortfall)', + values: ['satisfied', 'unsatisfied', 'rebaselined'], + description: + 'verdict: satisfied | unsatisfied | rebaselined (the observation shortfalls repeated often ' + + 'enough to be the page\u2019s new shape). unmet/shortfall: the configured clause or ' + + 'observation name, so the enumeration above applies to the verdict series only. Null on ' + + 'satisfied_ms.', + }, + }, + }), + queue_health: metric('queue_health', { kind: 'value', emittedBy: @@ -717,6 +763,21 @@ export const metrics = Object.freeze({ /** What became of one posted render result — exactly one call per result; the `render` outcome series. */ renderOutcome: (outcome, detail) => server.recordAnalytics(true, 'render', 'outcome', outcome, detail ?? null), + /** How one render's readiness contract ended — one call per governed variant. */ + renderReadiness: (contract, verdict) => + server.recordAnalytics(true, 'render_readiness', 'verdict', contract, verdict), + + /** One clause that did not hold. Emitted per clause, so it does not sum to renders. */ + renderReadinessUnmet: (contract, clause) => + server.recordAnalytics(true, 'render_readiness', 'unmet', contract, clause), + + /** One observation that fell far below what this URL last produced. */ + renderReadinessShortfall: (contract, observation) => + server.recordAnalytics(true, 'render_readiness', 'shortfall', contract, observation), + + /** How long the contract took to first hold — what `timeoutMs` should be tuned from. */ + renderReadinessMs: (ms, contract) => server.recordAnalytics(ms, 'render_readiness', 'satisfied_ms', contract, null), + /** One claim pass's duration and how it ended — a queue_health series, so the queue reads in one scan. */ claimScan: (durationMs, result) => server.recordAnalytics(durationMs, 'queue_health', 'claim_scan_ms', result, null), diff --git a/packages/plugin/src/resources/RenderQueue.js b/packages/plugin/src/resources/RenderQueue.js index 9df0c52..485447f 100644 --- a/packages/plugin/src/resources/RenderQueue.js +++ b/packages/plugin/src/resources/RenderQueue.js @@ -13,6 +13,7 @@ import { PRERENDER, } from '../util/routeClass.js'; import { decideInterval } from '../util/demandLadder.js'; +import { recordReadinessExpectation } from '../util/readinessExpectation.js'; import { recordPageClaim } from '../util/changeProbe.js'; import { backoffWait } from '../util/failureBackoff.js'; import { recordUnroutedPath } from '../util/unrouted.js'; @@ -539,6 +540,30 @@ export class RenderQueue extends Resource { metrics.renderTime(variant.renderTime, variant.statusCode, candidacy); } + // What each governed render's completeness contract said. This is the whole point of contracts + // being on the wire: a render that finished INCOMPLETE is otherwise indistinguishable from a + // good one — 200, non-empty, indexable, and nothing anywhere saying it is missing a widget. + // Measured under CPU contention, 8 of 15 renders finished with their contract unsatisfied and + // every one of them reported outcome=ok. + for (const variant of variants) { + const readiness = variant.readiness; + if (!readiness) continue; // no contract governed this render + metrics.renderReadiness( + readiness.contract, + readiness.rebaselined ? 'rebaselined' : readiness.satisfied ? 'satisfied' : 'unsatisfied' + ); + // Per CLAUSE, so a contract that has rotted against a template change reads as one clause + // failing across every render of its page type rather than as an unexplained slowdown. + for (const clause of readiness.unmet ?? []) metrics.renderReadinessUnmet(readiness.contract, clause); + for (const shortfall of readiness.shortfalls ?? []) + metrics.renderReadinessShortfall(readiness.contract, shortfall.name); + // How long it took to first hold — the distribution `timeoutMs` should be tuned from. Only + // when it held: an unsatisfied contract has no such time, and emitting its timeout here would + // drag the percentile toward the very ceiling the reader is checking against. + if (typeof readiness.firstSatisfiedMs === 'number') + metrics.renderReadinessMs(readiness.firstSatisfiedMs, readiness.contract); + } + // 1. A redirect the browser bailed on at navigation, or a rendered-through client-side redirect // that produced nothing. Decided by `processRedirectResult` for the whole URL, exactly as one // device's result decided it before. The lane comes back so the fast-retry branch inside it @@ -711,8 +736,14 @@ export class RenderQueue extends Resource { // // `recordPageClaim` never rejects (it catches and warns internally), so it cannot fail the // result from inside this set — a probe optimisation must not cost a render. + // The readiness observations, compared against what this URL last produced. Rides in the same + // concurrent set as the page writes and the probe claim: it is one node-local point read and + // one small write, and like `recordPageClaim` it never rejects — a regression signal must not + // cost a render. + const governed = stored.find((variant) => variant.readiness?.learned); await Promise.all([ recordPageClaim(scheduleUrl, claiming.structuredOffers, cachedAt), + ...(governed ? [recordReadinessExpectation(scheduleUrl, governed.readiness)] : []), ...stored.map((variant) => { variant.headers['x-harper-rendered'] = '1'; return databases.page_cache.PrerenderedPage.put(variant.storeKey, { diff --git a/packages/plugin/src/schemas/schema.graphql b/packages/plugin/src/schemas/schema.graphql index 09d7202..5c8526d 100644 --- a/packages/plugin/src/schemas/schema.graphql +++ b/packages/plugin/src/schemas/schema.graphql @@ -430,6 +430,37 @@ type ProbeState @table(database: "probe_state", replicate: false) @sealed { ruleFingerprint: String } +# What the last accepted render of a URL produced — the defence against content disappearing with +# nothing to notice. +# +# A readiness contract bounds what it NAMES. Recommendation rails cannot usefully be named: they have +# no server-rendered placeholder, so a contract can only assert "every rail that exists is filled", +# which is true of a page that ended up with one rail instead of three. The page's own history is the +# only oracle that covers what no clause can — a URL that carried three rails and 350 product links +# yesterday and none today has lost something, and nothing else here would say so. +# +# `replicate: false` and node-local for the same reason ProbeState is: the render result is handled +# on the key's owner, so the row is written and read by the same node. A URL that moves owners loses +# its history and re-learns on its next render, which is the same self-healing behaviour as a first +# render — never a false alarm. +# +# NOT read at claim time, deliberately. Handing the expectation to the renderer would mean either a +# cross-database point read per claimed job or denormalizing onto RenderSchedule, and that table is +# residency-pinned and rewritten on every render — the two costs its own schema comments exist to +# avoid. The comparison happens where the data already is: on the result. +type RenderExpectation @table(database: "probe_state", replicate: false) @sealed { + url: String @primaryKey + # Observation name -> count, as JSON. A map rather than columns because what a contract observes + # is configuration: the names come from `readiness.contracts[].observe` and change with it. + counts: String + # How many renders in a row have reported the same shortfall. THE CONVERGENCE MECHANISM: a rail + # removed site-wide must not fail this URL forever, so once the same shortfall has been seen + # `rebaselineAfter` times the expectation is re-learned instead of alarmed. Once is a lost rail; + # three times in a row is the new shape of the page. + consecutiveShortfalls: Int + updatedAt: Date +} + # Same anchor, same reason as CoordinationAnchor: ProbeState is `replicate: false`, and a # database with zero replicable tables spins the replication subscription (harper-pro#685). type ProbeStateAnchor @table(database: "probe_state") @sealed { diff --git a/packages/plugin/src/util/readinessExpectation.js b/packages/plugin/src/util/readinessExpectation.js new file mode 100644 index 0000000..3a7d2a2 --- /dev/null +++ b/packages/plugin/src/util/readinessExpectation.js @@ -0,0 +1,137 @@ +import { metrics } from '../metrics.js'; + +/** + * What the last accepted render of a URL produced, and whether this render fell short of it. + * + * ## The gap this closes + * + * A readiness contract bounds what it NAMES, and the most valuable thing on a commerce page cannot + * usefully be named. Recommendation rails have no server-rendered placeholder — wrapper and content + * appear in the same frame — so a contract can only assert "every rail that exists is filled", which + * is true of a page that ended up with one rail instead of three. Measured: a render that satisfied + * its contract stored 610 product links where the same URL normally stores 674, and every clause + * held. + * + * The page's own history is the oracle that covers it. Nothing has to be written down and nothing + * rots: the expectation is whatever this URL last actually produced. + * + * ## Why it converges instead of alarming forever + * + * The naive version is a trap. Compare against history, flag anything that dropped, and the first + * legitimate change — a rail removed site-wide, a product delisted, a redesign — flags that URL on + * every render from then on. So a shortfall is a VOTE, not a verdict: once the same shortfall has + * been seen `rebaselineAfter` times in a row the expectation is re-learned and the counter resets. + * Once is a lost rail; three times in a row is the new shape of the page. + * + * ## Where this runs, and what it deliberately does not do + * + * On the render-result path, owner-scoped and node-local, exactly like `recordPageClaim` — and + * best-effort for the same reason: a render must never fail because a regression signal could not + * be recorded. + * + * It does NOT hand the expectation to the renderer at claim time. That would mean a cross-database + * point read per claimed job, or denormalizing onto RenderSchedule, which is residency-pinned and + * rewritten on every render. The browser supports being given expectations (it can then hold a + * render briefly while a shortfall recovers), and a consumer that wants that can plumb it; the + * detection itself does not need it, because the comparison can happen where the data already is. + * + * NOTE: the same comparison exists in the browser package (`readiness.assessExpectations`) for that + * in-render path. The rule is small but it is the same rule, and the two must agree — change both. + */ + +const DEFAULTS = Object.freeze({ + /** Fractional drop that counts as a shortfall. Measured run-to-run churn on these pages is ~5%. */ + tolerance: 0.5, + /** Consecutive shortfalls after which the expectation is stale and the page has simply changed. */ + rebaselineAfter: 3, +}); + +const table = () => databases.probe_state.RenderExpectation; + +/** + * Compare one render's observations against what this URL last produced, record what is worth + * recording, and store the new expectation. + * + * Pure decision, impure edges: `assess` below is exported for the tests, because the interesting + * cases (first render, churn, convergence) are all in the rule rather than in the storage. + */ +export const assess = (learned, stored, policy = DEFAULTS) => { + const previous = stored?.counts ?? null; + // A first render has nothing to regress from, and treating "unknown" as "zero" would make every + // new URL look broken. + if (!previous) return { shortfalls: [], rebaselined: false, next: { counts: learned, consecutive: 0 } }; + + const shortfalls = []; + for (const [name, count] of Object.entries(learned)) { + const expected = previous[name]; + // No history for this observation, or the page never had any: nothing to fall short OF. + if (typeof expected !== 'number' || expected <= 0) continue; + if (count < expected * (1 - policy.tolerance)) shortfalls.push({ name, expected, got: count }); + } + + if (!shortfalls.length) return { shortfalls: [], rebaselined: false, next: { counts: learned, consecutive: 0 } }; + + const consecutive = (stored.consecutiveShortfalls ?? 0) + 1; + // Enough renders have agreed on the drop that it is the page, not the render. Accept it, re-learn + // from this render, and reset — so the next real regression is judged against the page as it is. + if (consecutive >= policy.rebaselineAfter) { + return { shortfalls: [], rebaselined: true, next: { counts: learned, consecutive: 0 } }; + } + // Still suspicious: report it, and KEEP the old expectation. Learning from a render we believe is + // short would ratchet the expectation down to whatever the page just failed to produce, which is + // exactly how a real regression would erase its own evidence. + return { shortfalls, rebaselined: false, next: { counts: previous, consecutive } }; +}; + +/** + * Record one render's readiness observations against this URL's history. Never throws. + * + * `readiness` is the report the browser posted (`VariantMetadata.readiness`); `learned` on it is the + * observation counts. Called once per URL per result, from the render-result path. + */ +export const recordReadinessExpectation = async (url, readiness, policy = DEFAULTS) => { + try { + const learned = readiness?.learned; + // Nothing observed means nothing to compare and nothing to store — a contract with no + // `observe` clauses, or a renderer that predates the field. + if (!learned || Object.keys(learned).length === 0) return; + + const stored = await table().get({ id: url, select: ['counts', 'consecutiveShortfalls'] }); + const parsed = stored + ? { counts: safeParse(stored.counts), consecutiveShortfalls: stored.consecutiveShortfalls } + : null; + const verdict = assess(learned, parsed, policy); + + for (const shortfall of verdict.shortfalls) metrics.renderReadinessShortfall(readiness.contract, shortfall.name); + if (verdict.rebaselined) metrics.renderReadiness(readiness.contract, 'rebaselined'); + if (verdict.shortfalls.length) { + logger.warn( + `Prerender ${url}: rendered fewer than this URL last produced — ` + + verdict.shortfalls.map((s) => `${s.name} ${s.got} vs ${s.expected}`).join(', ') + ); + } + + const fields = { + counts: JSON.stringify(verdict.next.counts), + consecutiveShortfalls: verdict.next.consecutive, + updatedAt: new Date(), + }; + // patch cannot create and put would clobber nothing else here, but the read already happened + // for the comparison, so choosing costs nothing extra. + if (stored) await table().patch(url, fields); + else await table().put(url, { url, ...fields }); + } catch (error) { + logger.warn(error, `Prerender: could not record the readiness expectation for ${url}`); + } +}; + +const safeParse = (value) => { + try { + const parsed = JSON.parse(value ?? 'null'); + return parsed && typeof parsed === 'object' ? parsed : null; + } catch { + // A row we cannot read is a row we re-learn from this render, which is the same self-healing + // path as a first render. + return null; + } +}; diff --git a/packages/plugin/test/metrics.test.js b/packages/plugin/test/metrics.test.js index 15176de..7b68a4b 100644 --- a/packages/plugin/test/metrics.test.js +++ b/packages/plugin/test/metrics.test.js @@ -304,3 +304,44 @@ test('describeMetrics is JSON-serializable and covers both the plugin and the bu described.plugin[0].summary = 'mutated'; assert.notEqual(Object.values(METRICS)[0].summary, 'mutated'); }); + +test('render_readiness puts the series in the path slot and the contract in the method slot', () => { + const v = emitted(() => metrics.renderReadiness('product', 'unsatisfied')); + assert.deepEqual(v, { + value: true, + metric: 'render_readiness', + path: 'verdict', + method: 'product', + type: 'unsatisfied', + }); + + const u = emitted(() => metrics.renderReadinessUnmet('product', 'rails-filled')); + assert.deepEqual([u.metric, u.path, u.method, u.type], ['render_readiness', 'unmet', 'product', 'rails-filled']); + + const s = emitted(() => metrics.renderReadinessShortfall('product', 'product-links')); + assert.deepEqual([s.metric, s.path, s.method, s.type], ['render_readiness', 'shortfall', 'product', 'product-links']); + + // The only series carrying a measurement rather than a count, and the one `timeoutMs` is tuned from. + const ms = emitted(() => metrics.renderReadinessMs(1629, 'product')); + assert.deepEqual( + [ms.value, ms.metric, ms.path, ms.method, ms.type], + [1629, 'render_readiness', 'satisfied_ms', 'product', null] + ); +}); + +test('every render_readiness series and verdict the emitters can produce is declared', () => { + const declared = METRICS.render_readiness.dimensions; + for (const emit of [ + () => metrics.renderReadiness('c', 'satisfied'), + () => metrics.renderReadinessUnmet('c', 'x'), + () => metrics.renderReadinessShortfall('c', 'x'), + () => metrics.renderReadinessMs(1, 'c'), + ]) { + assert.ok(declared.path.values.includes(emitted(emit).path), 'undeclared series'); + } + // The verdict enumeration is closed; the clause/observation names in the same slot are not, which + // is why the catalog says the enumeration applies to the verdict series only. + for (const verdict of ['satisfied', 'unsatisfied', 'rebaselined']) { + assert.ok(declared.type.values.includes(emitted(() => metrics.renderReadiness('c', verdict)).type)); + } +}); diff --git a/packages/plugin/test/readinessExpectation.test.js b/packages/plugin/test/readinessExpectation.test.js new file mode 100644 index 0000000..221ce59 --- /dev/null +++ b/packages/plugin/test/readinessExpectation.test.js @@ -0,0 +1,68 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +// What a URL produced LAST time is the only expectation that cannot rot — and the only one that can +// turn into a permanent false alarm. These tests are about the second half: the rule has to catch a +// page that quietly lost content AND get out of the way when the page really changed. +// +// The same rule exists in the browser package (`readiness.assessExpectations`) for the in-render +// path. They must agree; if you change one, change both. + +globalThis.server = { hostname: 'test-node', recordAnalytics: () => {} }; +globalThis.databases = { probe_state: { RenderExpectation: {} } }; +globalThis.logger = { warn: () => {}, info: () => {}, error: () => {} }; + +const { assess } = await import('../src/util/readinessExpectation.js'); + +const history = (counts, consecutiveShortfalls = 0) => ({ counts, consecutiveShortfalls }); + +test('a first render has nothing to regress from, and is learned from', () => { + const v = assess({ rails: 3, links: 350 }, null); + assert.deepEqual(v.shortfalls, []); + assert.equal(v.rebaselined, false); + assert.deepEqual(v.next.counts, { rails: 3, links: 350 }); +}); + +test('run-to-run churn is not a shortfall', () => { + // Measured churn on these pages is ~5% on link and image counts; the tolerance is 50%, so + // ordinary personalisation must never trip this. + assert.deepEqual(assess({ links: 332 }, history({ links: 350 })).shortfalls, []); +}); + +test('losing the rails is caught even though no contract clause names them', () => { + const v = assess({ rails: 0, links: 3 }, history({ rails: 3, links: 350 })); + assert.deepEqual(v.shortfalls.map((s) => s.name).sort(), ['links', 'rails']); + assert.equal(v.shortfalls.find((s) => s.name === 'rails').expected, 3); +}); + +test('a page that never had the thing cannot fall short of it', () => { + assert.deepEqual(assess({ rails: 0 }, history({ rails: 0 })).shortfalls, []); + // And an observation with no history at all is ignored rather than read as zero. + assert.deepEqual(assess({ brandNew: 0 }, history({ rails: 3 })).shortfalls, []); +}); + +test('a suspected shortfall does NOT re-learn — that is how a regression would erase its evidence', () => { + const v = assess({ rails: 0 }, history({ rails: 3 }, 0)); + assert.equal(v.shortfalls.length, 1); + assert.deepEqual(v.next.counts, { rails: 3 }, 'the old expectation is kept'); + assert.equal(v.next.consecutive, 1); +}); + +test('the same shortfall, repeated, stops being a regression and becomes the page', () => { + // THE CONVERGENCE MECHANISM. A rail removed site-wide must not fail this URL forever. + assert.equal(assess({ rails: 0 }, history({ rails: 3 }, 0)).rebaselined, false, 'once is a lost rail'); + assert.equal(assess({ rails: 0 }, history({ rails: 3 }, 1)).rebaselined, false, 'twice is still suspicious'); + + const third = assess({ rails: 0 }, history({ rails: 3 }, 2)); + assert.equal(third.rebaselined, true, 'three times in a row is the new shape of the page'); + assert.deepEqual(third.shortfalls, [], 'and it stops being reported as a shortfall'); + assert.deepEqual(third.next.counts, { rails: 0 }, 'the page as it is now becomes the expectation'); + assert.equal(third.next.consecutive, 0, 'so the next real regression starts from zero'); +}); + +test('a recovery resets the counter rather than leaving it armed', () => { + const v = assess({ rails: 3 }, history({ rails: 3 }, 2)); + assert.deepEqual(v.shortfalls, []); + assert.equal(v.rebaselined, false, 'nothing fell short, so nothing is being re-learned'); + assert.equal(v.next.consecutive, 0); +}); From 74f02c3dba42bc097f55458bc7c33c3fbf3bfb3a Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 18:21:13 -0400 Subject: [PATCH 4/6] feat(browser): a report-only mode, so a contract's timeout can be set from the fleet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A contract's `timeoutMs` cannot honestly be chosen from a developer machine. Measured on live pages with the timeout lifted to 30s so each render reports what it needed: the product contract holds at 856ms unthrottled, 1,749ms at 2x CPU throttle, and NOT AT ALL within 30s at 4x. The fleet's mean render is ~11s against ~4s on that machine, so production sits inside that band — and a gate armed with the first number would clip its own tail, presenting as "renders are incomplete" rather than as "the timeout is wrong". `onSatisfied: 'report'` evaluates the contract alongside the ordinary settle and reports the verdict while gating nothing: same clauses, same per-clause timings, same wire format, and a render that behaves exactly as it does with no contract configured. The observer stops as soon as the settle it watches finishes, which is what guarantees it cannot extend a render, and takes one final look afterwards so the verdict describes the DOM that is about to be serialized. That yields the `satisfied_ms` distribution the gate should be tuned from, at no risk, which is the same shape of rollout the change probe used. It also documents a real limit found while measuring: the rot valve interprets "every clause either holds or has never been true, and the DOM is quiet" as a rotted contract, and a CPU-starved page looks exactly like that — it goes quiet between bursts of work. At 4x throttle a product render reported unsatisfied after ~8.7s while its final content was complete. It fails safe (the fallback settle runs, the content is fine) but it is a false alarm in the one metric that exists to say renders are incomplete. Rot is a property of a page type ACROSS renders, which is what the per-URL `unsatisfiable` list is for; report mode sidesteps it entirely, and arming should follow the fleet's own numbers. Co-Authored-By: Claude Opus 5 (1M context) --- packages/browser/src/readiness.ts | 14 +++- packages/browser/src/renderer.ts | 90 ++++++++++++++++++++++++- packages/browser/test/readiness.test.ts | 63 ++++++++++++++++- 3 files changed, 161 insertions(+), 6 deletions(-) diff --git a/packages/browser/src/readiness.ts b/packages/browser/src/readiness.ts index 6dde4fe..cb8f86b 100644 --- a/packages/browser/src/readiness.ts +++ b/packages/browser/src/readiness.ts @@ -208,6 +208,14 @@ export type ReadinessConfig = { /** * What the settle phase does once a contract is satisfied. * + * - `report` — DO NOT GATE. The contract is evaluated alongside the ordinary settle and its + * verdict is reported, while the render behaves exactly as it does without a contract. This is + * how a contract should be rolled out: `timeoutMs` has to be set against the fleet, and the + * fleet's own `satisfied_ms` distribution is the only honest source for it. Measured on a + * laptop a product contract holds at 856ms; at 2x CPU throttle 1,749ms; at 4x it does not hold + * at all within 30s. Picking the number from the first of those would arm a gate that clips its + * own tail in production, and the failure would look like "renders are incomplete" rather than + * "the timeout is wrong". * - `quiet` — stop once the contract holds AND the DOM has been unchanged for `quietMs`. * - `plateau` — stop once the contract holds, then still run the full final plateau. The * conservative setting for a page type whose contract is new or known to be partial. @@ -224,7 +232,7 @@ export type ReadinessConfig = { * defensible — a page that goes quiet early because it is broken now fails a check instead of * being cached. */ - onSatisfied: 'quiet' | 'plateau'; + onSatisfied: 'report' | 'quiet' | 'plateau'; /** * How long to keep waiting for a clause that has NEVER been true in this render, once every other * clause holds and the DOM has gone quiet. Default 1000ms. @@ -509,8 +517,8 @@ export function validateReadiness(readiness: unknown): void { if (readiness === undefined) return; const cfg = readiness as ReadinessConfig; if (typeof cfg !== 'object' || cfg === null) throw new Error('prerender config: readiness must be an object'); - if (cfg.onSatisfied !== undefined && cfg.onSatisfied !== 'quiet' && cfg.onSatisfied !== 'plateau') { - throw new Error("prerender config: readiness.onSatisfied must be 'quiet' or 'plateau'"); + if (cfg.onSatisfied !== undefined && !['report', 'quiet', 'plateau'].includes(cfg.onSatisfied)) { + throw new Error("prerender config: readiness.onSatisfied must be 'report', 'quiet' or 'plateau'"); } if (!Array.isArray(cfg.contracts)) throw new Error('prerender config: readiness.contracts must be an array'); for (const contract of cfg.contracts) { diff --git a/packages/browser/src/renderer.ts b/packages/browser/src/renderer.ts index ee8b8bf..18b6dd9 100644 --- a/packages/browser/src/renderer.ts +++ b/packages/browser/src/renderer.ts @@ -735,12 +735,94 @@ const renderer: Renderer = async (page, job) => { return satisfied; }; + /** + * Evaluate the contract ALONGSIDE the ordinary settle, reporting what it saw and gating nothing. + * + * This is the rollout mode, and it exists because the gate's own `timeoutMs` cannot honestly be + * chosen from a developer machine: measured on live pages, a product contract holds at 856ms + * unthrottled, 1,749ms at 2x CPU throttle, and not at all within 30s at 4x. A fleet sits somewhere + * in that band. Running as an observer first produces the distribution the gate should be tuned + * from, at no risk — the render behaves exactly as it does with no contract configured. + * + * Stops as soon as the settle it is watching finishes, so it can never extend a render. + */ + const observeContract = async ( + governing: NonNullable>, + until: { done: boolean } + ): Promise => { + const started = Date.now(); + const pollMs = governing.pollMs ?? config.navigation.domStablePollMs; + const payload = { + require: governing.require, + observe: governing.observe ?? [], + tolerance: config.navigation.domStableTolerance, + }; + const firstTrue = new Map(); + let firstSatisfiedMs: number | null = null; + let last: Awaited> | null = null; + + while (!until.done) { + try { + last = await page.evaluate(evaluateContract, payload); + } catch { + break; + } + for (const clause of last.require) { + if (clause.ok && !firstTrue.has(clause.name)) firstTrue.set(clause.name, Date.now() - started); + } + if (last.started && last.require.every((r) => r.ok) && firstSatisfiedMs === null) { + firstSatisfiedMs = Date.now() - started; + } + if (until.done) break; + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } + + // One last look, so the verdict describes the DOM that is about to be serialized rather than + // the last poll before the settle ended. + try { + last = (await page.evaluate(evaluateContract, payload)) ?? last; + for (const clause of last.require) { + if (clause.ok && !firstTrue.has(clause.name)) firstTrue.set(clause.name, Date.now() - started); + } + if (last.started && last.require.every((r) => r.ok) && firstSatisfiedMs === null) { + firstSatisfiedMs = Date.now() - started; + } + } catch { + /* page gone — report what the loop saw */ + } + + const expectation = assessExpectations(last?.observe ?? [], job.expectations, { + ...DEFAULT_EXPECTATION_POLICY, + ...(config.readiness?.expectations ?? {}), + }); + job.readiness = { + contract: governing.name, + satisfied: !!last && last.started && last.require.every((r) => r.ok), + shortfalls: expectation.shortfalls, + rebaselined: expectation.rebaselined, + learned: expectation.learned, + waitedMs: Date.now() - started, + firstSatisfiedMs, + require: (last?.require ?? governing.require.map((a) => ({ name: a.name, ok: false, count: 0 }))).map((r) => ({ + ...r, + firstTrueMs: firstTrue.get(r.name), + })), + observe: last?.observe ?? [], + } satisfies ReadinessResult; + }; + // A contract REPLACES the timer-based settle rather than joining it: scroll once to trip whatever // is lazy, then hold until the page says it is complete. Everything below is what runs when no // contract governs this render, or when one is not satisfied. + const observing = { done: false }; + let observer: Promise | null = null; let contractSatisfied = false; let contractScrolled = false; - if (contract) { + // Report mode watches; it never decides. Everything below then runs exactly as it would with no + // contract configured. + if (contract && config.readiness?.onSatisfied === 'report') { + observer = observeContract(contract, observing); + } else if (contract) { if (config.scroll.enabled) { await page.evaluate(scrollToBottom, config.scroll.stepMs); await scrollToTop(); @@ -790,6 +872,12 @@ const renderer: Renderer = async (page, job) => { // `topSettleMs` exists — so a plateau measured before it would not cover the churn the scroll // causes. This way the last thing checked is the state that gets serialized. if (config.navigation.finalDomStable && !contractStopped) await domStable(); + // The observer must not outlive the settle it is watching: stopping it here is what guarantees + // report mode cannot change a render's duration. + if (observer) { + observing.done = true; + await observer; + } timings.settle = Date.now() - settleStart; if (finalRes) { diff --git a/packages/browser/test/readiness.test.ts b/packages/browser/test/readiness.test.ts index 02fa996..d5e13db 100644 --- a/packages/browser/test/readiness.test.ts +++ b/packages/browser/test/readiness.test.ts @@ -82,12 +82,23 @@ before(async () => { after(() => origin.close()); -const render = async (path: string, contract: Record, extra: Record = {}) => { +const render = async ( + path: string, + contract: Record, + extra: Record = {}, + navigation: Record = {} +) => { const result = await renderOnce({ url: `${base}${path}`, captureNonIndexable: true, config: { - navigation: { networkIdleMs: 50, networkIdleTimeoutMs: 200, domStableMs: 0, domStableTimeoutMs: 500 }, + navigation: { + networkIdleMs: 50, + networkIdleTimeoutMs: 200, + domStableMs: 0, + domStableTimeoutMs: 500, + ...navigation, + }, scroll: { enabled: false }, readiness: { onSatisfied: 'quiet', quietMs: 100, contracts: [contract], ...extra }, } as never, @@ -260,3 +271,51 @@ test('the render still serializes when a contract is never satisfied', async () assert.equal(result.job.readiness?.satisfied, false); assert.match(result.html ?? '', /nothing to hydrate/); }); + +test('report mode reports the verdict and changes nothing about the render', async () => { + // The rollout mode. A contract naming content that never arrives must NOT hold the render — the + // whole point is that the timeout can be chosen from the fleet's own distribution later, at no + // risk now. + const gated = await render('/no-islands', { + name: 'impossible', + require: [{ name: 'nope', selector: '#absent', minCount: 1 }], + timeoutMs: 3000, + }); + + const started = Date.now(); + const reported = await render( + '/no-islands', + { name: 'impossible', require: [{ name: 'nope', selector: '#absent', minCount: 1 }], timeoutMs: 3000 }, + { onSatisfied: 'report' } + ); + const reportedMs = Date.now() - started; + + // Both know the contract did not hold, and name the clause. + assert.equal(gated.job.readiness?.satisfied, false); + assert.equal(reported.job.readiness?.satisfied, false); + assert.equal(reported.job.readiness?.require[0].name, 'nope'); + // But report mode did not spend the contract's wait on it. + assert.ok( + reportedMs < (gated.renderTimeMs ?? 3000) + 1500, + `report mode must not gate: took ${reportedMs}ms against a gated ${gated.renderTimeMs}ms` + ); + assert.match(reported.html ?? '', /nothing to hydrate/); +}); + +test('report mode still times how long a satisfiable contract took to hold', async () => { + // This is the number the gate's timeoutMs is meant to be tuned from, so it has to survive the + // mode that exists to collect it. The settle is given enough room to reach the late content ON + // ITS OWN — report mode must not extend it, so a settle that ends first would (correctly) report + // the render as incomplete, which is a different test. + const result = await render( + '/late', + { name: 'late', require: [{ name: 'items', selector: '.item', minCount: 2 }], timeoutMs: 5000 }, + { onSatisfied: 'report' }, + { domStableMs: 300, domStableTimeoutMs: 3000 } + ); + assert.equal(result.job.readiness?.satisfied, true); + assert.ok( + (result.job.readiness?.firstSatisfiedMs ?? 0) >= 200, + 'it must report WHEN the content arrived, not merely that it did' + ); +}); From b3d06c6b0801aa310da37c80a83cf5971a2cf210 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 18:32:11 -0400 Subject: [PATCH 5/6] fix(browser,plugin): five defects a pre-merge review pass found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit None are in the happy path; all five are the shapes that go unnoticed. 1. A malformed `textMatches` built its RegExp outside the evaluator's try, so one bad pattern threw out of the WHOLE evaluation and discarded every other clause's result for that tick. It now fails only its own clause — and is rejected at config load, which is where it should have been stopped. 2. Readiness numerics were unvalidated. A NaN or negative `timeoutMs` makes the gate's deadline NaN, the loop never runs, and the contract is silently disabled: a config that looks enabled and protects nothing, which is the worst outcome available. `waitFor` already validated its numerics; this now matches. 3. `stableMs` was documented on the contract type and never read. Dead config is worse than absent config — someone sets it and believes it. Removed. 4. Report mode claimed it could never change a render's duration, and that was nearly true rather than true: stopping the observer waited out whatever remained of its poll interval. The sleep is now interruptible, so the stop is prompt. The one final evaluation stays — the verdict has to describe the DOM being serialized — and it is a single ~1-2ms call. 5. The `verdict` metric double-counted. RenderQueue emits one verdict per variant and the expectation store emitted an extra `rebaselined` on top, so the series stopped summing to one per render — which is exactly how its own documentation frames it. Rebaselines now ride their own series. Review provenance, stated because it matters: every outside-model leg was unavailable on this machine (agy not installed, cursor-agent missing, codex auth failure), so this was an advisory same-family pass and NOT independent coverage. Co-Authored-By: Claude Opus 5 (1M context) --- packages/browser/src/readiness.ts | 43 +++++++++++++++++-- packages/browser/src/renderer.ts | 17 ++++++-- packages/browser/test/readiness.test.ts | 42 ++++++++++++++++++ packages/plugin/METRICS.md | 6 +-- packages/plugin/src/metrics.js | 22 ++++++---- packages/plugin/src/resources/RenderQueue.js | 7 ++- .../plugin/src/util/readinessExpectation.js | 2 +- packages/plugin/test/metrics.test.js | 6 ++- 8 files changed, 121 insertions(+), 24 deletions(-) diff --git a/packages/browser/src/readiness.ts b/packages/browser/src/readiness.ts index cb8f86b..e49ecfa 100644 --- a/packages/browser/src/readiness.ts +++ b/packages/browser/src/readiness.ts @@ -185,8 +185,6 @@ export type ReadinessContract = { * (rail counts, image counts): visibility without letting churn hold a render open. */ observe?: ReadinessAssertion[]; - /** Hold the contract true for this long before accepting it. 0 = accept on first sight. */ - stableMs?: number; /** Give up after this long and report what was still false. Clamped by the render budget. */ timeoutMs?: number; /** Sample interval. Defaults to `navigation.domStablePollMs`. */ @@ -468,7 +466,16 @@ export function evaluateContract(payload: { if (a.nonEmptyText === true && typeof a.selector === 'string') { let ok = false; let seen = 0; - const pattern = typeof a.textMatches === 'string' ? new RegExp(a.textMatches as string) : null; + let pattern: RegExp | null = null; + try { + pattern = typeof a.textMatches === 'string' ? new RegExp(a.textMatches as string) : null; + } catch { + // Defence in depth: `validateReadiness` rejects a malformed pattern at config load, so this + // is unreachable through the public API. It stays because the alternative shape — building + // the RegExp outside the loop's try — threw out of the WHOLE evaluator, discarding every + // other clause's result for that tick rather than failing this one. + return { name: assertion.name, ok: false, count: 0 }; + } for (const root of roots) { try { for (const el of root.querySelectorAll(a.selector as string)) { @@ -526,6 +533,18 @@ export function validateReadiness(readiness: unknown): void { if (!Array.isArray(contract.require) || contract.require.length === 0) { throw new Error(`prerender config: readiness contract "${contract.name}" needs at least one require assertion`); } + for (const [field, value] of Object.entries({ + timeoutMs: contract.timeoutMs, + pollMs: contract.pollMs, + quietMs: contract.quietMs, + })) { + if (value === undefined) continue; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error( + `prerender config: readiness contract "${contract.name}" ${field} must be a non-negative number` + ); + } + } if (contract.pathPattern) { try { new RegExp(contract.pathPattern); @@ -535,6 +554,24 @@ export function validateReadiness(readiness: unknown): void { } for (const assertion of [...contract.require, ...(contract.observe ?? [])]) { const a = assertion as Record; + for (const field of ['minCount', 'maxRemaining', 'minContained'] as const) { + const value = a[field]; + if (value === undefined) continue; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error( + `prerender config: readiness assertion "${assertion.name}" ${field} must be a non-negative number` + ); + } + } + // Compiled here so a malformed pattern is a config error rather than a clause that can never + // hold — the same rule `waitFor.pathPattern` follows. + if (typeof a.textMatches === 'string') { + try { + new RegExp(a.textMatches); + } catch { + throw new Error(`prerender config: readiness assertion "${assertion.name}" has an invalid textMatches`); + } + } const forms = [ a.selector && !a.shed && !a.nonEmptyText, a.anyOf, diff --git a/packages/browser/src/renderer.ts b/packages/browser/src/renderer.ts index 18b6dd9..32b90f6 100644 --- a/packages/browser/src/renderer.ts +++ b/packages/browser/src/renderer.ts @@ -748,7 +748,7 @@ const renderer: Renderer = async (page, job) => { */ const observeContract = async ( governing: NonNullable>, - until: { done: boolean } + until: { done: boolean; wake?: (() => void) | undefined } ): Promise => { const started = Date.now(); const pollMs = governing.pollMs ?? config.navigation.domStablePollMs; @@ -774,7 +774,17 @@ const renderer: Renderer = async (page, job) => { firstSatisfiedMs = Date.now() - started; } if (until.done) break; - await new Promise((resolve) => setTimeout(resolve, pollMs)); + // Interruptible, so stopping the observer is PROMPT. A plain sleep meant the settle could be + // finished and still waiting up to a full poll interval for this loop to notice — which + // would make report mode extend the render it exists to leave alone. + await new Promise((resolve) => { + const timer = setTimeout(resolve, pollMs); + until.wake = () => { + clearTimeout(timer); + resolve(undefined); + }; + }); + until.wake = undefined; } // One last look, so the verdict describes the DOM that is about to be serialized rather than @@ -814,7 +824,7 @@ const renderer: Renderer = async (page, job) => { // A contract REPLACES the timer-based settle rather than joining it: scroll once to trip whatever // is lazy, then hold until the page says it is complete. Everything below is what runs when no // contract governs this render, or when one is not satisfied. - const observing = { done: false }; + const observing: { done: boolean; wake?: (() => void) | undefined } = { done: false }; let observer: Promise | null = null; let contractSatisfied = false; let contractScrolled = false; @@ -876,6 +886,7 @@ const renderer: Renderer = async (page, job) => { // report mode cannot change a render's duration. if (observer) { observing.done = true; + observing.wake?.(); await observer; } timings.settle = Date.now() - settleStart; diff --git a/packages/browser/test/readiness.test.ts b/packages/browser/test/readiness.test.ts index d5e13db..ceadc4f 100644 --- a/packages/browser/test/readiness.test.ts +++ b/packages/browser/test/readiness.test.ts @@ -319,3 +319,45 @@ test('report mode still times how long a satisfiable contract took to hold', asy 'it must report WHEN the content arrived, not merely that it did' ); }); + +test('a contract with an unusable number or pattern is rejected at config load', async () => { + // A NaN or negative timeout makes the gate's deadline NaN, the loop never runs, and the contract + // is silently disabled — a config that looks enabled and protects nothing, which is the worst of + // the available outcomes. Same rule `waitFor` already applies to its numerics. + const { mergeConfig } = await import('../dist/config.js'); + const contract = (over: Record) => ({ + readiness: { + onSatisfied: 'quiet', + contracts: [{ name: 'c', require: [{ name: 'x', selector: 'p' }], ...over }], + }, + }); + + assert.throws(() => mergeConfig(contract({ timeoutMs: Number.NaN }) as never), /timeoutMs must be a non-negative/); + assert.throws(() => mergeConfig(contract({ quietMs: -1 }) as never), /quietMs must be a non-negative/); + assert.throws(() => mergeConfig(contract({ pollMs: 'soon' }) as never), /pollMs must be a non-negative/); + assert.doesNotThrow(() => mergeConfig(contract({ timeoutMs: 5000, quietMs: 250 }) as never)); + + // A malformed regex used to fail only at evaluation time, where it threw out of the whole + // evaluator and discarded every other clause's result for that tick. + assert.throws( + () => + mergeConfig({ + readiness: { + onSatisfied: 'quiet', + contracts: [{ name: 'c', require: [{ name: 'x', selector: 'p', nonEmptyText: true, textMatches: '([' }] }], + }, + } as never), + /invalid textMatches/ + ); + + assert.throws( + () => + mergeConfig({ + readiness: { + onSatisfied: 'quiet', + contracts: [{ name: 'c', require: [{ name: 'x', selector: 'p', minCount: -3 }] }], + }, + } as never), + /minCount must be a non-negative/ + ); +}); diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index 2bf039b..f5b94a7 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -129,7 +129,7 @@ reasoning behind it. | `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | | `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | | `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied/rebaselined) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per render so shares read as fractions of throughput) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `rebaseline` (counter) is a URL whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per render. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | | `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | | `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set), `raw_cache` (outcome: `stored`, or the refusal — `not-200`, `staging`, `has-cookie`, `content-type`, `no-store`, `no-body`, `oversize`, `capture-failed`, `write-failed`, `empty` (a 200 with no body — never stored), `capture-busy` (`maxConcurrentCaptures` reached; served, not stored). **Read the refusals, not the successes**: an enabled route that is filling nothing looks exactly like a disabled one unless the reason is recorded. `oversize` climbing means `render.raw.maxBytes` is under the route's real document size; `has-cookie` climbing means the origin is personalizing a route that was assumed shared, which is the one outcome worth an alert). | | `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | @@ -141,7 +141,7 @@ reasoning behind it. | `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | | `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | | `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied/rebaselined) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per render so shares read as fractions of throughput) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `rebaseline` (counter) is a URL whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per render. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | | `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | | `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | | `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | @@ -153,7 +153,7 @@ reasoning behind it. | `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | | `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | | `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied/rebaselined) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | +| `render_readiness` | value | series | contract | per-series | What each page type's completeness contract said about the renders it governed. `verdict` (counter, type = satisfied/unsatisfied, exactly one per render so shares read as fractions of throughput) is the share of renders that finished COMPLETE — the only signal that distinguishes a render missing a widget from a good one, since those renders are 200, non-empty and indexable either way (measured under CPU contention: 8 of 15 renders finished unsatisfied and all 15 reported `outcome=ok`). `unmet` (counter, type = clause name, one row per failing clause so it does NOT sum to renders) names WHICH clause, so a contract that has rotted against a template change reads as one clause failing across every render of its page type rather than as an unexplained slowdown. `shortfall` (counter, type = observation name) is an observation that fell far below what that URL last produced — silent on a first render and after a rebaseline. `rebaseline` (counter) is a URL whose expectation was re-learned after repeated shortfalls, kept OUT of `verdict` so that series keeps summing to one per render. `satisfied_ms` (distribution, type = null) is what `timeoutMs` should be tuned from: a p95 approaching the configured timeout means the contract is being abandoned under load and the optimisation is quietly gone. Emitted only for renders a contract governed. | | `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | | `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_not_modified` — documents the origin answered 304 to, whose entries were never re-parsed and whose prune scan never ran. **A steady ZERO where `sitemap.conditional` is enabled means the origin is not honouring `If-Modified-Since`** and every pass is doing full work, so the refresh frequency should come back down; plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | | `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | diff --git a/packages/plugin/src/metrics.js b/packages/plugin/src/metrics.js index 22f7f7f..e40d50b 100644 --- a/packages/plugin/src/metrics.js +++ b/packages/plugin/src/metrics.js @@ -372,10 +372,13 @@ export const METRICS = Object.freeze({ dimensions: { path: { name: 'series', - values: ['verdict', 'unmet', 'shortfall', 'satisfied_ms'], + values: ['verdict', 'unmet', 'shortfall', 'rebaseline', 'satisfied_ms'], description: - 'verdict = counter of how the contract ended. unmet = counter, one per clause that did not ' + - 'hold. shortfall = counter, one per observation that fell far below this URL\u2019s history. ' + + 'verdict = counter of how the contract ended, EXACTLY ONE PER RENDER so shares read as ' + + 'fractions of render throughput. unmet = counter, one per clause that did not hold. ' + + 'shortfall = counter, one per observation that fell far below this URL\u2019s history. ' + + 'rebaseline = counter, one per URL whose expectation was re-learned after repeated ' + + 'shortfalls — kept OUT of verdict so that series keeps summing to one per render. ' + 'satisfied_ms = distribution of how long the contract took to first hold.', }, method: { @@ -384,12 +387,11 @@ export const METRICS = Object.freeze({ }, type: { name: 'verdict (verdict) / clause (unmet) / observation (shortfall)', - values: ['satisfied', 'unsatisfied', 'rebaselined'], + values: ['satisfied', 'unsatisfied'], description: - 'verdict: satisfied | unsatisfied | rebaselined (the observation shortfalls repeated often ' + - 'enough to be the page\u2019s new shape). unmet/shortfall: the configured clause or ' + - 'observation name, so the enumeration above applies to the verdict series only. Null on ' + - 'satisfied_ms.', + 'verdict: satisfied | unsatisfied. unmet/shortfall: the configured clause or observation ' + + 'name, so the enumeration above applies to the verdict series only. Null on rebaseline ' + + 'and satisfied_ms.', }, }, }), @@ -771,6 +773,10 @@ export const metrics = Object.freeze({ renderReadinessUnmet: (contract, clause) => server.recordAnalytics(true, 'render_readiness', 'unmet', contract, clause), + /** One URL whose expectation was re-learned. Its own series, so `verdict` stays one per render. */ + renderReadinessRebaseline: (contract) => + server.recordAnalytics(true, 'render_readiness', 'rebaseline', contract, null), + /** One observation that fell far below what this URL last produced. */ renderReadinessShortfall: (contract, observation) => server.recordAnalytics(true, 'render_readiness', 'shortfall', contract, observation), diff --git a/packages/plugin/src/resources/RenderQueue.js b/packages/plugin/src/resources/RenderQueue.js index 485447f..891d5c4 100644 --- a/packages/plugin/src/resources/RenderQueue.js +++ b/packages/plugin/src/resources/RenderQueue.js @@ -548,10 +548,9 @@ export class RenderQueue extends Resource { for (const variant of variants) { const readiness = variant.readiness; if (!readiness) continue; // no contract governed this render - metrics.renderReadiness( - readiness.contract, - readiness.rebaselined ? 'rebaselined' : readiness.satisfied ? 'satisfied' : 'unsatisfied' - ); + // Exactly one verdict per variant, so the series sums to renders. A rebaseline is a separate + // series (emitted by the expectation store, which is the only thing that can know). + metrics.renderReadiness(readiness.contract, readiness.satisfied ? 'satisfied' : 'unsatisfied'); // Per CLAUSE, so a contract that has rotted against a template change reads as one clause // failing across every render of its page type rather than as an unexplained slowdown. for (const clause of readiness.unmet ?? []) metrics.renderReadinessUnmet(readiness.contract, clause); diff --git a/packages/plugin/src/util/readinessExpectation.js b/packages/plugin/src/util/readinessExpectation.js index 3a7d2a2..3885dca 100644 --- a/packages/plugin/src/util/readinessExpectation.js +++ b/packages/plugin/src/util/readinessExpectation.js @@ -103,7 +103,7 @@ export const recordReadinessExpectation = async (url, readiness, policy = DEFAUL const verdict = assess(learned, parsed, policy); for (const shortfall of verdict.shortfalls) metrics.renderReadinessShortfall(readiness.contract, shortfall.name); - if (verdict.rebaselined) metrics.renderReadiness(readiness.contract, 'rebaselined'); + if (verdict.rebaselined) metrics.renderReadinessRebaseline(readiness.contract); if (verdict.shortfalls.length) { logger.warn( `Prerender ${url}: rendered fewer than this URL last produced — ` + diff --git a/packages/plugin/test/metrics.test.js b/packages/plugin/test/metrics.test.js index 7b68a4b..d94ff82 100644 --- a/packages/plugin/test/metrics.test.js +++ b/packages/plugin/test/metrics.test.js @@ -336,12 +336,14 @@ test('every render_readiness series and verdict the emitters can produce is decl () => metrics.renderReadinessUnmet('c', 'x'), () => metrics.renderReadinessShortfall('c', 'x'), () => metrics.renderReadinessMs(1, 'c'), + () => metrics.renderReadinessRebaseline('c'), ]) { assert.ok(declared.path.values.includes(emitted(emit).path), 'undeclared series'); } // The verdict enumeration is closed; the clause/observation names in the same slot are not, which - // is why the catalog says the enumeration applies to the verdict series only. - for (const verdict of ['satisfied', 'unsatisfied', 'rebaselined']) { + // is why the catalog says the enumeration applies to the verdict series only. `rebaselined` is + // deliberately NOT a verdict — it rides its own series so verdict keeps summing to one per render. + for (const verdict of ['satisfied', 'unsatisfied']) { assert.ok(declared.type.values.includes(emitted(() => metrics.renderReadiness('c', verdict)).type)); } }); From f2da0971c38d89dd83c53464dc4afc670ba34457 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 18:36:22 -0400 Subject: [PATCH 6/6] fix(browser): validate every readiness timer against the setTimeout ceiling, and refuse a zero poll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo's CI reviewer caught two things the local pass did not, both real. A zero or negative `pollMs` was accepted. The gate and the observer both sleep on it, so zero is a tight loop calling into the page as fast as the event loop allows — a burned core per render. It must be POSITIVE, not merely non-negative. None of the ms fields were checked against 2147483647. Past that, setTimeout fires after 1ms instead of waiting, so an over-large dwell silently becomes NO dwell — exactly the trap `scroll.topSettleMs` is already guarded against in this same config module, which makes the omission a consistency bug as much as a correctness one. Coverage was also too narrow: only the contract-level numbers were validated. `readiness.quietMs`, `readiness.unmetGraceMs`, `expectations.graceMs`, `expectations.tolerance` (0..1) and `expectations.rebaselineAfter` (a positive integer) all reach a timer or a comparison where a wrong value does nothing visible. All of them now fail at config load, through one shared checker so the next field added cannot be forgotten. The reviewer also flagged `textMatches` — validated at load AND caught per-clause in the evaluator as of the previous commit — and suggested validating `contract.stableMs`, which no longer exists: it was documented and never read, and was removed rather than validated. Co-Authored-By: Claude Opus 5 (1M context) --- packages/browser/src/readiness.ts | 51 +++++++++++++++++++------ packages/browser/test/readiness.test.ts | 27 ++++++++++++- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/packages/browser/src/readiness.ts b/packages/browser/src/readiness.ts index e49ecfa..a614381 100644 --- a/packages/browser/src/readiness.ts +++ b/packages/browser/src/readiness.ts @@ -519,6 +519,25 @@ export function evaluateContract(payload: { return { require: payload.require.map(run), observe: payload.observe.map(run), quietMs: quiet, started }; } +/** + * The ceiling `setTimeout` accepts. Past it Node fires the callback after 1ms instead of waiting, so + * an over-large dwell silently becomes NO dwell — the same trap `scroll.topSettleMs` already guards. + * A poll interval that large is nonsense anyway; what matters is that it fails loudly. + */ +const MAX_TIMER_MS = 2147483647; + +/** Every ms-valued readiness field, checked the same way, so none of them can silently misbehave. */ +const checkMs = (label: string, value: unknown, { positive = false } = {}): void => { + if (value === undefined) return; + const bad = + typeof value !== 'number' || !Number.isFinite(value) || value > MAX_TIMER_MS || (positive ? value <= 0 : value < 0); + if (bad) { + throw new Error( + `prerender config: ${label} must be a ${positive ? 'positive' : 'non-negative'} number up to ${MAX_TIMER_MS}` + ); + } +}; + /** Validate contracts at config load, so a broken one cannot first surface inside a render. */ export function validateReadiness(readiness: unknown): void { if (readiness === undefined) return; @@ -527,24 +546,32 @@ export function validateReadiness(readiness: unknown): void { if (cfg.onSatisfied !== undefined && !['report', 'quiet', 'plateau'].includes(cfg.onSatisfied)) { throw new Error("prerender config: readiness.onSatisfied must be 'report', 'quiet' or 'plateau'"); } + checkMs('readiness.quietMs', cfg.quietMs); + checkMs('readiness.unmetGraceMs', cfg.unmetGraceMs); + if (cfg.expectations !== undefined) { + if (typeof cfg.expectations !== 'object' || cfg.expectations === null) { + throw new Error('prerender config: readiness.expectations must be an object'); + } + checkMs('readiness.expectations.graceMs', cfg.expectations.graceMs); + const { tolerance, rebaselineAfter } = cfg.expectations; + if (tolerance !== undefined && (typeof tolerance !== 'number' || !(tolerance >= 0 && tolerance <= 1))) { + throw new Error('prerender config: readiness.expectations.tolerance must be a number between 0 and 1'); + } + if (rebaselineAfter !== undefined && (!Number.isInteger(rebaselineAfter) || rebaselineAfter < 1)) { + throw new Error('prerender config: readiness.expectations.rebaselineAfter must be a positive integer'); + } + } if (!Array.isArray(cfg.contracts)) throw new Error('prerender config: readiness.contracts must be an array'); for (const contract of cfg.contracts) { if (!contract.name) throw new Error('prerender config: every readiness contract needs a name'); if (!Array.isArray(contract.require) || contract.require.length === 0) { throw new Error(`prerender config: readiness contract "${contract.name}" needs at least one require assertion`); } - for (const [field, value] of Object.entries({ - timeoutMs: contract.timeoutMs, - pollMs: contract.pollMs, - quietMs: contract.quietMs, - })) { - if (value === undefined) continue; - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - throw new Error( - `prerender config: readiness contract "${contract.name}" ${field} must be a non-negative number` - ); - } - } + checkMs(`readiness contract "${contract.name}" timeoutMs`, contract.timeoutMs); + checkMs(`readiness contract "${contract.name}" quietMs`, contract.quietMs); + // POSITIVE, not merely non-negative: a zero poll interval is a tight loop calling into the page + // as fast as the event loop allows, which would burn a core per render. + checkMs(`readiness contract "${contract.name}" pollMs`, contract.pollMs, { positive: true }); if (contract.pathPattern) { try { new RegExp(contract.pathPattern); diff --git a/packages/browser/test/readiness.test.ts b/packages/browser/test/readiness.test.ts index ceadc4f..6876c17 100644 --- a/packages/browser/test/readiness.test.ts +++ b/packages/browser/test/readiness.test.ts @@ -334,8 +334,31 @@ test('a contract with an unusable number or pattern is rejected at config load', assert.throws(() => mergeConfig(contract({ timeoutMs: Number.NaN }) as never), /timeoutMs must be a non-negative/); assert.throws(() => mergeConfig(contract({ quietMs: -1 }) as never), /quietMs must be a non-negative/); - assert.throws(() => mergeConfig(contract({ pollMs: 'soon' }) as never), /pollMs must be a non-negative/); - assert.doesNotThrow(() => mergeConfig(contract({ timeoutMs: 5000, quietMs: 250 }) as never)); + assert.throws(() => mergeConfig(contract({ pollMs: 'soon' }) as never), /pollMs must be a positive/); + // A zero poll interval is a tight loop calling into the page as fast as the event loop allows. + assert.throws(() => mergeConfig(contract({ pollMs: 0 }) as never), /pollMs must be a positive/); + // Past the timer ceiling setTimeout fires after 1ms, so an over-large dwell becomes NO dwell — + // the same trap `scroll.topSettleMs` already guards against. + assert.throws(() => mergeConfig(contract({ timeoutMs: 2147483648 }) as never), /up to 2147483647/); + assert.doesNotThrow(() => mergeConfig(contract({ timeoutMs: 5000, quietMs: 250, pollMs: 250 }) as never)); + + // The top-level knobs and the expectation policy are validated too — every one of them ends up in + // a setTimeout or a comparison that silently does nothing when it is wrong. + const top = (over: Record) => ({ + readiness: { onSatisfied: 'quiet', contracts: [{ name: 'c', require: [{ name: 'x', selector: 'p' }] }], ...over }, + }); + assert.throws(() => mergeConfig(top({ quietMs: -5 }) as never), /readiness.quietMs/); + assert.throws(() => mergeConfig(top({ unmetGraceMs: 2147483648 }) as never), /readiness.unmetGraceMs/); + assert.throws(() => mergeConfig(top({ expectations: { tolerance: 1.5 } }) as never), /tolerance must be a number/); + assert.throws( + () => mergeConfig(top({ expectations: { rebaselineAfter: 0 } }) as never), + /rebaselineAfter must be a positive integer/ + ); + assert.doesNotThrow(() => + mergeConfig( + top({ quietMs: 250, unmetGraceMs: 1000, expectations: { tolerance: 0.5, rebaselineAfter: 3 } }) as never + ) + ); // A malformed regex used to fail only at evaluation time, where it threw out of the whole // evaluator and discarded every other clause's result for that tick.