From 554d4551bd1400624532afa4454688f403a729bf Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 17 Sep 2026 19:52:21 -0400 Subject: [PATCH 1/3] feat(plugin): render a newly declared sitemap URL soon, not a full interval later; v0.74.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sitemap CREATE took `Target.put` with no explicit time, so the first render landed at `getInitialRenderTime` = `hash(url) % interval`: jitter across the target's WHOLE render interval. On a 48h PDP cadence a product published this morning can wait two days to be rendered once, while the sitemap has been declaring it the whole time. A declaration is the strongest signal a site gives that a URL matters. That jitter exists for a real reason — the first ingest of a large sitemap must not stampede the queue — but it is sized for bulk population and applies just as hard to the handful of genuinely new URLs a mature corpus gains each day (tens to hundreds). New `sitemap.newTargets`: window (15m) jitter window for a new target's FIRST render. Jittered rather than set to "now" for exactly the reason the interval jitter exists: a batch must land across minutes, not in one. 0 disables and restores the old behaviour. maxPerRun (5000) creates per WALK that may take the fast path; past it, new targets fall back to full-interval jitter. This is the bulk-population guard — a first ingest creating hundreds of thousands exceeds it immediately and behaves exactly as before. ONLY THE FIRST RENDER MOVES. `Target.put` still files `effectiveInterval` from the route/stored cadence, so every render after this one is on the normal schedule. The cap is per WALK, not per child: an index fanning out to 17 children must not multiply it by 17, so it reads the run's own running total (`fastPathTaken`). Reported as `createdSoon` on the result, the progress row and the finish log, and as the `sitemap_created_soon` metric — `created - created_soon` is the overflow that fell back, which is the number that says whether the cap bound. Co-Authored-By: Claude Opus 5 (1M context) --- package-lock.json | 2 +- packages/plugin/METRICS.md | 22 +++++++------- packages/plugin/package.json | 2 +- packages/plugin/src/configSchema.js | 32 ++++++++++++++++++++ packages/plugin/src/resources/Sitemap.js | 38 ++++++++++++++++++++---- packages/plugin/src/util/sitemapRun.js | 13 ++++++++ packages/plugin/test/sitemapRun.test.js | 25 ++++++++++++++++ packages/plugin/test/time.test.js | 31 +++++++++++++++++++ 8 files changed, 146 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index f3c5b61..74b1e4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.72.0", + "version": "0.74.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index df31f71..b117c84 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -121,17 +121,17 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi One-line summaries; `src/metrics.js` carries the full description of every dimension value and the reasoning behind it. -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_created_soon` — the subset of `created` that took the new-target fast path (`sitemap.newTargets`), so `created - created_soon` is the bulk-population overflow that fell back to full-interval jitter. A `created_soon` that is persistently well below `created` means `newTargets.maxPerRun` is binding, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: diff --git a/packages/plugin/package.json b/packages/plugin/package.json index c9f114b..22ee3e3 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.72.0", + "version": "0.74.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 7209a9a..2485d28 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -1732,6 +1732,38 @@ export const configSchema = group('Prerender plugin configuration.', { { min: 0 } ), failedCap: option(100, 'Max failed-entry samples carried back in a refresh result.', { min: 0 }), + newTargets: group( + 'How soon a URL the sitemap has just DECLARED gets its first render.\n\n' + + 'Without this, a newly created target takes `getInitialRenderTime`, which jitters the first ' + + 'render across the target’s WHOLE render interval — `hash(url) % interval`. That jitter ' + + 'exists for a real reason (the first ingest of a large sitemap must not stampede the queue), ' + + 'but it is sized for bulk population and applies just as hard to the handful of genuinely new ' + + 'URLs a mature corpus gains each day: on a 48h cadence a product published this morning can ' + + 'wait two days to be rendered once, while the sitemap has been telling us about it the whole ' + + 'time. A declaration is the strongest signal a site gives that a URL matters.\n\n' + + 'So the first render is jittered across `window` instead of the interval, and only for the ' + + 'first `maxPerRun` creates in a walk. The cap is what keeps the bulk case safe: a first ' + + 'ingest creating hundreds of thousands of targets exceeds it immediately and everything past ' + + 'it falls back to full-interval jitter, which is exactly the old behaviour. Steady-state ' + + 'churn (tens to hundreds a day on a real corpus) never comes close to the cap.\n\n' + + 'Only the FIRST render moves. The target’s cadence is untouched — `effectiveInterval` is ' + + 'still the route/stored interval, so every render after this one is on the normal schedule.', + { + window: option( + 15 * MINUTE, + 'Jitter window for a newly declared target’s first render. Small values approximate ' + + '"immediately" while still spreading a batch across minutes rather than firing it into ' + + 'one. `0` disables the fast path entirely and restores full-interval jitter.', + { unit: 'ms', min: 0 } + ), + maxPerRun: option( + 5000, + 'Creates per walk that may take the fast path. Past this, new targets fall back to ' + + 'full-interval jitter — the bulk-population guard.', + { min: 0 } + ), + } + ), departure: group( 'What a refresh does about URLs that LEAVE a sitemap, beyond unlinking them. The action is ' + 'declared PER ROUTE (`ingress.routes[].departureAction`); this group bounds and observes it, ' + diff --git a/packages/plugin/src/resources/Sitemap.js b/packages/plugin/src/resources/Sitemap.js index 539edf5..34d6801 100644 --- a/packages/plugin/src/resources/Sitemap.js +++ b/packages/plugin/src/resources/Sitemap.js @@ -3,7 +3,7 @@ import { metrics } from '../metrics.js'; import { describeError } from '../util/errors.js'; import { Target } from './Target.js'; import { anyRouteDeparts, classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/routeClass.js'; -import { currentMinuteMs, epochMsOf, getNextSitemapRefreshTime } from '../util/time.js'; +import { currentMinuteMs, epochMsOf, getInitialRenderTime, getNextSitemapRefreshTime } from '../util/time.js'; import { parseSitemap, partitionSitemapEntries } from '../util/sitemap.js'; import { actionForExisting, canSkipLookup, createRefreshRun, TargetAction } from '../util/sitemapRun.js'; import { configuredStagingIp, dispatcherFor } from '../util/upstream.js'; @@ -363,6 +363,7 @@ const progressFields = (snapshot) => ({ created: snapshot.created, updated: snapshot.updated, skipped: snapshot.skipped, + createdSoon: snapshot.createdSoon, duplicates: snapshot.duplicates, deferred: snapshot.deferred, removed: snapshot.removed, @@ -408,7 +409,8 @@ async function runTrackedRefresh(rootUrl, options) { logger.info( `[prerender] Sitemap refresh for ${rootUrl} finished: ${result.sitemapsProcessed} sitemaps, ` + - `${result.created} created, ${result.updated} re-attributed, ${result.skipped} unchanged, ` + + `${result.created} created (${result.createdSoon} fast-path), ${result.updated} re-attributed, ` + + `${result.skipped} unchanged, ` + `${result.removed} unlinked, ${result.failed.length} failed` ); @@ -417,6 +419,7 @@ async function runTrackedRefresh(rootUrl, options) { try { metrics.sitemapRun(result.sitemapsProcessed, 'sitemaps'); metrics.sitemapRun(result.created, 'created'); + metrics.sitemapRun(result.createdSoon, 'created_soon'); metrics.sitemapRun(result.updated, 'updated'); metrics.sitemapRun(result.skipped, 'skipped'); metrics.sitemapRun(result.removed, 'removed'); @@ -565,6 +568,9 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate, }); run.addRemoved(departed); + // Read once per child rather than per entry: config is a live object and this is the hot loop. + const { window: newTargetWindow, maxPerRun: newTargetCap } = config.sitemap.newTargets; + let inflight = []; let considered = 0; @@ -614,12 +620,32 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate, inflight.push(Target.patch(cacheUrl, { sitemapUrl, renderInterval })); break; - case TargetAction.CREATE: - // No explicit time, so Target.put jitters the first render across the - // interval — bulk sitemap population must not stampede the queue. + case TargetAction.CREATE: { + // A DECLARATION IS A STRONG SIGNAL, so a newly listed URL does not wait out a full + // interval of jitter to be rendered once. `getInitialRenderTime` spreads the first + // render across `hash(url) % interval`, which is sized for the first ingest of a large + // sitemap and applies just as hard to the handful of genuinely new URLs a mature corpus + // gains each day — on a 48h cadence, up to two days. + // + // The window is jittered rather than set to "now" for the same reason the interval jitter + // exists: a batch of creates must land across minutes, not in one. And the cap is what + // keeps bulk population safe — past `maxPerRun` this falls back to the old full-interval + // jitter by passing no explicit time at all, so a first ingest behaves exactly as before. + // + // Only the FIRST render moves: `Target.put` still files `effectiveInterval` from the + // route/stored cadence, so every render after this one is on the normal schedule. run.count('created'); - inflight.push(Target.put(cacheUrl, { renderInterval, sitemapUrl })); + const fast = newTargetWindow > 0 && run.fastPathTaken() < newTargetCap; + if (fast) run.count('createdSoon'); + inflight.push( + Target.put(cacheUrl, { + renderInterval, + sitemapUrl, + ...(fast ? { nextRenderTime: getInitialRenderTime(cacheUrl, newTargetWindow) } : {}), + }) + ); break; + } case TargetAction.RENDER: run.count('created'); diff --git a/packages/plugin/src/util/sitemapRun.js b/packages/plugin/src/util/sitemapRun.js index f6b0290..19c1bb0 100644 --- a/packages/plugin/src/util/sitemapRun.js +++ b/packages/plugin/src/util/sitemapRun.js @@ -101,6 +101,11 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar // worth seeing, because nothing else in the system would ever mention it. duplicates: 0, deferred: 0, + // Creates that took the new-target fast path (first render inside + // `sitemap.newTargets.window` rather than a full interval of jitter). `created` counts every + // new target; this counts the subset that was not capped, so `created - createdSoon` is the + // bulk-population overflow that fell back to the old behaviour. + createdSoon: 0, removed: 0, sitemapsProcessed: 0, sitemapsDiscovered: 0, @@ -147,6 +152,14 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar } }, + /** + * How many creates have taken the fast path in this WALK — the cap is per walk, not per + * child, so an index index fanning out to 17 children cannot multiply it by 17. + */ + fastPathTaken() { + return totals.createdSoon; + }, + /** The departed URLs to re-read once the walk has finished. */ departureCandidates() { return departure.candidates; diff --git a/packages/plugin/test/sitemapRun.test.js b/packages/plugin/test/sitemapRun.test.js index f9931c0..09c7478 100644 --- a/packages/plugin/test/sitemapRun.test.js +++ b/packages/plugin/test/sitemapRun.test.js @@ -210,3 +210,28 @@ test('a snapshot is a copy, so persisting it mid-walk cannot be mutated afterwar assert.equal(first.created, 0); assert.equal(run.snapshot().failed.length, 2); }); + +// ---- new-target fast path ---- + +test('the fast-path cap is per WALK, not per child sitemap', () => { + // An index fanning out to 17 children must not multiply the cap by 17: `fastPathTaken` reads the + // walk's own running total, which is what the CREATE branch compares against maxPerRun. + const run = createRefreshRun(); + assert.equal(run.fastPathTaken(), 0); + run.count('createdSoon'); + run.count('createdSoon'); + assert.equal(run.fastPathTaken(), 2, 'the count carries across children within one walk'); + assert.equal(run.snapshot().createdSoon, 2); +}); + +test('createdSoon is a SUBSET of created, so the overflow is readable', () => { + // `created - createdSoon` is the bulk-population overflow that fell back to full-interval + // jitter — the number that says whether the cap bound. + const run = createRefreshRun(); + for (let i = 0; i < 5; i++) run.count('created'); + for (let i = 0; i < 2; i++) run.count('createdSoon'); + const s = run.snapshot(); + assert.equal(s.created, 5); + assert.equal(s.createdSoon, 2); + assert.equal(s.created - s.createdSoon, 3, 'three creates fell back to the old behaviour'); +}); diff --git a/packages/plugin/test/time.test.js b/packages/plugin/test/time.test.js index 9eeb80d..b2dc822 100644 --- a/packages/plugin/test/time.test.js +++ b/packages/plugin/test/time.test.js @@ -171,3 +171,34 @@ test('epochMsOf returns NaN for an unparseable value', () => { test('epochMsOf keeps epoch 0 distinguishable from absent', () => { assert.equal(epochMsOf(0), 0); }); + +test('a short window bounds a new target’s first render, where the interval does not', () => { + // The new-target fast path (sitemap.newTargets.window) relies entirely on this: the same helper, + // handed a 15-minute window instead of the target’s cadence, confines the first render to + // minutes. Without it a newly declared URL waits `hash(url) % interval` — up to the whole + // interval, which on a 48h PDP cadence is two days. + const MIN = 60 * 1000; + const WINDOW = 15 * MIN; + const INTERVAL = 48 * 60 * MIN; + const urls = Array.from({ length: 200 }, (_, i) => `https://example.com/product/prd-${i}/thing.jsp`); + + const before = Date.now(); + const windowed = urls.map((u) => getInitialRenderTime(u, WINDOW)); + const intervalled = urls.map((u) => getInitialRenderTime(u, INTERVAL)); + const after = Date.now(); + + for (const t of windowed) { + assert.ok(t >= before - 60_000, `${t} is before now`); + assert.ok(t <= after + WINDOW, `${t} escaped the ${WINDOW}ms window`); + } + + // And it is still JITTERED, not a single instant — a batch of creates must land across the + // window rather than all in one minute, which is the whole reason this is not simply "now". + assert.ok(new Set(windowed).size > 5, `expected spread across the window, got ${new Set(windowed).size} distinct`); + + // The contrast the feature exists for: the interval version reaches far beyond the window. + assert.ok( + Math.max(...intervalled) > after + WINDOW * 10, + 'precondition: full-interval jitter spreads far past a short window' + ); +}); From 6fef6137c3fd2cb48539ff603ee02210e2763c8a Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 16:18:24 -0400 Subject: [PATCH 2/3] fix(plugin): ignore a newTarget window wider than the route's own interval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review guards on the fast path. A `window` WIDER than the route's `renderInterval` made the fast path SLOWER than the jitter it replaces, while still counting as `createdSoon` — so the metric would report an acceleration that did not happen. Not reachable on a corpus whose shortest interval is a day, but the guard is free and the metric has to stay honest. And the option now says what happens below ~2 minutes: `getInitialRenderTime` floors to the minute, so a window under 60s collapses every create in a walk onto ONE minute — the stampede the jitter exists to avoid, reached by asking for less jitter. Capped at 2147483647 for the same reason sweepInterval is: a larger delay is not 'effectively never', it overflows the signed 32-bit timer and fires immediately. Co-Authored-By: Claude Opus 5 (1M context) --- packages/plugin/src/configSchema.js | 11 +++++++++-- packages/plugin/src/resources/Sitemap.js | 6 +++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 2485d28..a183955 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -1753,8 +1753,15 @@ export const configSchema = group('Prerender plugin configuration.', { 15 * MINUTE, 'Jitter window for a newly declared target’s first render. Small values approximate ' + '"immediately" while still spreading a batch across minutes rather than firing it into ' + - 'one. `0` disables the fast path entirely and restores full-interval jitter.', - { unit: 'ms', min: 0 } + 'one. `0` disables the fast path entirely and restores full-interval jitter.\n\n' + + 'BELOW ~2 MINUTES IT STOPS SPREADING. `getInitialRenderTime` floors to the minute, so a ' + + 'window under 60,000ms collapses every create in a walk onto ONE minute — the stampede ' + + 'this is jittered to avoid, arrived at by asking for less jitter. Capped at 2147483647 ' + + 'for the same reason `sweepInterval` is: a larger delay is not "effectively never", it ' + + 'overflows the signed 32-bit timer and fires immediately.\n\n' + + 'A window WIDER than the route’s own `renderInterval` is ignored — the fast path would be ' + + 'slower than the jitter it replaces — and does not count as `createdSoon`.', + { unit: 'ms', min: 0, max: 2147483647 } ), maxPerRun: option( 5000, diff --git a/packages/plugin/src/resources/Sitemap.js b/packages/plugin/src/resources/Sitemap.js index 34d6801..725274c 100644 --- a/packages/plugin/src/resources/Sitemap.js +++ b/packages/plugin/src/resources/Sitemap.js @@ -635,7 +635,11 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate, // Only the FIRST render moves: `Target.put` still files `effectiveInterval` from the // route/stored cadence, so every render after this one is on the normal schedule. run.count('created'); - const fast = newTargetWindow > 0 && run.fastPathTaken() < newTargetCap; + // `< renderInterval`, because a window WIDER than the route's own cadence makes the "fast" + // path slower than the jitter it replaces — and would still count as `createdSoon`, so the + // metric would report an acceleration that did not happen. Not reachable on a corpus whose + // shortest interval is a day, but the guard is free and the metric has to stay honest. + const fast = newTargetWindow > 0 && newTargetWindow < renderInterval && run.fastPathTaken() < newTargetCap; if (fast) run.count('createdSoon'); inflight.push( Target.put(cacheUrl, { From 3174a3100f5677d4d29c3b78e26a28513152ea17 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 18 Sep 2026 16:20:48 -0400 Subject: [PATCH 3/3] chore: regenerate the lockfile after the merge resolve --- package-lock.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 753d424..74b1e4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,11 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", -<<<<<<< HEAD "version": "0.74.0", -======= - "version": "0.73.0", ->>>>>>> origin/main "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9",