Render a newly declared sitemap URL soon, not a full interval later (v0.74.0)

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender",
-	"version": "0.73.0",
+	"version": "0.74.0",
 	"type": "module",
 	"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",
 	"license": "Apache-2.0",

packages/plugin/src/configSchema.js

@@ -1732,6 +1732,45 @@ export const configSchema = group('Prerender plugin configuration.', {
 			{ min: 0 }
 		),
 		failedCap: option(100, 'Max failed-entry samples carried back in a refresh result.', { min: 0 }),
+		newTargets: group(
+			'How soon a URL the sitemap has just DECLARED gets its first render.\n\n' +
+				'Without this, a newly created target takes `getInitialRenderTime`, which jitters the first ' +
+				'render across the target’s WHOLE render interval — `hash(url) % interval`. That jitter ' +
+				'exists for a real reason (the first ingest of a large sitemap must not stampede the queue), ' +
+				'but it is sized for bulk population and applies just as hard to the handful of genuinely new ' +
+				'URLs a mature corpus gains each day: on a 48h cadence a product published this morning can ' +
+				'wait two days to be rendered once, while the sitemap has been telling us about it the whole ' +
+				'time. A declaration is the strongest signal a site gives that a URL matters.\n\n' +
+				'So the first render is jittered across `window` instead of the interval, and only for the ' +
+				'first `maxPerRun` creates in a walk. The cap is what keeps the bulk case safe: a first ' +
+				'ingest creating hundreds of thousands of targets exceeds it immediately and everything past ' +
+				'it falls back to full-interval jitter, which is exactly the old behaviour. Steady-state ' +
+				'churn (tens to hundreds a day on a real corpus) never comes close to the cap.\n\n' +
+				'Only the FIRST render moves. The target’s cadence is untouched — `effectiveInterval` is ' +
+				'still the route/stored interval, so every render after this one is on the normal schedule.',
+			{
+				window: option(
+					15 * MINUTE,
+					'Jitter window for a newly declared target’s first render. Small values approximate ' +
+						'"immediately" while still spreading a batch across minutes rather than firing it into ' +
+						'one. `0` disables the fast path entirely and restores full-interval jitter.\n\n' +
+						'BELOW ~2 MINUTES IT STOPS SPREADING. `getInitialRenderTime` floors to the minute, so a ' +
+						'window under 60,000ms collapses every create in a walk onto ONE minute — the stampede ' +
+						'this is jittered to avoid, arrived at by asking for less jitter. Capped at 2147483647 ' +
+						'for the same reason `sweepInterval` is: a larger delay is not "effectively never", it ' +
+						'overflows the signed 32-bit timer and fires immediately.\n\n' +
+						'A window WIDER than the route’s own `renderInterval` is ignored — the fast path would be ' +
+						'slower than the jitter it replaces — and does not count as `createdSoon`.',
+					{ unit: 'ms', min: 0, max: 2147483647 }
+				),
+				maxPerRun: option(
+					5000,
+					'Creates per walk that may take the fast path. Past this, new targets fall back to ' +
+						'full-interval jitter — the bulk-population guard.',
+					{ min: 0 }
+				),
+			}
+		),
 		conditional: group(
 			'Conditional sitemap fetching: send `If-Modified-Since` and skip the whole reconcile for a ' +
 				'document the origin answers 304 to.\n\n' +

packages/plugin/src/resources/Sitemap.js

@@ -3,7 +3,7 @@ import { metrics } from '../metrics.js';
 import { describeError } from '../util/errors.js';
 import { Target } from './Target.js';
 import { anyRouteDeparts, classifyUrl, PASSTHROUGH, PRERENDER, UNCLASSIFIED } from '../util/routeClass.js';
-import { currentMinuteMs, epochMsOf, getNextSitemapRefreshTime } from '../util/time.js';
+import { currentMinuteMs, epochMsOf, getInitialRenderTime, getNextSitemapRefreshTime } from '../util/time.js';
 import { parseSitemap, partitionSitemapEntries } from '../util/sitemap.js';
 import { actionForExisting, canSkipLookup, createRefreshRun, TargetAction } from '../util/sitemapRun.js';
 import { configuredStagingIp, dispatcherFor } from '../util/upstream.js';
@@ -364,6 +364,7 @@ const progressFields = (snapshot) => ({
 	created: snapshot.created,
 	updated: snapshot.updated,
 	skipped: snapshot.skipped,
+	createdSoon: snapshot.createdSoon,
 	notModified: snapshot.notModified,
 	duplicates: snapshot.duplicates,
 	deferred: snapshot.deferred,
@@ -410,15 +411,17 @@ async function runTrackedRefresh(rootUrl, options) {
 
 		logger.info(
 			`[prerender] Sitemap refresh for ${rootUrl} finished: ${result.sitemapsProcessed} sitemaps ` +
-				`(${result.notModified} not modified), ${result.created} created, ${result.updated} re-attributed, ` +
-				`${result.skipped} unchanged, ${result.removed} unlinked, ${result.failed.length} failed`
+				`(${result.notModified} not modified), ${result.created} created ` +
+				`(${result.createdSoon} fast-path), ${result.updated} re-attributed, ${result.skipped} unchanged, ` +
+				`${result.removed} unlinked, ${result.failed.length} failed`
 		);
 
 		// The same numbers as METRICS — corpus churn and walk health, previously log-only.
 		// Guarded: a gauge must never cost the run its completed progress row.
 		try {
 			metrics.sitemapRun(result.sitemapsProcessed, 'sitemaps');
 			metrics.sitemapRun(result.created, 'created');
+			metrics.sitemapRun(result.createdSoon, 'created_soon');
 			metrics.sitemapRun(result.updated, 'updated');
 			metrics.sitemapRun(result.skipped, 'skipped');
 			metrics.sitemapRun(result.notModified, 'not_modified');
@@ -596,6 +599,9 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate,
 	});
 	run.addRemoved(departed);
 
+	// Read once per child rather than per entry: config is a live object and this is the hot loop.
+	const { window: newTargetWindow, maxPerRun: newTargetCap } = config.sitemap.newTargets;
+
 	let inflight = [];
 	let considered = 0;
 
@@ -645,12 +651,36 @@ async function reconcileSitemapEntries(sitemapUrl, latestSitemap, { revalidate,
 				inflight.push(Target.patch(cacheUrl, { sitemapUrl, renderInterval }));
 				break;
 
-			case TargetAction.CREATE:
-				// No explicit time, so Target.put jitters the first render across the
-				// interval — bulk sitemap population must not stampede the queue.
+			case TargetAction.CREATE: {
+				// A DECLARATION IS A STRONG SIGNAL, so a newly listed URL does not wait out a full
+				// interval of jitter to be rendered once. `getInitialRenderTime` spreads the first
+				// render across `hash(url) % interval`, which is sized for the first ingest of a large
+				// sitemap and applies just as hard to the handful of genuinely new URLs a mature corpus
+				// gains each day — on a 48h cadence, up to two days.
+				//
+				// The window is jittered rather than set to "now" for the same reason the interval jitter
+				// exists: a batch of creates must land across minutes, not in one. And the cap is what
+				// keeps bulk population safe — past `maxPerRun` this falls back to the old full-interval
+				// jitter by passing no explicit time at all, so a first ingest behaves exactly as before.
+				//
+				// Only the FIRST render moves: `Target.put` still files `effectiveInterval` from the
+				// route/stored cadence, so every render after this one is on the normal schedule.
 				run.count('created');
-				inflight.push(Target.put(cacheUrl, { renderInterval, sitemapUrl }));
+				// `< renderInterval`, because a window WIDER than the route's own cadence makes the "fast"
+				// path slower than the jitter it replaces — and would still count as `createdSoon`, so the
+				// metric would report an acceleration that did not happen. Not reachable on a corpus whose
+				// shortest interval is a day, but the guard is free and the metric has to stay honest.
+				const fast = newTargetWindow > 0 && newTargetWindow < renderInterval && run.fastPathTaken() < newTargetCap;
+				if (fast) run.count('createdSoon');
+				inflight.push(
+					Target.put(cacheUrl, {
+						renderInterval,
+						sitemapUrl,
+						...(fast ? { nextRenderTime: getInitialRenderTime(cacheUrl, newTargetWindow) } : {}),
+					})
+				);
 				break;
+			}
 
 			case TargetAction.RENDER:
 				run.count('created');

packages/plugin/src/util/sitemapRun.js

@@ -101,6 +101,11 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar
 		// worth seeing, because nothing else in the system would ever mention it.
 		duplicates: 0,
 		deferred: 0,
+		// Creates that took the new-target fast path (first render inside
+		// `sitemap.newTargets.window` rather than a full interval of jitter). `created` counts every
+		// new target; this counts the subset that was not capped, so `created - createdSoon` is the
+		// bulk-population overflow that fell back to the old behaviour.
+		createdSoon: 0,
 		removed: 0,
 		// Documents the origin answered 304 to, so their entries were never re-parsed and their
 		// prune scan never ran. On a healthy corpus this is most of every pass between rebuilds;
@@ -152,6 +157,14 @@ export const createRefreshRun = ({ removedSampleCap = 20, failedCap = 100, depar
 			}
 		},
 
+		/**
+		 * How many creates have taken the fast path in this WALK — the cap is per walk, not per
+		 * child, so an index index fanning out to 17 children cannot multiply it by 17.
+		 */
+		fastPathTaken() {
+			return totals.createdSoon;
+		},
+
 		/** The departed URLs to re-read once the walk has finished. */
 		departureCandidates() {
 			return departure.candidates;

packages/plugin/test/sitemapRun.test.js

@@ -210,3 +210,28 @@ test('a snapshot is a copy, so persisting it mid-walk cannot be mutated afterwar
 	assert.equal(first.created, 0);
 	assert.equal(run.snapshot().failed.length, 2);
 });
+
+// ---- new-target fast path ----
+
+test('the fast-path cap is per WALK, not per child sitemap', () => {
+	// An index fanning out to 17 children must not multiply the cap by 17: `fastPathTaken` reads the
+	// walk's own running total, which is what the CREATE branch compares against maxPerRun.
+	const run = createRefreshRun();
+	assert.equal(run.fastPathTaken(), 0);
+	run.count('createdSoon');
+	run.count('createdSoon');
+	assert.equal(run.fastPathTaken(), 2, 'the count carries across children within one walk');
+	assert.equal(run.snapshot().createdSoon, 2);
+});
+
+test('createdSoon is a SUBSET of created, so the overflow is readable', () => {
+	// `created - createdSoon` is the bulk-population overflow that fell back to full-interval
+	// jitter — the number that says whether the cap bound.
+	const run = createRefreshRun();
+	for (let i = 0; i < 5; i++) run.count('created');
+	for (let i = 0; i < 2; i++) run.count('createdSoon');
+	const s = run.snapshot();
+	assert.equal(s.created, 5);
+	assert.equal(s.createdSoon, 2);
+	assert.equal(s.created - s.createdSoon, 3, 'three creates fell back to the old behaviour');
+});

packages/plugin/test/time.test.js

@@ -171,3 +171,34 @@ test('epochMsOf returns NaN for an unparseable value', () => {
 test('epochMsOf keeps epoch 0 distinguishable from absent', () => {
 	assert.equal(epochMsOf(0), 0);
 });
+
+test('a short window bounds a new target’s first render, where the interval does not', () => {
+	// The new-target fast path (sitemap.newTargets.window) relies entirely on this: the same helper,
+	// handed a 15-minute window instead of the target’s cadence, confines the first render to
+	// minutes. Without it a newly declared URL waits `hash(url) % interval` — up to the whole
+	// interval, which on a 48h PDP cadence is two days.
+	const MIN = 60 * 1000;
+	const WINDOW = 15 * MIN;
+	const INTERVAL = 48 * 60 * MIN;
+	const urls = Array.from({ length: 200 }, (_, i) => `https://example.com/product/prd-${i}/thing.jsp`);
+
+	const before = Date.now();
+	const windowed = urls.map((u) => getInitialRenderTime(u, WINDOW));
+	const intervalled = urls.map((u) => getInitialRenderTime(u, INTERVAL));
+	const after = Date.now();
+
+	for (const t of windowed) {
+		assert.ok(t >= before - 60_000, `${t} is before now`);
+		assert.ok(t <= after + WINDOW, `${t} escaped the ${WINDOW}ms window`);
+	}
+
+	// And it is still JITTERED, not a single instant — a batch of creates must land across the
+	// window rather than all in one minute, which is the whole reason this is not simply "now".
+	assert.ok(new Set(windowed).size > 5, `expected spread across the window, got ${new Set(windowed).size} distinct`);
+
+	// The contrast the feature exists for: the interval version reaches far beyond the window.
+	assert.ok(
+		Math.max(...intervalled) > after + WINDOW * 10,
+		'precondition: full-interval jitter spreads far past a short window'
+	);
+});