diff --git a/package-lock.json b/package-lock.json index f3c5b61..1da42fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.72.0", + "version": "0.75.0", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index df31f71..afdaf41 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -121,17 +121,17 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi One-line summaries; `src/metrics.js` carries the full description of every dimension value and the reasoning behind it. -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue during the pass — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain rate is behind the detection rate and changes are being deferred for want of QUEUE rather than of budget; those two are indistinguishable in `probe_deferred` alone), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy, one sample per device variant — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — and a result is one URL, every device in it, since v0.66.0 — the render-failure alert). | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*` (refresh-run counters: sitemaps/created/updated/skipped/removed/failed, plus `sitemap_departure_*` — one series per outcome of the post-walk sitemap-departure check, `departure_render`/`departure_expire` for actions taken, `departure_would_render`/`departure_would_expire` under `sitemap.departure.dryRun`, and `departure_reattached`/`departure_suppressed`/`departure_route_opted_out`/`departure_target_gone`/`departure_capped` for the candidates nothing happened to. **`departure_reattached` is the one to watch**: it counts URLs that only LOOKED departed because they shifted across a paginated sitemap's child boundary, so a large share means the corpus is shearing and the raw `sitemap_removed` count is not a departure count. `departure_capped` means `maxActions` bound and some departed URLs were left for the next walk), `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope — including the cross-node outcomes `forwarded`/`forward-failed`; `forwarded` means this node handed the heal to the key's owner, which counts its OWN verdict in this same series, so the two are deliberately not double-counted), `page_verification` (outcome: `written`/`read-error`/`write-error` — per-page invalidation exemptions being recorded; the exemptions actually GRANTED are `bot_serve` cacheStatus `verified`, not this), `probe_*` (change-probe pass counters: probed/seeded/rebaselined/changed/triggered/deferred/failed per pass, plus `probe_canary_trip` and `probe_invalidated`; `probe_rebaselined` counts URLs whose stored baseline was taken under a different rule fingerprint and were re-seeded without comparison — expect one pass of them after a rule edit, and treat a steady count as a rule that keeps changing; `probe_changed`/`probe_probed` is the measured change rate, a rising `probe_failed` share is the endpoint-changed-shape alarm), `discovery_gated` (gate, bot: cacheable misses the discovery gate held out of target creation — the corpus growth being prevented, not denied mints), `probe_fresh` (probes skipped because a baseline was younger than `reprobeAfter` — the work a restarted sweep skipped), `probe_throttled` (probes the origin refused with pushback — **alert on this**: it is the only signal that the probe is loading an origin that cannot take it), `probe_unreadable` (registry rows whose key failed to decode, skipped by the sweep's walk — a nonzero count means the table holds rows the application layer cannot address; escalate to the database layer), `probe_page_mismatch` (cached pages that disagreed with the origin — the round-trip-blindness class `pageCheck` catches; a rising share means renders are landing on transient states, and each one is a served page carrying wrong price/availability until it re-renders), `probe_trigger_queue_depth` (high-water depth of the trigger queue — the queue OUTLIVES the pass, so this is the peak of a drain that carries across passes, and it is also published on the sweep heartbeat so the queue is visible WHILE a pass runs — triggers are submitted to a bounded queue that drains beside the walk, so a value steadily at `changeProbe.trigger.maxPending` means the drain has fallen far behind detection and submissions are being REFUSED — an overload alarm, not a routine outcome, since `maxTriggersPerSweep` now defaults to 0 (no ceiling) and nothing is deferred in normal operation. Raise `trigger.ratePerSecond` rather than `maxPending` if it sits steadily close), `probe_cycle_behind` (CONTINUOUS MODE: batches that needed more than `ratePerSecond` to hit `cycleTarget` — the pass is flat out against its agreed origin ceiling and still losing ground. **Alert on a sustained count**: it is the explicit replacement for the interval model's silently skipped pass, and it means the corpus has outgrown the rate, so either `cycleTarget` is too ambitious or the ceiling needs renegotiating. Zero in interval mode, where no target is set). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `claim_granted` (per claim, method = ready/index), `ready_sweep_ms` (per sweep, method = complete/capped), `ready_published`, `ready_cadence` (per sweep, method = carried/resolved), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: diff --git a/packages/plugin/package.json b/packages/plugin/package.json index c9f114b..61fcd0c 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.72.0", + "version": "0.75.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 7209a9a..8be760a 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -1200,21 +1200,33 @@ export const configSchema = group('Prerender plugin configuration.', { ), concurrency: option(4, 'Triggers in flight at once.', { min: 1 }), maxPending: option( - 5000, - 'Queue depth before submissions are refused and counted as `deferred`. Bounds memory ' + - 'across a pass that can detect hundreds of thousands of changes; it is NOT the ' + - 'per-pass budget, which stays `maxTriggersPerSweep`.', + 50000, + 'Queue depth before submissions are refused. THE ONLY REMAINING BOUND once ' + + '`maxTriggersPerSweep` is off, and it is a MEMORY bound rather than a policy one: an ' + + 'entry carries the observed signature, so at ~1.4 KB each, 50,000 pending is ~70 MB ' + + 'per node. Sized so it does not bind in normal operation — a refusal is an OVERLOAD ' + + 'ALARM saying the drain has fallen far behind detection, not a routine outcome. ' + + 'Watch `probe_trigger_queue_depth` against it, and raise `trigger.ratePerSecond` ' + + 'rather than this if it sits steadily close.', { min: 1 } ), } ), maxTriggersPerSweep: option( - 5000, - 'Ceiling on re-renders one sweep pass may file (per node). Changes past it stay detected but ' + - 'DEFERRED — the signature is left stale so the next pass retries — bounding how much queue ' + - 'injection a widespread change can cause. A genuinely mass change is the canary’s job, where ' + - 'one invalidation row replaces thousands of due-now writes.', - { min: 1 } + 0, + 'Ceiling on changes one pass may SUBMIT for re-render (per node). `0`, the default, means no ' + + 'ceiling.\n\n' + + 'This used to default to a finite number and drop everything past it, counting the remainder ' + + 'as `deferred` with its signature left stale so a later pass would re-detect it. That is a ' + + 'bad trade and the default is now off: the origin read that proved the URL changed has ' + + 'already been paid for, and discarding the result re-buys it on the next pass — which in ' + + '`anchored` mode is a DAY later. The render queue is itself a backlog, so a trigger files a ' + + 'row and the fleet drains it at whatever rate it can; `trigger.ratePerSecond` bounds how ' + + 'fast those writes land, not whether they land at all.\n\n' + + 'Set a finite value only to cap what ONE pass may inject — while sizing a new rule, say, ' + + 'where a mistake would otherwise queue the whole corpus. A genuinely mass change is the ' + + 'canary’s job, where one invalidation row replaces thousands of due-now writes.', + { min: 0 } ), requestTimeout: option(10 * SECOND, 'Per-probe timeout, headers and body both.', { unit: 'ms', diff --git a/packages/plugin/src/util/changeProbe.js b/packages/plugin/src/util/changeProbe.js index 85e8a94..c4a5c37 100644 --- a/packages/plugin/src/util/changeProbe.js +++ b/packages/plugin/src/util/changeProbe.js @@ -141,7 +141,8 @@ const newStats = () => ({ rebaselined: 0, // baseline was taken under a DIFFERENT rule fingerprint: observation stored, nothing compared or triggered (a rule edit, not a content change) unchanged: 0, changed: 0, - queued: 0, // changes handed to the trigger queue (accepted, not necessarily settled yet) + queued: 0, // changes handed to the trigger queue by THIS pass (accepted, not necessarily settled) + triggerQueuePending: 0, // still queued when the pass ended — the drain outlives the pass triggered: 0, // changes that scheduled a re-render — merged from the queue after it drains deferred: 0, // changes past maxTriggersPerSweep, or refused by a full queue — signature kept stale so the next pass retries failed: 0, // fetch/parse/extraction failures — signature untouched, nothing triggered @@ -313,6 +314,45 @@ const probeOnce = async (rule, url) => { * invalidated page outright); `Target.revalidate` keeps the plain `Date.now()` expiry * deliberately — an operator asking for a re-render is not asserting the content is wrong. */ +/** + * The sweep's trigger queue, created once and kept for the life of the process. + * + * MODULE-SCOPED BECAUSE THE DRAIN OUTLIVES THE PASS. A pass can detect changes faster than the + * bounded drain places them (probing at the ceiling with a high change rate does exactly that), and + * the alternative to carrying the remainder forward is dropping it — which throws away an origin + * read already paid for and re-buys it on the next pass, a full day later in anchored mode. + * + * Rebuilt only when the knobs change, so a live config edit takes effect without losing what is + * already queued. + */ +let triggerQueue = null; +let triggerQueueKey = null; +const sweepTriggers = () => { + const { ratePerSecond, concurrency, maxPending } = config.changeProbe.trigger; + const key = `${ratePerSecond}|${concurrency}|${maxPending}`; + if (triggerQueue && triggerQueueKey === key) return triggerQueue; + // A rebuild inherits nothing: the old queue's pending items were never baselined, so the next + // pass re-detects them. Stopping it prevents two queues draining at once at double the rate. + triggerQueue?.stop(); + triggerQueueKey = key; + triggerQueue = createTriggerQueue({ + trigger: triggerRevalidate, + write: writeSignature, + maxPending, + ratePerSecond, + concurrency, + onError: (e, item) => logger.error(e, `[prerender] change-probe trigger failed for ${item.row.url}`), + }); + return triggerQueue; +}; + +/** Drop the queue — probe disabled, or a test resetting module state. */ +export const resetTriggerQueue = () => { + triggerQueue?.stop(); + triggerQueue = null; + triggerQueueKey = null; +}; + export const triggerRevalidate = async (row) => { const keys = cacheKeysOf(row.url); const hardExpiredAt = Date.now() - config.page.swrTtl; @@ -703,11 +743,14 @@ export const runProbePass = async ({ await write(row.url, observed, { rowExists: stored !== null, fingerprint: rule.fingerprint }); return; } - if (stats.queued >= maxTriggers) { - // Budget spent: leave the signature STALE so the next pass re-detects and retries. - // Bounds how much queue injection one pass can do (a mass change is the canary's job). - // Counted on ACCEPTANCE, not completion: the budget has to be decided synchronously here - // or a burst would race past it while earlier triggers were still settling. + // A per-pass ceiling, and by default there ISN'T ONE (0 = unlimited). Deferring used to be the + // routine outcome of a busy pass, and it is a bad trade: the origin read that proved this URL + // changed has already been paid for, and dropping the result throws that away and re-buys it + // next pass — which in anchored mode is a DAY later. The render queue is itself a backlog; a + // trigger files a row and the fleet drains it at whatever rate it can. So the default is to + // queue everything and let that queue do the absorbing, with the drain rate bounding how fast + // the writes land rather than whether they land at all. + if (maxTriggers > 0 && stats.queued >= maxTriggers) { stats.deferred++; return; } @@ -1069,17 +1112,8 @@ export const runProbeSweepOnce = async ({ dryRun, label = null, reseed = false } const collectors = new Map(rules.map((rule) => [rule.label, cohortCollector(count)])); let unreadable = 0; let yields = 0; - // Triggers drain BESIDE the walk, not inside it. `submit` returns immediately, so the pass - // runs at its probe-rate floor whatever the change rate — see util/triggerQueue.js for the - // feedback loop this breaks. - const triggers = createTriggerQueue({ - trigger: triggerRevalidate, - write: writeSignature, - maxPending: config.changeProbe.trigger.maxPending, - ratePerSecond: config.changeProbe.trigger.ratePerSecond, - concurrency: config.changeProbe.trigger.concurrency, - onError: (e, item) => logger.error(e, `[prerender] change-probe trigger failed for ${item.row.url}`), - }); + // Triggers drain BESIDE the walk, and OUTLIVE it — see `sweepTriggers`. + const triggers = sweepTriggers(); const stats = await runProbePass({ rows: walkTargets(config.changeProbe.chunkSize, () => { unreadable++; @@ -1104,7 +1138,10 @@ export const runProbeSweepOnce = async ({ dryRun, label = null, reseed = false } // `stats` in the temporal dead zone while this callback runs, so touching it here // throws a ReferenceError rather than reading undefined. yields++; - await beat({ examinedApprox: yields * YIELD_EVERY }); + // Queue depth rides the heartbeat so the queue is visible WHILE the pass runs. Without + // it the only reading came from a finished pass, which is useless for a drain whose + // whole purpose is to outlive the pass. + await beat({ examinedApprox: yields * YIELD_EVERY, triggerQueuePending: triggers.depth }); }, // A reseed re-baselines everything, so it must not skip fresh-looking rows. reprobeAfter: reseed ? 0 : config.changeProbe.reprobeAfter, @@ -1112,14 +1149,23 @@ export const runProbeSweepOnce = async ({ dryRun, label = null, reseed = false } isCanceled: () => !config.changeProbe.enabled || sweepInterrupt !== null, collectCohort: (rule, url) => collectors.get(rule.label).add(url), }); - // A cancelled pass abandons what is still queued: those rows never had their baseline - // written, so the next pass re-detects them. Otherwise wait the queue out — the pass is not - // finished while re-renders it decided on are still unfiled, and `triggered` would under-report. - if (stats.aborted) triggers.stop(); - await triggers.drain(); + // THE PASS DOES NOT WAIT FOR THE QUEUE. Draining is bounded by `trigger.ratePerSecond`, so a + // pass that detected more change than the drain can place would otherwise be held open by it + // — and in anchored mode a pass still running at the next anchor causes that anchor to be + // SKIPPED, turning a busy night into a missed one. The queue keeps going after the pass ends + // and the next pass adds to it. + // + // An aborted pass does NOT stop the queue either: what is already in it was genuinely + // detected and its baseline is unwritten, so draining it is still the right thing. Only + // disabling the probe clears it. + // + // `triggered`/`errors` are therefore CUMULATIVE counters read at pass end, not per-pass + // totals — a trigger submitted by this pass may settle during the next. `queued` is the + // honest per-pass number, and `triggerQueuePending` says how much of it had not landed yet. stats.triggered = triggers.stats.triggered; stats.errors = triggers.stats.errors; stats.triggerQueueDepth = triggers.stats.maxDepth; + stats.triggerQueuePending = triggers.depth; stats.unreadable = unreadable; // An interrupted pass keeps the OLD cohorts — a partial walk's sample covers only the key // range it reached, and the chained reseed rebuilds them properly. @@ -1724,6 +1770,7 @@ export const __passLimitsForTest = passLimits; /** Tests only — module state that outlives a beforeEach. */ export const resetChangeProbeState = () => { + resetTriggerQueue(); clearProbeTimers(); schedulerStarted = false; armedSweep = armedCanary = null; diff --git a/packages/plugin/test/changeProbe.test.js b/packages/plugin/test/changeProbe.test.js index d30cd6f..d122618 100644 --- a/packages/plugin/test/changeProbe.test.js +++ b/packages/plugin/test/changeProbe.test.js @@ -301,6 +301,24 @@ test('past the trigger budget a change DEFERS: signature left stale so the next assert.deepEqual(written, [{ url: URL_A, signature: '[2]' }]); }); +test('maxTriggers 0 means NO ceiling: nothing is deferred, everything is queued', async () => { + // The default. Deferring throws away an origin read already paid for and re-buys it next pass — + // a day later in anchored mode — so the render queue, which is itself a backlog, absorbs the + // volume instead. `trigger.ratePerSecond` bounds how fast the writes land, not whether they do. + const { stats, written, triggered } = await runPass({ + rows: [row(URL_A), row(URL_B), row(URL_C)], + stored: { [URL_A]: '[1]', [URL_B]: '[1]', [URL_C]: '[1]' }, + answers: { [URL_A]: '[2]', [URL_B]: '[2]', [URL_C]: '[2]' }, + maxTriggers: 0, + concurrency: 1, + }); + assert.equal(stats.deferred, 0, 'no change may be dropped when there is no ceiling'); + assert.equal(stats.queued, 3, 'every detected change was accepted'); + assert.equal(stats.triggered, 3); + assert.deepEqual(triggered.sort(), [URL_A, URL_B, URL_C].sort()); + assert.equal(written.length, 3, 'and each got its baseline, after its trigger'); +}); + test('a failed trigger keeps the signature stale too', async () => { // The property is unchanged by the move to a submitted trigger; only the seam moved. Driven // through the REAL inline shape rather than a stub, because the ordering under test — baseline