From 712da39aca8ca69b866f8f303f393a50798ec014 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 27 Aug 2026 21:14:51 +0000 Subject: [PATCH 1/9] perf_hooks: implement Histogram meanCI API Signed-off-by: James M Snell Assisted-by: Opencode --- doc/api/perf_hooks.md | 35 ++++++++ lib/internal/histogram.js | 24 ++++++ src/histogram.cc | 76 +++++++++++++---- src/histogram.h | 10 ++- .../test-perf-hooks-histogram-stats.js | 81 +++++++++++++++++++ 5 files changed, 208 insertions(+), 18 deletions(-) diff --git a/doc/api/perf_hooks.md b/doc/api/perf_hooks.md index b9e00c63d2aa..c24082aa20e4 100644 --- a/doc/api/perf_hooks.md +++ b/doc/api/perf_hooks.md @@ -2207,6 +2207,41 @@ added: v11.10.0 The mean of the recorded event loop delays. +### `histogram.meanCI([options])` + + + +* `options` {Object} + * `confidence` {number} The confidence level for the interval, between + 0 and 1 (exclusive). **Default:** `0.95`. +* Returns: {Object} + * `mean` {number} The mean estimate, equivalent to `histogram.mean`. + * `lower` {number} The lower bound of the confidence interval. + * `upper` {number} The upper bound of the confidence interval. + +Returns a two-sided confidence interval for the mean using Student's +t-distribution and the sample standard error. A higher confidence level +produces a wider interval. This interval assumes that samples are independent +and approximately normally distributed, although the approximation is robust +for sufficiently large samples. + +The result reflects the histogram's configured precision and is calculated +from the values represented by its buckets. With fewer than two recorded +values, `lower` and `upper` are `NaN`. When all recorded values are equal, +`lower` and `upper` equal `mean`. + +```js +const { createHistogram } = require('node:perf_hooks'); + +const h = createHistogram(); +for (let i = 1; i <= 100; i++) h.record(i); + +const { mean, lower, upper } = h.meanCI(); +console.log(`mean=${mean}, 95% CI=[${lower}, ${upper}]`); +``` + ### `histogram.min` + + + +> Stability: 1 - Experimental + + + +The `node:bench` module supports defining and running JavaScript benchmarks in +the current process. To access it: + +```mjs +import { bench, suite } from 'node:bench'; +``` + +```cjs +const { bench, suite } = require('node:bench'); +``` + +This module is only available under the `node:` scheme. + +```mjs +import { bench, suite } from 'node:bench'; + +suite('URL', () => { + const input = 'https://example.com/a?b=c'; + + bench('construct', { + samples: 30, + params: { input: 'short' }, + }, (b) => { + const operations = 10_000; + + b.start(); + for (let i = 0; i < operations; i++) { + new URL(input); + } + b.end(operations); + }); +}); +``` + +Benchmarks are executed serially in declaration order. Declared benchmarks are +scheduled automatically. Call `run()` during the same turn as the declarations +to consume the event stream or configure filtering. +If an automatically scheduled run fails and `run()` was not called, the process +exit code is set to `1`. + +## Measurement model + +Each warmup and measured sample invokes the benchmark function once with a +fresh {BenchContext}. The function must call `context.start()` and +`context.end(operations)` exactly once. Setup before `start()` and cleanup after +`end()` are outside the measured region. Promise-returning functions are +awaited. + +An event loop turn occurs between sample invocations. The runner executes +benchmarks serially, but it does not provide process isolation. Other work in +the process, JIT compilation, garbage collection, CPU frequency changes, and +system load can all affect results. Keep raw samples when comparing results and +investigate noisy or skewed distributions rather than treating a confidence +interval as a pass/fail threshold. + +## `bench([name][, options], fn)` + + + +* `name` {string} The benchmark name. **Default:** The `name` property of `fn`, + or `''` when `fn` has no name. +* `options` {Object} + * `only` {boolean} When any benchmark or containing suite has `only` set, + benchmarks without `only` in their hierarchy are skipped. **Default:** + `false`. + * `params` {Object} String, finite number, or boolean metadata identifying + this benchmark configuration. Parameter keys are sorted when constructing + the stable benchmark identity. **Default:** An empty object. + * `samples` {number} The number of measured callback invocations. Must be a + positive 32-bit unsigned integer. **Default:** `30`. + * `signal` {AbortSignal} Allows aborting this benchmark. + * `skip` {boolean|string} If truthy, the benchmark is skipped. A string is + included in the result as the skip reason. **Default:** `false`. + * `tags` {string\[]} Labels associated with the benchmark. Tags are + lowercased, deduplicated, and inherited from containing suites by union. + **Default:** `[]`. + * `timeout` {number} The number of milliseconds after which the benchmark + fails. **Default:** `Infinity`. + * `warmup` {number} The number of unreported callback invocations before + measured samples. Must be a 32-bit unsigned integer. **Default:** `0`. +* `fn` {Function|AsyncFunction} The benchmark function. It receives a + {BenchContext}. +* Returns: {Promise} Fulfilled with the benchmark result after a top-level + benchmark finishes, or with `undefined` immediately when declared in a + suite. + +Warmup invocations use the same callback and timing contract as measured +samples, but their samples are discarded. An exception, rejection, timeout, +abort, missing timing call, or duplicate timing call stops the current +benchmark. Later benchmarks continue to run. + +A timeout or abort cannot interrupt synchronous JavaScript and does not forcibly +cancel asynchronous work that ignores `context.signal`. + +The stable `benchId` is based on the source file, hierarchical suite and +benchmark names, and canonicalized parameters. Declaring the same identity +more than once reports an error rather than merging the samples. + +### `bench.skip([name][, options], fn)` + + + +Shorthand for `bench(name, { ...options, skip: true }, fn)`. + +### `bench.only([name][, options], fn)` + + + +Shorthand for `bench(name, { ...options, only: true }, fn)`. + +## `suite([name][, options], fn)` + + + +* `name` {string} The suite name. **Default:** The `name` property of `fn`, or + `''` when `fn` has no name. +* `options` {Object} + * `only` {boolean} Selects all benchmarks nested in this suite. **Default:** + `false`. + * `skip` {boolean|string} Skips all benchmarks nested in this suite. + **Default:** `false`. + * `tags` {string\[]} Labels inherited by nested suites and benchmarks. + **Default:** `[]`. +* `fn` {Function|AsyncFunction} A function that declares nested suites, + benchmarks, and hooks. +* Returns: {Promise} Fulfilled when a top-level suite finishes, or with + `undefined` immediately when declared in another suite. + +Suite functions run while declarations are collected. Promise-returning suite +functions are awaited before benchmark execution begins. + +## `describe([name][, options], fn)` + + + +Alias for `suite()`. + +## `before(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. + +Registers a hook that runs once before the benchmarks in the current suite. + +## `after(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. + +Registers a hook that runs once after the benchmarks in the current suite. + +## `beforeEach(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. It receives an object with + the benchmark's `name`, `params`, and `signal`. + +Registers a hook that runs once before each complete logical benchmark in the +current suite. It does not run before every sample. Per-sample setup belongs in +the benchmark function before `context.start()`. + +## `afterEach(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. It receives an object with + the benchmark's `name`, `params`, and `signal`. + +Registers a hook that runs once after each complete logical benchmark in the +current suite. It does not run after every sample. Per-sample cleanup belongs +in the benchmark function after `context.end()`. + +## `run([options])` + + + +* `options` {Object} + * `namePattern` {string|RegExp} Only runs benchmarks whose full hierarchical + name matches the pattern. String values are interpreted as JavaScript + regular expressions. + * `signal` {AbortSignal} Allows aborting in-progress benchmark execution. +* Returns: {BenchmarksStream} + +Returns the object-mode event stream for the in-process benchmark run. Call +`run()` during the same turn in which benchmarks are declared, before automatic +execution begins. Calling `run()` is optional when the returned stream is not +needed. + +```mjs +import { bench, run } from 'node:bench'; + +bench('example', { samples: 3 }, (b) => { + b.start(); + doWork(); + b.end(1); +}); + +for await (const { type, data } of run()) { + if (type === 'bench:complete' && data.error === undefined) { + console.log(data.name, data.summary.mean); + } +} +``` + +## Class: `BenchContext` + +An instance of `BenchContext` is passed to every benchmark invocation. A new +instance is created for every warmup and measured sample. + +### `context.name` + + + +* {string} + +The benchmark name. + +### `context.params` + + + +* {Object} + +The benchmark's canonicalized parameter metadata. + +### `context.signal` + + + +* {AbortSignal} + +An abort signal that is triggered when the benchmark is aborted, times out, or +finishes. + +### `context.start()` + + + +Starts the measured region using `process.hrtime.bigint()`. Calling `start()` +more than once is an error. + +### `context.end(operations)` + + + +* `operations` {number} The number of completed operations. Must be a positive + safe integer. + +Ends the measured region. The end timestamp is captured before `operations` is +validated. Calling `end()` before `start()`, calling it more than once, or +recording a zero-duration sample is an error. + +## Class: `BenchmarksStream` + +`BenchmarksStream` is an object-mode {stream.Readable}. Each lifecycle record is +both emitted as a named event and made available on the stream as +`{ type, data }`. + +The events are emitted in execution order: + +* `'bench:start'` +* `'bench:sample'` +* `'bench:complete'` +* `'bench:diagnostic'` +* `'bench:summary'` + +Every benchmark-scoped event contains `benchId` and `parentId`. +`'bench:complete'` data contains a [benchmark result][]. A failed result has an +additional `error` property and may contain samples recorded before the error. +A skipped result has an additional `skip` property and an empty `samples` +array. `'bench:diagnostic'` reports suite and hook errors. `'bench:summary'` +contains overall `success`, `counts`, `duration_ns`, and `file` properties. + +## Sample result + +Each measured sample has the following properties: + +* `operations` {number} The positive operation count passed to + `context.end()`. +* `duration_ns` {bigint} The measured duration in nanoseconds. +* `rate` {number} Operations per second. + +## Benchmark result + +A completed benchmark result contains: + +* `benchId` {string} The stable benchmark identity. +* `parentId` {string|null} The stable containing suite identity. +* `name` {string} The benchmark name. +* `file` {string} The source file. +* `line` {number} The source line. +* `column` {number} The source column. +* `tags` {string\[]} The inherited canonical tags. +* `params` {Object} The canonical parameter metadata. +* `samples` {Object\[]} The exact measured samples. +* `summary` {Object} + * `mean` {number} The arithmetic mean of per-sample rates. + * `median` {number} The median per-sample rate. + * `min` {number} The minimum per-sample rate. + * `max` {number} The maximum per-sample rate. + * `stddev` {number} The population standard deviation of rates. + * `coefficientOfVariation` {number} `stddev / mean`. + * `confidenceInterval` {Object} The 95% Student's t confidence interval for + the mean rate, with `lower` and `upper` properties. + * `medianConfidenceInterval` {Object} The 95% nonparametric confidence + interval for the median rate, with `lower` and `upper` properties. + * `skewness` {number} The skewness of the scaled rate histogram. + +[benchmark result]: #benchmark-result diff --git a/doc/api/index.md b/doc/api/index.md index 146c0e13df65..a30724c064e1 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -9,6 +9,7 @@ * [Assertion testing](assert.md) * [Asynchronous context tracking](async_context.md) * [Async hooks](async_hooks.md) +* [Benchmark runner](bench.md) * [Buffer](buffer.md) * [C++ addons](addons.md) * [C/C++ addons with Node-API](n-api.md) diff --git a/lib/bench.js b/lib/bench.js new file mode 100644 index 000000000000..454fad4f8570 --- /dev/null +++ b/lib/bench.js @@ -0,0 +1,30 @@ +'use strict'; + +const { + ObjectAssign, +} = primordials; + +const { emitExperimentalWarning } = require('internal/util'); +const { + after, + afterEach, + before, + beforeEach, + bench, + suite, +} = require('internal/bench_runner/harness'); +const { run } = require('internal/bench_runner/runner'); + +emitExperimentalWarning('Benchmarks'); + +module.exports = bench; +ObjectAssign(module.exports, { + after, + afterEach, + before, + beforeEach, + bench, + describe: suite, + run, + suite, +}); diff --git a/lib/internal/bench_runner/benchmark.js b/lib/internal/bench_runner/benchmark.js new file mode 100644 index 000000000000..27f372d7295f --- /dev/null +++ b/lib/internal/bench_runner/benchmark.js @@ -0,0 +1,411 @@ +'use strict'; + +const { + ArrayIsArray, + ArrayPrototypeJoin, + ArrayPrototypePush, + ArrayPrototypeReverse, + ArrayPrototypeSlice, + ArrayPrototypeSort, + JSONStringify, + MathFloor, + MathMax, + MathMin, + MathRound, + MathSqrt, + Number, + NumberIsFinite, + NumberMAX_SAFE_INTEGER, + ObjectFreeze, + ObjectKeys, + PromiseWithResolvers, + SafeSet, + StringPrototypeToLowerCase, +} = primordials; +const { AsyncResource } = require('async_hooks'); +const { + codes: { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_ARG_VALUE, + ERR_INVALID_STATE, + ERR_OUT_OF_RANGE, + }, +} = require('internal/errors'); +const { createHistogram } = require('internal/histogram'); +const { TIMEOUT_MAX } = require('internal/timers'); +const { kEmptyObject } = require('internal/util'); +const { + validateAbortSignal, + validateFunction, + validateInteger, + validateNumber, + validateObject, + validateString, + validateUint32, +} = require('internal/validators'); + +const { bigint: hrtime } = process.hrtime; +const kDefaultSamples = 30; +const kDefaultWarmup = 0; +const kEmptyParams = ObjectFreeze({ __proto__: null }); +const kEmptyTags = ObjectFreeze([]); + +function validateSkip(skip) { + if (skip !== undefined && typeof skip !== 'boolean' && + typeof skip !== 'string') { + throw new ERR_INVALID_ARG_TYPE('options.skip', ['boolean', 'string'], skip); + } +} + +function canonicalizeTags(tags, parentTags = kEmptyTags) { + if (tags === undefined) return parentTags; + if (!ArrayIsArray(tags)) { + throw new ERR_INVALID_ARG_TYPE('options.tags', 'Array', tags); + } + + const result = ArrayPrototypeSlice(parentTags); + const seen = new SafeSet(parentTags); + for (let i = 0; i < tags.length; i++) { + validateString(tags[i], `options.tags[${i}]`); + if (tags[i].length === 0) { + throw new ERR_INVALID_ARG_VALUE( + `options.tags[${i}]`, tags[i], 'must not be empty'); + } + const tag = StringPrototypeToLowerCase(tags[i]); + if (!seen.has(tag)) { + seen.add(tag); + ArrayPrototypePush(result, tag); + } + } + return ObjectFreeze(result); +} + +function canonicalizeParams(params) { + if (params === undefined) return kEmptyParams; + validateObject(params, 'options.params'); + + const result = { __proto__: null }; + const keys = ObjectKeys(params); + ArrayPrototypeSort(keys); + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const value = params[key]; + if (typeof value !== 'string' && typeof value !== 'boolean' && + (typeof value !== 'number' || !NumberIsFinite(value))) { + if (typeof value === 'number') { + throw new ERR_OUT_OF_RANGE( + `options.params.${key}`, 'a finite number', value); + } + throw new ERR_INVALID_ARG_TYPE( + `options.params.${key}`, ['string', 'number', 'boolean'], value); + } + result[key] = value; + } + return ObjectFreeze(result); +} + +function validateNodeOptions(options, parentTags) { + validateObject(options, 'options'); + const { only = false, skip, tags } = options; + if (typeof only !== 'boolean') { + throw new ERR_INVALID_ARG_TYPE('options.only', 'boolean', only); + } + validateSkip(skip); + return { + __proto__: null, + only, + skip, + tags: canonicalizeTags(tags, parentTags), + }; +} + +function createLocation(loc, fallbackFile) { + return { + __proto__: null, + file: loc?.[2] ?? fallbackFile, + line: loc?.[0], + column: loc?.[1], + }; +} + +function getNamePath(parent, name) { + const path = []; + for (let current = parent; current?.parent !== null; current = current.parent) { + ArrayPrototypePush(path, current.name); + } + ArrayPrototypeReverse(path); + ArrayPrototypePush(path, name); + return path; +} + +class Suite extends AsyncResource { + constructor(harness, parent, name, options, fn, loc, isRoot = false) { + super('BenchSuite'); + const validated = validateNodeOptions( + options, parent?.tags ?? kEmptyTags); + + this.harness = harness; + this.parent = parent; + this.name = name; + this.fn = fn; + this.loc = createLocation(loc, harness.entryFile); + this.only = validated.only; + this.skip = validated.skip; + this.tags = validated.tags; + this.isRoot = isRoot; + this.children = []; + this.hooks = { + __proto__: null, + after: [], + afterEach: [], + before: [], + beforeEach: [], + }; + this.buildError = null; + this.buildPromise = null; + this.finished = false; + this.completion = PromiseWithResolvers(); + } +} + +class Bench extends AsyncResource { + constructor(harness, parent, name, options, fn, loc) { + super('Benchmark'); + const validated = validateNodeOptions(options, parent.tags); + const { + params, + samples = kDefaultSamples, + signal, + timeout = Infinity, + warmup = kDefaultWarmup, + } = options; + + validateUint32(samples, 'options.samples', true); + validateUint32(warmup, 'options.warmup'); + validateAbortSignal(signal, 'options.signal'); + if (timeout !== Infinity) { + validateNumber(timeout, 'options.timeout', 0, TIMEOUT_MAX); + } + + this.harness = harness; + this.parent = parent; + this.name = name; + this.fn = fn; + this.loc = createLocation(loc, harness.entryFile); + this.only = validated.only; + this.skip = validated.skip; + this.tags = validated.tags; + this.params = canonicalizeParams(params); + this.samples = samples; + this.warmup = warmup; + this.timeout = timeout; + this.outerSignal = signal; + this.namePath = getNamePath(parent, name); + this.fullName = ArrayPrototypeJoin(this.namePath, ' '); + this.benchId = JSONStringify([ + this.loc.file, + this.namePath, + this.params, + ]); + this.parentId = parent.isRoot ? null : JSONStringify([ + this.loc.file, + getNamePath(parent.parent, parent.name), + ]); + this.finished = false; + this.result = null; + this.completion = PromiseWithResolvers(); + } +} + +class BenchContext { + #closed = false; + #endCalled = false; + #invalid = false; + #sample = null; + #startCalled = false; + #startTime; + + constructor(bench, signal) { + this.name = bench.name; + this.params = bench.params; + this.signal = signal; + } + + start() { + if (this.#closed) { + this.#invalid = true; + throw new ERR_INVALID_STATE('benchmark sample is no longer active'); + } + if (this.#startCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'start() must be called exactly once per benchmark sample'); + } + this.#startCalled = true; + this.#startTime = hrtime(); + } + + end(operations) { + const endTime = hrtime(); + if (this.#closed) { + this.#invalid = true; + throw new ERR_INVALID_STATE('benchmark sample is no longer active'); + } + if (this.#endCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'end() must be called exactly once per benchmark sample'); + } + this.#endCalled = true; + if (!this.#startCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE('end() cannot be called before start()'); + } + + try { + validateInteger(operations, 'operations', 1, NumberMAX_SAFE_INTEGER); + } catch (error) { + this.#invalid = true; + throw error; + } + + const duration = endTime - this.#startTime; + if (duration === 0n) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'insufficient clock precision for benchmark sample'); + } + this.#sample = { + __proto__: null, + operations, + duration_ns: duration, + rate: operations / (Number(duration) / 1e9), + }; + } + + finish() { + this.#closed = true; + if (this.#invalid) { + throw new ERR_INVALID_STATE( + 'benchmark sample violated the start()/end() contract'); + } + if (!this.#startCalled) { + throw new ERR_INVALID_STATE( + 'benchmark callback did not call start()'); + } + if (!this.#endCalled || this.#sample === null) { + throw new ERR_INVALID_STATE( + 'benchmark callback did not call end()'); + } + return this.#sample; + } + + close() { + this.#closed = true; + } +} + +function arithmeticMean(values) { + let sum = 0; + let compensation = 0; + for (let i = 0; i < values.length; i++) { + const adjusted = values[i] - compensation; + const next = sum + adjusted; + compensation = (next - sum) - adjusted; + sum = next; + } + return sum / values.length; +} + +function summarizeSamples(samples) { + const rates = []; + let min = Infinity; + let max = -Infinity; + for (let i = 0; i < samples.length; i++) { + const rate = samples[i].rate; + ArrayPrototypePush(rates, rate); + min = MathMin(min, rate); + max = MathMax(max, rate); + } + + const mean = arithmeticMean(rates); + let variance = 0; + for (let i = 0; i < rates.length; i++) { + const difference = rates[i] - mean; + variance += difference * difference; + } + variance /= rates.length; + const stddev = MathSqrt(variance); + + const sorted = ArrayPrototypeSlice(rates); + ArrayPrototypeSort(sorted, (a, b) => a - b); + const middle = MathFloor(sorted.length / 2); + const median = sorted.length % 2 === 0 ? + (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; + + const scale = MathMin(1_000_000, NumberMAX_SAFE_INTEGER / max); + const histogram = createHistogram({ __proto__: null, figures: 5 }); + for (let i = 0; i < rates.length; i++) { + const value = MathMax( + 1, + MathMin(NumberMAX_SAFE_INTEGER, MathRound(rates[i] * scale)), + ); + histogram.record(value); + } + + const meanCI = histogram.meanCI(); + const histogramMean = meanCI.mean / scale; + const medianCI = histogram.percentileCI(50); + + return { + __proto__: null, + mean, + median, + min, + max, + stddev, + coefficientOfVariation: stddev / mean, + confidenceInterval: { + __proto__: null, + lower: mean + meanCI.lower / scale - histogramMean, + upper: mean + meanCI.upper / scale - histogramMean, + }, + medianConfidenceInterval: { + __proto__: null, + lower: medianCI.lower / scale, + upper: medianCI.upper / scale, + }, + skewness: histogram.skewness, + }; +} + +function normalizeArgs(type, name, options, fn) { + if (typeof name === 'function') { + fn = name; + name = fn.name || ''; + options = kEmptyObject; + } else if (name !== null && typeof name === 'object') { + fn = options; + options = name; + name = fn?.name || ''; + } else if (typeof options === 'function') { + fn = options; + options = kEmptyObject; + } + + validateFunction(fn, `${type} function`); + validateString(name, `${type} name`); + if (name.length === 0) { + throw new ERR_INVALID_ARG_VALUE(`${type} name`, name, 'must not be empty'); + } + validateObject(options, 'options'); + return { __proto__: null, fn, name, options }; +} + +module.exports = { + Bench, + BenchContext, + Suite, + normalizeArgs, + summarizeSamples, +}; diff --git a/lib/internal/bench_runner/benchmarks_stream.js b/lib/internal/bench_runner/benchmarks_stream.js new file mode 100644 index 000000000000..ac3896d0587e --- /dev/null +++ b/lib/internal/bench_runner/benchmarks_stream.js @@ -0,0 +1,74 @@ +'use strict'; + +const { + ArrayPrototypePush, + ArrayPrototypeShift, + NumberMAX_SAFE_INTEGER, + Symbol, +} = primordials; +const Readable = require('internal/streams/readable'); + +const kEmitMessage = Symbol('kEmitMessage'); + +class BenchmarksStream extends Readable { + #buffer = []; + #canPush = true; + + constructor() { + super({ + __proto__: null, + objectMode: true, + highWaterMark: NumberMAX_SAFE_INTEGER, + }); + } + + _read() { + this.#canPush = true; + while (this.#buffer.length > 0) { + const record = ArrayPrototypeShift(this.#buffer); + if (!this.#tryPush(record)) return; + } + } + + start(data) { + this[kEmitMessage]('bench:start', data); + } + + sample(data) { + this[kEmitMessage]('bench:sample', data); + } + + complete(data) { + this[kEmitMessage]('bench:complete', data); + } + + diagnostic(data) { + this[kEmitMessage]('bench:diagnostic', data); + } + + summary(data) { + this[kEmitMessage]('bench:summary', data); + } + + end() { + this.#tryPush(null); + } + + [kEmitMessage](type, data) { + this.emit(type, data); + this.#tryPush({ type, data }); + } + + #tryPush(record) { + if (this.#canPush) { + this.#canPush = this.push(record); + } else { + ArrayPrototypePush(this.#buffer, record); + } + return this.#canPush; + } +} + +module.exports = { + BenchmarksStream, +}; diff --git a/lib/internal/bench_runner/harness.js b/lib/internal/bench_runner/harness.js new file mode 100644 index 000000000000..0409b38e6b91 --- /dev/null +++ b/lib/internal/bench_runner/harness.js @@ -0,0 +1,687 @@ +'use strict'; + +const { + ArrayPrototypePush, + ArrayPrototypeReverse, + ArrayPrototypeSlice, + FunctionPrototypeCall, + Promise, + PromisePrototypeThen, + PromiseResolve, + PromiseWithResolvers, + ReflectApply, + RegExp, + RegExpPrototypeExec, + SafeMap, + SafePromiseRace, + SymbolDispose, +} = primordials; +const { getCallerLocation } = internalBinding('util'); +const { exitCodes: { kGenericUserError } } = internalBinding('errors'); +const { AsyncLocalStorage } = require('async_hooks'); +const { AbortController } = require('internal/abort_controller'); +const { + AbortError, + codes: { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_STATE, + ERR_OPERATION_FAILED, + }, +} = require('internal/errors'); +const { addAbortListener } = require('internal/events/abort_listener'); +const { + kEmptyObject, +} = require('internal/util'); +const { isRegExp } = require('internal/util/types'); +const { + validateAbortSignal, + validateFunction, + validateObject, +} = require('internal/validators'); +const { queueMicrotask } = require('internal/process/task_queues'); +const { clearTimeout, setImmediate, setTimeout } = require('timers'); +const { + Bench, + BenchContext, + Suite, + normalizeArgs, + summarizeSamples, +} = require('internal/bench_runner/benchmark'); +const { + BenchmarksStream, +} = require('internal/bench_runner/benchmarks_stream'); + +const { bigint: hrtime } = process.hrtime; +const kHookNames = ['after', 'afterEach', 'before', 'beforeEach']; + +function eventLoopTurn() { + return new Promise((resolve) => setImmediate(resolve)); +} + +function createAbortError(signal) { + return new AbortError(undefined, { __proto__: null, cause: signal.reason }); +} + +class Harness { + #buildPromises = []; + #duplicateErrors = new SafeMap(); + #explicitRun = false; + #hasOnly = false; + #runPromise = null; + #scheduled = false; + #storage = new AsyncLocalStorage(); + + constructor() { + this.entryFile = process.argv?.[1]; + this.stream = new BenchmarksStream(); + this.state = 'collecting'; + this.namePattern = null; + this.outerSignal = undefined; + this.success = true; + this.counts = { + __proto__: null, + completed: 0, + failed: 0, + skipped: 0, + total: 0, + }; + this.root = new Suite( + this, + null, + '', + kEmptyObject, + undefined, + undefined, + true, + ); + } + + #ensureCollecting() { + if (this.state === 'collecting' || + (this.state === 'building' && + this.#storage.getStore() instanceof Suite)) return; + throw new ERR_INVALID_STATE( + 'benchmarks cannot be declared after execution has started'); + } + + #getParent() { + const current = this.#storage.getStore(); + return current instanceof Suite ? current : this.root; + } + + createBench(name, options, fn, overrides = kEmptyObject) { + this.#ensureCollecting(); + const normalized = normalizeArgs('benchmark', name, options, fn); + const parent = this.#getParent(); + const benchmark = new Bench( + this, + parent, + normalized.name, + { __proto__: null, ...normalized.options, ...overrides }, + normalized.fn, + overrides.loc, + ); + ArrayPrototypePush(parent.children, benchmark); + this.#schedule(); + return parent.isRoot ? benchmark.completion.promise : PromiseResolve(); + } + + createSuite(name, options, fn, overrides = kEmptyObject) { + this.#ensureCollecting(); + const normalized = normalizeArgs('suite', name, options, fn); + const parent = this.#getParent(); + const suite = new Suite( + this, + parent, + normalized.name, + { __proto__: null, ...normalized.options, ...overrides }, + normalized.fn, + overrides.loc, + ); + ArrayPrototypePush(parent.children, suite); + this.#buildSuite(suite); + this.#schedule(); + return parent.isRoot ? suite.completion.promise : PromiseResolve(); + } + + createHook(name, fn, options = kEmptyObject) { + this.#ensureCollecting(); + validateFunction(fn, 'hook function'); + validateObject(options, 'options'); + const parent = this.#getParent(); + ArrayPrototypePush(parent.hooks[name], { + __proto__: null, + fn, + loc: getCallerLocation(), + }); + this.#schedule(); + } + + #buildSuite(suite) { + let result; + try { + result = suite.runInAsyncScope(() => this.#storage.run( + suite, + () => FunctionPrototypeCall(suite.fn), + )); + } catch (error) { + suite.buildError = error; + result = undefined; + } + + suite.buildPromise = PromisePrototypeThen( + PromiseResolve(result), + undefined, + (error) => { + suite.buildError = error; + }, + ); + ArrayPrototypePush(this.#buildPromises, suite.buildPromise); + } + + configure(options = kEmptyObject) { + validateObject(options, 'options'); + if (this.#runPromise !== null) { + if (options !== kEmptyObject) { + throw new ERR_INVALID_STATE('benchmark execution has already started'); + } + return; + } + + const { namePattern, signal } = options; + if (namePattern !== undefined) { + if (typeof namePattern === 'string') { + this.namePattern = new RegExp(namePattern); + } else if (isRegExp(namePattern)) { + this.namePattern = namePattern; + } else { + throw new ERR_INVALID_ARG_TYPE( + 'options.namePattern', ['string', 'RegExp'], namePattern); + } + } + validateAbortSignal(signal, 'options.signal'); + this.outerSignal = signal; + this.#explicitRun = true; + } + + run(options = kEmptyObject) { + this.configure(options); + this.#schedule(); + return this.stream; + } + + #schedule() { + if (this.#scheduled) return; + this.#scheduled = true; + queueMicrotask(() => { + if (this.#runPromise === null) { + this.#runPromise = this.#execute(); + PromisePrototypeThen(this.#runPromise, undefined, (error) => { + this.#diagnostic(error, undefined, 'error'); + this.#finish(); + }); + } + }); + } + + async #waitForBuild() { + for (let i = 0; i < this.#buildPromises.length; i++) { + await this.#buildPromises[i]; + } + } + + #walk(node, callback) { + for (let i = 0; i < node.children.length; i++) { + const child = node.children[i]; + callback(child); + if (child instanceof Suite) this.#walk(child, callback); + } + } + + #prepare() { + const identities = new SafeMap(); + this.#walk(this.root, (node) => { + if (node.only) this.#hasOnly = true; + if (!(node instanceof Bench)) return; + + this.counts.total++; + const existing = identities.get(node.benchId); + if (existing === undefined) { + identities.set(node.benchId, node); + } else { + this.#duplicateErrors.set(node, new ERR_INVALID_STATE( + `duplicate benchmark identity for "${node.fullName}"`)); + } + }); + } + + #hasSelectedAncestor(benchmark) { + for (let current = benchmark; current !== null; current = current.parent) { + if (current.only) return true; + } + return false; + } + + #getSkip(benchmark) { + for (let current = benchmark; current !== null; current = current.parent) { + if (current.skip !== undefined && current.skip !== false) { + return current.skip; + } + } + if (this.#hasOnly && !this.#hasSelectedAncestor(benchmark)) return 'only'; + if (this.namePattern !== null) { + this.namePattern.lastIndex = 0; + if (RegExpPrototypeExec(this.namePattern, benchmark.fullName) === null) { + return 'name pattern'; + } + } + return null; + } + + #suiteHasActiveBench(suite) { + for (let i = 0; i < suite.children.length; i++) { + const child = suite.children[i]; + if (child instanceof Suite) { + if (this.#suiteHasActiveBench(child)) return true; + } else if (this.#getSkip(child) === null) { + return true; + } + } + return false; + } + + async #invoke(resource, store, fn, args) { + const result = resource.runInAsyncScope(() => this.#storage.run( + store, + () => ReflectApply(fn, undefined, args), + )); + return PromiseResolve(result); + } + + async #runHooks(suite, name, resource, store, context) { + const hooks = suite.hooks[name]; + for (let i = 0; i < hooks.length; i++) { + await this.#invoke(resource, store, hooks[i].fn, [context]); + } + } + + async #runSuiteHooks(suite, name) { + const context = { + __proto__: null, + name: suite.name, + signal: this.outerSignal, + }; + await this.#runHooks(suite, name, suite, suite, context); + } + + #diagnostic(error, loc, level = 'info') { + this.success = false; + this.stream.diagnostic({ + __proto__: null, + message: error?.message ?? `${error}`, + error, + level, + file: loc?.file ?? loc?.[2], + line: loc?.line ?? loc?.[0], + column: loc?.column ?? loc?.[1], + }); + } + + async #completeSubtree(node, error) { + for (let i = 0; i < node.children.length; i++) { + const child = node.children[i]; + if (child instanceof Suite) { + await this.#completeSubtree(child, error); + child.finished = true; + child.completion.resolve(); + child.emitDestroy(); + } else { + await this.#executeBench(child, error); + } + } + } + + async #executeSuite(suite) { + if (suite.buildError !== null) { + this.#diagnostic(suite.buildError, suite.loc, 'error'); + await this.#completeSubtree(suite, suite.buildError); + suite.finished = true; + suite.completion.resolve(); + suite.emitDestroy(); + return; + } + + const active = this.#suiteHasActiveBench(suite); + let beforeError; + if (active) { + try { + await this.#runSuiteHooks(suite, 'before'); + } catch (error) { + beforeError = error; + this.#diagnostic(error, suite.loc, 'error'); + } + } + + if (beforeError !== undefined) { + await this.#completeSubtree(suite, beforeError); + } else { + for (let i = 0; i < suite.children.length; i++) { + const child = suite.children[i]; + if (child instanceof Suite) { + await this.#executeSuite(child); + } else { + await this.#executeBench(child); + } + } + } + + if (active) { + try { + await this.#runSuiteHooks(suite, 'after'); + } catch (error) { + this.#diagnostic(error, suite.loc, 'error'); + } + } + suite.finished = true; + suite.completion.resolve(); + if (!suite.isRoot) suite.emitDestroy(); + } + + #getHookSuites(benchmark) { + const suites = []; + for (let current = benchmark.parent; current !== null; current = current.parent) { + ArrayPrototypePush(suites, current); + } + ArrayPrototypeReverse(suites); + return suites; + } + + async #runBenchHooks(benchmark, name, context) { + const suites = this.#getHookSuites(benchmark); + if (name === 'afterEach') ArrayPrototypeReverse(suites); + for (let i = 0; i < suites.length; i++) { + await this.#runHooks( + suites[i], name, benchmark, benchmark, context); + } + } + + async #runWithStop(benchmark, controller, callback) { + const signals = []; + if (this.outerSignal !== undefined) { + ArrayPrototypePush(signals, this.outerSignal); + } + if (benchmark.outerSignal !== undefined && + benchmark.outerSignal !== this.outerSignal) { + ArrayPrototypePush(signals, benchmark.outerSignal); + } + + for (let i = 0; i < signals.length; i++) { + if (signals[i].aborted) { + const error = createAbortError(signals[i]); + controller.abort(error); + throw error; + } + } + + const stop = PromiseWithResolvers(); + const listeners = []; + let timer; + for (let i = 0; i < signals.length; i++) { + const signal = signals[i]; + ArrayPrototypePush(listeners, addAbortListener(signal, () => { + const error = createAbortError(signal); + controller.abort(error); + stop.reject(error); + })); + } + if (benchmark.timeout !== Infinity) { + timer = setTimeout(() => { + const error = new ERR_OPERATION_FAILED( + `Benchmark timed out after ${benchmark.timeout}ms`); + controller.abort(error); + stop.reject(error); + }, benchmark.timeout); + } + + const work = callback(); + try { + if (signals.length === 0 && timer === undefined) return await work; + return await SafePromiseRace([work, stop.promise]); + } finally { + if (timer !== undefined) clearTimeout(timer); + for (let i = 0; i < listeners.length; i++) { + listeners[i][SymbolDispose](); + } + } + } + + async #runSample(benchmark, signal) { + const context = new BenchContext(benchmark, signal); + try { + await this.#invoke( + benchmark, benchmark, benchmark.fn, [context]); + return context.finish(); + } catch (error) { + context.close(); + throw error; + } + } + + #createResult(benchmark, samples, extra = kEmptyObject) { + return { + __proto__: null, + benchId: benchmark.benchId, + parentId: benchmark.parentId, + name: benchmark.name, + file: benchmark.loc.file, + line: benchmark.loc.line, + column: benchmark.loc.column, + tags: ArrayPrototypeSlice(benchmark.tags), + params: benchmark.params, + samples, + ...extra, + }; + } + + #recordResult(benchmark, result) { + benchmark.finished = true; + benchmark.result = result; + this.stream.complete(result); + benchmark.completion.resolve(result); + benchmark.emitDestroy(); + } + + async #executeBench(benchmark, forcedError = undefined) { + const duplicateError = this.#duplicateErrors.get(benchmark); + if (duplicateError !== undefined) { + this.success = false; + this.counts.failed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + [], + { __proto__: null, error: duplicateError }, + )); + return; + } + + const skip = this.#getSkip(benchmark); + if (skip !== null) { + this.counts.skipped++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + [], + { __proto__: null, skip }, + )); + return; + } + + if (forcedError !== undefined) { + this.success = false; + this.counts.failed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + [], + { __proto__: null, error: forcedError }, + )); + return; + } + + this.stream.start({ + __proto__: null, + benchId: benchmark.benchId, + parentId: benchmark.parentId, + name: benchmark.name, + file: benchmark.loc.file, + line: benchmark.loc.line, + column: benchmark.loc.column, + tags: ArrayPrototypeSlice(benchmark.tags), + params: benchmark.params, + }); + + const controller = new AbortController(); + const samples = []; + const hookContext = { + __proto__: null, + name: benchmark.name, + params: benchmark.params, + signal: controller.signal, + }; + let error; + + try { + await this.#runWithStop(benchmark, controller, async () => { + try { + await this.#runBenchHooks( + benchmark, 'beforeEach', hookContext); + const total = benchmark.warmup + benchmark.samples; + for (let i = 0; i < total; i++) { + if (controller.signal.aborted) { + throw controller.signal.reason; + } + const sample = await this.#runSample( + benchmark, controller.signal); + if (controller.signal.aborted) { + throw controller.signal.reason; + } + if (i >= benchmark.warmup) { + ArrayPrototypePush(samples, sample); + this.stream.sample({ + __proto__: null, + benchId: benchmark.benchId, + parentId: benchmark.parentId, + name: benchmark.name, + index: i - benchmark.warmup, + ...sample, + }); + } + if (i + 1 < total) await eventLoopTurn(); + } + } finally { + await this.#runBenchHooks( + benchmark, 'afterEach', hookContext); + } + }); + } catch (cause) { + error = cause; + } finally { + controller.abort(); + } + + if (error !== undefined) { + this.success = false; + this.counts.failed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + samples, + { __proto__: null, error }, + )); + return; + } + + this.counts.completed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + samples, + { __proto__: null, summary: summarizeSamples(samples) }, + )); + } + + #finish(startTime) { + if (this.state === 'finished') return; + this.state = 'finished'; + const duration = startTime === undefined ? 0n : hrtime() - startTime; + this.stream.summary({ + __proto__: null, + success: this.success, + counts: this.counts, + duration_ns: duration, + file: this.entryFile, + }); + this.stream.end(); + this.root.finished = true; + this.root.completion.resolve(); + this.root.emitDestroy(); + this.#storage.disable(); + if (!this.#explicitRun && !this.success) { + process.exitCode = kGenericUserError; + } + } + + async #execute() { + this.state = 'building'; + const startTime = hrtime(); + await this.#waitForBuild(); + this.#prepare(); + this.state = 'running'; + await this.#executeSuite(this.root); + this.#finish(startTime); + } +} + +let globalHarness; + +function lazyHarness() { + globalHarness ??= new Harness(); + return globalHarness; +} + +function runInParentContext(type) { + const declare = (name, options, fn, overrides = kEmptyObject) => { + const harness = lazyHarness(); + const loc = getCallerLocation(); + const declarationOptions = { __proto__: null, ...overrides, loc }; + return type === 'benchmark' ? + harness.createBench(name, options, fn, declarationOptions) : + harness.createSuite(name, options, fn, declarationOptions); + }; + + if (type === 'benchmark') { + declare.skip = (name, options, fn) => declare( + name, options, fn, { __proto__: null, skip: true }); + declare.only = (name, options, fn) => declare( + name, options, fn, { __proto__: null, only: true }); + } + return declare; +} + +function hook(name) { + return (fn, options) => lazyHarness().createHook(name, fn, options); +} + +const bench = runInParentContext('benchmark'); +const suite = runInParentContext('suite'); + +function runBenchmarks(options) { + return lazyHarness().run(options); +} + +module.exports = { + Harness, + after: hook(kHookNames[0]), + afterEach: hook(kHookNames[1]), + before: hook(kHookNames[2]), + beforeEach: hook(kHookNames[3]), + bench, + runBenchmarks, + suite, +}; diff --git a/lib/internal/bench_runner/runner.js b/lib/internal/bench_runner/runner.js new file mode 100644 index 000000000000..ee55672b5f11 --- /dev/null +++ b/lib/internal/bench_runner/runner.js @@ -0,0 +1,12 @@ +'use strict'; + +const { kEmptyObject } = require('internal/util'); +const { runBenchmarks } = require('internal/bench_runner/harness'); + +function run(options = kEmptyObject) { + return runBenchmarks(options); +} + +module.exports = { + run, +}; diff --git a/lib/internal/bootstrap/realm.js b/lib/internal/bootstrap/realm.js index 8a4d179806aa..2761e2846f1f 100644 --- a/lib/internal/bootstrap/realm.js +++ b/lib/internal/bootstrap/realm.js @@ -124,6 +124,7 @@ const legacyWrapperList = new SafeSet([ // beginning with "internal/". // Modules that can only be imported via the node: scheme. const schemelessBlockList = new SafeSet([ + 'bench', 'dtls', 'ffi', 'sea', diff --git a/test/module-hooks/test-module-hooks-builtin-require.js b/test/module-hooks/test-module-hooks-builtin-require.js index 2086cbe062b0..b623f4157bea 100644 --- a/test/module-hooks/test-module-hooks-builtin-require.js +++ b/test/module-hooks/test-module-hooks-builtin-require.js @@ -11,6 +11,7 @@ const assert = require('assert'); const { registerHooks } = require('module'); const schemelessBlockList = new Set([ + 'bench', 'sea', 'test', 'test/reporters', diff --git a/test/module-hooks/test-module-hooks-load-builtin-require.js b/test/module-hooks/test-module-hooks-load-builtin-require.js index 962080b3c2c8..262aa1a0d32b 100644 --- a/test/module-hooks/test-module-hooks-load-builtin-require.js +++ b/test/module-hooks/test-module-hooks-load-builtin-require.js @@ -35,6 +35,7 @@ hook.deregister(); // the one with the `node:` prefix. The one with the prefix // stripped for internal lookups should not get passed into the hooks. const schemelessBlockList = new Set([ + 'bench', 'sea', 'test', 'test/reporters', diff --git a/test/parallel/test-bench-auto-run.js b/test/parallel/test-bench-auto-run.js new file mode 100644 index 000000000000..ed02fed9340a --- /dev/null +++ b/test/parallel/test-bench-auto-run.js @@ -0,0 +1,28 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const { bench } = require('node:bench'); + +const child = spawnSync(process.execPath, [ + '--no-warnings', + '-e', + 'require("node:bench").bench("failure", () => { throw new Error(); })', +]); +assert.strictEqual(child.status, 1); + +const completion = bench('automatic execution', common.mustCall((b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}, 30)); + +completion.then(common.mustCall((result) => { + assert.strictEqual(result.name, 'automatic execution'); + assert.strictEqual(result.samples.length, 30); + assert.strictEqual(result.error, undefined); + assert.strictEqual(result.skip, undefined); + assert.strictEqual(result.summary.mean > 0, true); +})); diff --git a/test/parallel/test-bench-errors.js b/test/parallel/test-bench-errors.js new file mode 100644 index 000000000000..7a6f7aec7e16 --- /dev/null +++ b/test/parallel/test-bench-errors.js @@ -0,0 +1,104 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { bench, run } = require('node:bench'); + +const options = { samples: 1 }; + +bench('missing start', options, () => {}); +bench('missing end', options, (b) => b.start()); +bench('end before start', options, (b) => b.end(1)); +bench('duplicate start', options, (b) => { + b.start(); + b.start(); +}); +bench('duplicate end', options, (b) => { + b.start(); + b.end(1); + b.end(1); +}); +bench('invalid operations', options, (b) => { + b.start(); + b.end(0); +}); +bench('throws', options, () => { + throw new Error('benchmark failure'); +}); +bench('timeout', { samples: 1, timeout: 10 }, async () => { + await new Promise(() => {}); +}); +bench('late timeout', { samples: 1, timeout: 5 }, async (b) => { + b.start(); + await new Promise((resolve) => setTimeout(resolve, 30)); + b.end(1); +}); + +const signal = AbortSignal.abort(new Error('stop')); +bench('aborted', { samples: 1, signal }, () => {}); + +function complete(b) { + b.start(); + process.hrtime.bigint(); + b.end(1); +} + +bench('duplicate', { samples: 1, params: { value: 1 } }, complete); +bench('duplicate', { samples: 1, params: { value: 1 } }, complete); +bench('continues', options, complete); + +const completions = []; +const sampleNames = []; +let summary; +const stream = run(); +stream.on('bench:complete', (result) => completions.push(result)); +stream.on('bench:sample', (sample) => sampleNames.push(sample.name)); +stream.on('bench:summary', (result) => { summary = result; }); +stream.on('end', common.mustCall(() => { + assert.strictEqual(completions.length, 13); + assert.deepStrictEqual(summary.counts, { + __proto__: null, + completed: 2, + failed: 11, + skipped: 0, + total: 13, + }); + assert.strictEqual(summary.success, false); + + const byName = new Map(); + for (const result of completions) { + const values = byName.get(result.name) ?? []; + values.push(result); + byName.set(result.name, values); + } + + assert.match(byName.get('missing start')[0].error.message, + /did not call start/); + assert.match(byName.get('missing end')[0].error.message, + /did not call end/); + assert.match(byName.get('end before start')[0].error.message, + /before start/); + assert.strictEqual(byName.get('duplicate start')[0].error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('duplicate end')[0].error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('invalid operations')[0].error.code, + 'ERR_OUT_OF_RANGE'); + assert.strictEqual(byName.get('throws')[0].error.message, + 'benchmark failure'); + assert.strictEqual(byName.get('timeout')[0].error.code, + 'ERR_OPERATION_FAILED'); + assert.strictEqual(byName.get('late timeout')[0].error.code, + 'ERR_OPERATION_FAILED'); + assert.strictEqual(byName.get('aborted')[0].error.code, 'ABORT_ERR'); + + const duplicates = byName.get('duplicate'); + assert.strictEqual(duplicates[0].error, undefined); + assert.match(duplicates[1].error.message, /duplicate benchmark identity/); + assert.strictEqual(byName.get('continues')[0].error, undefined); + setTimeout(common.mustCall(() => { + assert.strictEqual(sampleNames.includes('late timeout'), false); + }), 40); +})); +stream.resume(); diff --git a/test/parallel/test-bench-filtering.js b/test/parallel/test-bench-filtering.js new file mode 100644 index 000000000000..41784c0dafef --- /dev/null +++ b/test/parallel/test-bench-filtering.js @@ -0,0 +1,41 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { bench, run, suite } = require('node:bench'); + +const calls = []; + +function complete(name) { + return (b) => { + calls.push(name); + b.start(); + process.hrtime.bigint(); + b.end(1); + }; +} + +suite('selected', { only: true }, () => { + bench('included', { samples: 1 }, complete('included')); + bench.skip('explicitly skipped', { samples: 1 }, + common.mustNotCall()); + bench('pattern filtered', { samples: 1 }, + common.mustNotCall()); +}); +bench('only filtered', { samples: 1 }, common.mustNotCall()); + +const results = []; +const stream = run({ namePattern: /^selected (included|explicitly skipped)$/ }); +stream.on('bench:complete', (result) => results.push(result)); +stream.on('end', common.mustCall(() => { + assert.deepStrictEqual(calls, ['included']); + assert.strictEqual(results.length, 4); + + const byName = new Map(results.map((result) => [result.name, result])); + assert.strictEqual(byName.get('included').error, undefined); + assert.strictEqual(byName.get('explicitly skipped').skip, true); + assert.strictEqual(byName.get('pattern filtered').skip, 'name pattern'); + assert.strictEqual(byName.get('only filtered').skip, 'only'); +})); +stream.resume(); diff --git a/test/parallel/test-bench-hook-errors.js b/test/parallel/test-bench-hook-errors.js new file mode 100644 index 000000000000..72de600b8abd --- /dev/null +++ b/test/parallel/test-bench-hook-errors.js @@ -0,0 +1,84 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + after, + afterEach, + before, + beforeEach, + bench, + run, + suite, +} = require('node:bench'); + +function complete(b) { + b.start(); + process.hrtime.bigint(); + b.end(1); +} + +suite('before failure', () => { + before(() => { throw new Error('before failure'); }); + after(common.mustCall()); + bench('blocked by before', { samples: 1 }, common.mustNotCall()); +}); + +suite('beforeEach failure', () => { + beforeEach(() => { throw new Error('beforeEach failure'); }); + afterEach(common.mustCall()); + bench('blocked by beforeEach', { samples: 1 }, common.mustNotCall()); +}); + +suite('after failure', () => { + after(() => { throw new Error('after failure'); }); + bench('completes before after', { samples: 1 }, complete); +}); + +suite('build failure', async () => { + await new Promise((resolve) => setImmediate(resolve)); + throw new Error('build failure'); +}); + +bench('continues after suite failures', { samples: 1 }, complete); + +const completions = []; +const diagnostics = []; +let summary; +const stream = run(); +stream.on('bench:complete', (result) => completions.push(result)); +stream.on('bench:diagnostic', (diagnostic) => { + diagnostics.push(diagnostic); +}); +stream.on('bench:summary', (value) => { summary = value; }); +stream.on('end', common.mustCall(() => { + assert.deepStrictEqual(summary.counts, { + __proto__: null, + completed: 2, + failed: 2, + skipped: 0, + total: 4, + }); + assert.strictEqual(summary.success, false); + + const byName = new Map(completions.map((result) => [result.name, result])); + assert.strictEqual(byName.get('blocked by before').error.message, + 'before failure'); + assert.strictEqual(byName.get('blocked by beforeEach').error.message, + 'beforeEach failure'); + assert.strictEqual(byName.get('completes before after').error, undefined); + assert.strictEqual( + byName.get('continues after suite failures').error, undefined); + + assert.deepStrictEqual( + diagnostics.map(({ message }) => message).sort(), + ['after failure', 'before failure', 'build failure'], + ); + for (const diagnostic of diagnostics) { + assert.strictEqual(typeof diagnostic.file, 'string'); + assert.strictEqual(typeof diagnostic.line, 'number'); + assert.strictEqual(typeof diagnostic.column, 'number'); + } +})); +stream.resume(); diff --git a/test/parallel/test-bench-module.mjs b/test/parallel/test-bench-module.mjs new file mode 100644 index 000000000000..ec08413ed6b3 --- /dev/null +++ b/test/parallel/test-bench-module.mjs @@ -0,0 +1,41 @@ +// Flags: --no-warnings + +import '../common/index.mjs'; +import assert from 'node:assert'; +import { createRequire, builtinModules, isBuiltin } from 'node:module'; +import benchDefault, { + after, + afterEach, + before, + beforeEach, + bench, + describe, + run, + suite, +} from 'node:bench'; + +assert.strictEqual(benchDefault, bench); +assert.strictEqual(describe, suite); +for (const value of [ + after, + afterEach, + before, + beforeEach, + bench, + run, + suite, +]) { + assert.strictEqual(typeof value, 'function'); +} +assert.strictEqual(typeof bench.skip, 'function'); +assert.strictEqual(typeof bench.only, 'function'); + +assert.strictEqual(isBuiltin('node:bench'), true); +assert.strictEqual(isBuiltin('bench'), false); +assert.strictEqual(builtinModules.includes('node:bench'), true); +assert.strictEqual(process.getBuiltinModule('node:bench'), benchDefault); +assert.strictEqual(process.getBuiltinModule('bench'), undefined); + +const require = createRequire(import.meta.url); +assert.throws(() => require('bench'), { code: 'MODULE_NOT_FOUND' }); +await assert.rejects(import('bench'), { code: 'ERR_MODULE_NOT_FOUND' }); diff --git a/test/parallel/test-bench-run.js b/test/parallel/test-bench-run.js new file mode 100644 index 000000000000..fafc9e3309ad --- /dev/null +++ b/test/parallel/test-bench-run.js @@ -0,0 +1,139 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + after, + afterEach, + before, + beforeEach, + bench, + run, + suite, +} = require('node:bench'); + +const calls = []; +const contexts = new Set(); +let active = false; + +before(() => calls.push('root before')); +after(() => calls.push('root after')); +beforeEach(() => calls.push('root beforeEach')); +afterEach(() => calls.push('root afterEach')); + +const suiteCompletion = suite('group', { tags: ['Group'] }, async () => { + await new Promise((resolve) => setImmediate(resolve)); + + before(() => calls.push('suite before')); + after(() => calls.push('suite after')); + beforeEach(() => calls.push('suite beforeEach')); + afterEach(() => calls.push('suite afterEach')); + + bench('sync', { + params: { z: 2, a: true }, + samples: 2, + tags: ['SYNC'], + warmup: 1, + }, common.mustCall((b) => { + assert.strictEqual(active, false); + active = true; + contexts.add(b); + calls.push('sync sample'); + assert.deepStrictEqual(b.params, { __proto__: null, a: true, z: 2 }); + b.start(); + process.hrtime.bigint(); + b.end(1); + active = false; + }, 3)); + + bench('async', { samples: 2 }, common.mustCall(async (b) => { + assert.strictEqual(active, false); + active = true; + contexts.add(b); + calls.push('async sample'); + await new Promise((resolve) => setImmediate(resolve)); + b.start(); + process.hrtime.bigint(); + b.end(1); + await new Promise((resolve) => setImmediate(resolve)); + active = false; + }, 2)); + + bench.skip('skipped', { samples: 1 }, common.mustNotCall()); +}); + +const records = []; +const stream = run(); +stream.on('data', (record) => records.push(record)); +stream.on('end', common.mustCall(() => { + assert.strictEqual(active, false); + assert.strictEqual(contexts.size, 5); + + const starts = records.filter(({ type }) => type === 'bench:start'); + const samples = records.filter(({ type }) => type === 'bench:sample'); + const completions = records.filter(({ type }) => type === 'bench:complete'); + const summaries = records.filter(({ type }) => type === 'bench:summary'); + + assert.strictEqual(starts.length, 2); + assert.strictEqual(samples.length, 4); + assert.strictEqual(completions.length, 3); + assert.strictEqual(summaries.length, 1); + + const sync = completions.find(({ data }) => data.name === 'sync').data; + assert.strictEqual(sync.error, undefined); + assert.strictEqual(sync.skip, undefined); + assert.strictEqual(sync.samples.length, 2); + assert.strictEqual(Object.getPrototypeOf(sync), null); + assert.strictEqual(Object.getPrototypeOf(sync.params), null); + assert.deepStrictEqual(sync.tags, ['group', 'sync']); + assert.match(sync.benchId, /\{"a":true,"z":2\}/); + assert.notStrictEqual(sync.parentId, null); + assert.strictEqual(sync.summary.mean > 0, true); + assert.strictEqual(sync.summary.min <= sync.summary.mean, true); + assert.strictEqual(sync.summary.mean <= sync.summary.max, true); + assert.strictEqual(typeof sync.summary.confidenceInterval.lower, 'number'); + assert.strictEqual(typeof sync.samples[0].duration_ns, 'bigint'); + + const asyncResult = completions.find( + ({ data }) => data.name === 'async').data; + assert.strictEqual(asyncResult.error, undefined); + assert.strictEqual(asyncResult.skip, undefined); + assert.strictEqual(asyncResult.samples.length, 2); + + const skipped = completions.find( + ({ data }) => data.name === 'skipped').data; + assert.strictEqual(skipped.skip, true); + assert.deepStrictEqual(skipped.samples, []); + + assert.deepStrictEqual(summaries[0].data.counts, { + __proto__: null, + completed: 2, + failed: 0, + skipped: 1, + total: 3, + }); + assert.strictEqual(summaries[0].data.success, true); + + assert.deepStrictEqual(calls, [ + 'root before', + 'suite before', + 'root beforeEach', + 'suite beforeEach', + 'sync sample', + 'sync sample', + 'sync sample', + 'suite afterEach', + 'root afterEach', + 'root beforeEach', + 'suite beforeEach', + 'async sample', + 'async sample', + 'suite afterEach', + 'root afterEach', + 'suite after', + 'root after', + ]); +})); + +suiteCompletion.then(common.mustCall()); diff --git a/test/parallel/test-bench-validation.js b/test/parallel/test-bench-validation.js new file mode 100644 index 000000000000..25a53c1b0186 --- /dev/null +++ b/test/parallel/test-bench-validation.js @@ -0,0 +1,45 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { bench, run } = require('node:bench'); + +const noop = () => {}; + +assert.throws(() => bench('', noop), { code: 'ERR_INVALID_ARG_VALUE' }); +assert.throws(() => bench('name', null), { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { samples: 0 }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { warmup: -1 }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { timeout: -1 }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { signal: {} }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { tags: 'fast' }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { tags: [''] }, noop), + { code: 'ERR_INVALID_ARG_VALUE' }); +assert.throws(() => bench('name', { params: { value: null } }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { params: { value: NaN } }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { only: 'yes' }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { skip: 1 }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => run({ namePattern: 1 }), + { code: 'ERR_INVALID_ARG_TYPE' }); + +bench('valid', { samples: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); + +const stream = run(); +stream.on('bench:start', common.mustCall(() => { + assert.throws(() => bench('late', noop), { code: 'ERR_INVALID_STATE' }); +})); +stream.resume(); diff --git a/test/parallel/test-module-isBuiltin.js b/test/parallel/test-module-isBuiltin.js index a7815a8dfc1c..54f25e599858 100644 --- a/test/parallel/test-module-isBuiltin.js +++ b/test/parallel/test-module-isBuiltin.js @@ -7,10 +7,12 @@ const { isBuiltin } = require('module'); assert(isBuiltin('http')); assert(isBuiltin('sys')); assert(isBuiltin('node:fs')); +assert(isBuiltin('node:bench')); assert(isBuiltin('node:test')); // Does not include internal modules assert(!isBuiltin('internal/errors')); +assert(!isBuiltin('bench')); assert(!isBuiltin('test')); assert(!isBuiltin('')); assert(!isBuiltin(undefined)); From a2f10c53a85b29502090ca9ce3a41d4cc90ec46b Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 27 Aug 2026 22:29:00 +0000 Subject: [PATCH 3/9] lib: implement bench/reporters Signed-off-by: James M Snell Assisted-by: Opencode --- doc/api/bench.md | 60 +++++++ lib/bench/reporters.js | 34 ++++ lib/internal/bench_runner/reporter/json.js | 64 ++++++++ lib/internal/bench_runner/reporter/spec.js | 149 ++++++++++++++++++ lib/internal/bootstrap/realm.js | 1 + .../test-module-hooks-builtin-require.js | 1 + .../test-module-hooks-load-builtin-require.js | 1 + test/parallel/test-bench-custom-reporter.js | 37 +++++ test/parallel/test-bench-module.mjs | 13 ++ test/parallel/test-bench-reporters.js | 114 ++++++++++++++ test/parallel/test-module-isBuiltin.js | 2 + 11 files changed, 476 insertions(+) create mode 100644 lib/bench/reporters.js create mode 100644 lib/internal/bench_runner/reporter/json.js create mode 100644 lib/internal/bench_runner/reporter/spec.js create mode 100644 test/parallel/test-bench-custom-reporter.js create mode 100644 test/parallel/test-bench-reporters.js diff --git a/doc/api/bench.md b/doc/api/bench.md index 93b303fb2e2d..b4488e119cd1 100644 --- a/doc/api/bench.md +++ b/doc/api/bench.md @@ -65,6 +65,66 @@ system load can all affect results. Keep raw samples when comparing results and investigate noisy or skewed distributions rather than treating a confidence interval as a pass/fail threshold. +## Benchmark reporters + +The built-in reporters are available from the scheme-only +`node:bench/reporters` module: + +```mjs +import { json, spec } from 'node:bench/reporters'; +``` + +```cjs +const { json, spec } = require('node:bench/reporters'); +``` + +Reporter values can be passed directly to `stream.compose()`: + +```mjs +import { bench, run } from 'node:bench'; +import { spec } from 'node:bench/reporters'; +import process from 'node:process'; + +bench('example', (b) => { + b.start(); + doWork(); + b.end(1); +}); + +run().compose(spec).pipe(process.stdout); +``` + +The `spec` reporter buffers results and outputs a concise table containing the +sample count, mean rate, 95% confidence interval for the mean, median rate, and +warnings. A coefficient of variation above 5% is reported as `noisy`, and an +absolute skewness above 1 is reported as `skewed`. The exact human-readable +format is subject to change. + +The `json` reporter emits every lifecycle record as newline-delimited JSON. +BigInt values, including `duration_ns`, are encoded as decimal strings. Errors +are represented using their `name`, `message`, `stack`, `code`, `cause`, and +`errors` properties. As required by JSON, non-finite numbers are encoded as +`null`. + +Custom reporters use the same composition contract. They can be transforms or +functions accepted by `stream.compose()`. The composed readable can be piped to +any writable destination: + +```mjs +import { run } from 'node:bench'; +import process from 'node:process'; + +async function* names(source) { + for await (const { type, data } of source) { + if (type === 'bench:complete') { + yield `${data.name}\n`; + } + } +} + +run().compose(names).pipe(process.stdout); +``` + ## `bench([name][, options], fn)` + +> Stability: 1 - Experimental + +Starts the Node.js command-line benchmark runner. At least one explicit file or +glob pattern is required: + +```console +node --bench benchmark.mjs +node --bench 'benchmarks/**/*.js' +``` + +Quote glob patterns to prevent expansion by the shell. Matching files are +sorted and executed serially. By default, each file runs in a separate child +process. Benchmark files declare benchmarks using `node:bench`; they must not +call `run()` themselves. See the [benchmark runner][] documentation for more +details. + +This flag cannot be combined with `--test`, `--watch`, `--watch-path`, +`--check`, `--eval`, or `--interactive`. + +### `--bench-isolation=mode` + + + +> Stability: 1 - Experimental + +Configures benchmark file isolation. When `mode` is `'process'`, each matching +file runs in a separate child process. This is the default. Files are still run +serially so their measured work does not overlap. + +When `mode` is `'none'`, all matching files and benchmarks run serially in the +benchmark runner process. This reduces startup overhead but allows module, +heap, and process state to carry between files. User writes to stdout or stderr +also share destinations with benchmark reporters in this mode. + +### `--bench-name-pattern=pattern` + + + +> Stability: 1 - Experimental + +Only runs benchmarks whose full hierarchical name matches the JavaScript +regular expression `pattern`. Non-matching benchmarks are reported as skipped. + +### `--bench-reporter-destination=destination` + + + +> Stability: 1 - Experimental + +Specifies the destination for the corresponding benchmark reporter. The value +can be `stdout`, `stderr`, or a file path. A single reporter defaults to +`stdout` when no destination is specified. + +### `--bench-reporter=reporter` + + + +> Stability: 1 - Experimental + +Specifies a benchmark reporter. The built-in reporters are `spec` and `json`. +The `json` reporter emits newline-delimited JSON. A custom reporter can be +specified using a module specifier resolved from the current working directory. + +This option can be repeated. When multiple reporters are specified, each must +have a corresponding `--bench-reporter-destination`. The default reporter is +`spec`. + +### `--bench-samples=count` + + + +> Stability: 1 - Experimental + +Overrides the number of measured callback invocations for every selected +benchmark. `count` must be an integer between `1` and `4294967295`. + +### `--bench-warmup=count` + + + +> Stability: 1 - Experimental + +Overrides the number of unreported warmup callback invocations for every +selected benchmark. `count` must be an integer between `0` and `4294967295`. + ### `--build-sea=config` -> Stability: 1 - Experimental +> Stability: 1.0 - Early Development diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index a31e0e82aabc..beac2624b62b 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -17,6 +17,7 @@ * [Using `--analyze` (no external tools needed)](#using---analyze-no-external-tools-needed) * [Using R scripts or node-benchmark-compare](#using-r-scripts-or-node-benchmark-compare) * [Comparing parameters](#comparing-parameters) + * [Evaluating `node:bench` ports](#evaluating-nodebench-ports) * [Running benchmarks on the CI](#running-benchmarks-on-the-ci) * [Creating a benchmark](#creating-a-benchmark) * [Basics of a benchmark](#basics-of-a-benchmark) @@ -588,6 +589,41 @@ chunkLen encoding rate confidence.interval ![compare tool boxplot](doc_img/scatter-plot.png) +### Evaluating `node:bench` ports + +The experimental `compare-node-bench.js` and `scatter-node-bench.js` tools are +parallel versions of the existing tools for explicit `node:bench` files. They +do not modify or replace the legacy benchmark framework. Each repeated +observation for a benchmark identity uses one measured sample from a separate +process invocation. Configurations declared in the same file still execute +serially in that process, unlike the legacy framework's configuration-level +process isolation, and can share runtime state. + +Both parallel tools support inline analysis. `scatter-node-bench.js --analyze` +uses the same `--xaxis`, `--category`, and `--no-chart` interface described for +`scatter.js`. `compare-node-bench.js --analyze` performs Welch's t-test, while +`--max-regression N` adds a corrected regression gate. The gate requires both a +Holm-Bonferroni-adjusted p-value below 0.05 and a 95% confidence interval lying +entirely beyond `-N%`; the point estimate alone cannot fail the command. +Scatter analysis reduces aggregated configurations to one value per outer +process and uses disjoint process sets for consecutive Mann-Whitney comparisons +so configurations sharing a process are not treated as independent samples. + +Underscore-prefixed ports are kept beside selected legacy benchmarks and are +excluded from legacy discovery. For example: + +```console +./node benchmark/scatter.js --runs 30 \ + benchmark/crypto/create-hash.js > legacy.csv +./node benchmark/scatter-node-bench.js --runs 30 -- \ + benchmark/crypto/_create-hash.node-bench.js > node-bench.csv +``` + +The port uses the legacy relative filename as its benchmark name and preserves +the same parameter names. The two CSV files can therefore be analyzed with the +same scripts to check whether their rate distributions and measurement units +agree. See [`benchmark/README.md`][] for compare and scatter examples. + ### Running benchmarks on the CI To see the performance impact of a pull request by running benchmarks on @@ -769,6 +805,7 @@ Supported options keys are: * `benchmarker` - benchmarker to use, defaults to the first available http benchmarker +[`benchmark/README.md`]: ../../benchmark/README.md#nodebench-evaluation-tools [autocannon]: https://github.com/mcollina/autocannon [benchmark-ci]: https://github.com/nodejs/benchmarking/blob/HEAD/docs/core_benchmarks.md [git-for-windows]: https://git-scm.com/download/win diff --git a/test/fixtures/bench-runner/tools-collision.cjs b/test/fixtures/bench-runner/tools-collision.cjs new file mode 100644 index 000000000000..5fb27dd956d7 --- /dev/null +++ b/test/fixtures/bench-runner/tools-collision.cjs @@ -0,0 +1,14 @@ +'use strict'; + +const { bench, suite } = require('node:bench'); + +function register(name) { + bench(name, { params: { size: 1 } }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); + }); +} + +suite('first', () => register('same')); +suite('second', () => register('same')); diff --git a/test/fixtures/bench-runner/tools-no-params.cjs b/test/fixtures/bench-runner/tools-no-params.cjs new file mode 100644 index 000000000000..2dde0c2a0c69 --- /dev/null +++ b/test/fixtures/bench-runner/tools-no-params.cjs @@ -0,0 +1,9 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('tools/no-params.js', (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/tools-reserved-param.cjs b/test/fixtures/bench-runner/tools-reserved-param.cjs new file mode 100644 index 000000000000..00052b0c8e4b --- /dev/null +++ b/test/fixtures/bench-runner/tools-reserved-param.cjs @@ -0,0 +1,9 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('tools/reserved.js', { params: { rate: 'parameter' } }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/tools.cjs b/test/fixtures/bench-runner/tools.cjs new file mode 100644 index 000000000000..108a6f81dd63 --- /dev/null +++ b/test/fixtures/bench-runner/tools.cjs @@ -0,0 +1,20 @@ +'use strict'; + +const { bench } = require('node:bench'); + +if (process.env.NODE_BENCH_PID_LOG !== undefined) { + require('fs').appendFileSync( + process.env.NODE_BENCH_PID_LOG, `${process.pid}\n`); +} + +for (const size of [1, 2]) { + bench('tools/simple.js', { + params: { method: 'loop', size }, + }, (b) => { + let value = 0; + b.start(); + for (let i = 0; i < 1_000; i++) value += size; + b.end(1_000); + if (value === 0) throw new Error('unreachable'); + }); +} diff --git a/test/parallel/test-benchmark-node-bench-tools.js b/test/parallel/test-benchmark-node-bench-tools.js new file mode 100644 index 000000000000..84b5f98e9eec --- /dev/null +++ b/test/parallel/test-benchmark-node-bench-tools.js @@ -0,0 +1,261 @@ +// Flags: --no-warnings +'use strict'; + +require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const fixtures = require('../common/fixtures'); +const tmpdir = require('../common/tmpdir'); +const { + analyzeScatter, + holmAdjust, + isRegressionFailure, +} = require('../../benchmark/_node-bench-analysis.js'); +const { csvEncode } = require('../../benchmark/_node-bench.js'); + +const compare = path.resolve(__dirname, '../../benchmark/compare-node-bench.js'); +const legacyScatter = path.resolve(__dirname, '../../benchmark/scatter.js'); +const scatter = path.resolve(__dirname, '../../benchmark/scatter-node-bench.js'); +const benchmark = fixtures.path('bench-runner/tools.cjs'); + +tmpdir.refresh(); + +assert.strictEqual(csvEncode(true), 'true'); +assert.deepStrictEqual(holmAdjust([0.01, 0.03, 0.04]), [0.03, 0.06, 0.06]); +assert.strictEqual(isRegressionFailure({ + ci95: 3, + improvement: -12, + pAdjusted: 0.01, +}, 10), false); +assert.strictEqual(isRegressionFailure({ + ci95: 1, + improvement: -12, + pAdjusted: 0.06, +}, 10), false); +assert.strictEqual(isRegressionFailure({ + ci95: 1, + improvement: -12, + pAdjusted: 0.01, +}, 10), true); +assert.throws( + () => analyzeScatter([{ + observation: 0, + params: { size: 1 }, + rate: 1, + }], 'size', 'size', false), + /must name different parameters/, +); +assert.doesNotMatch(analyzeScatter([0, 1].map((observation) => ({ + observation, + params: { size: 1 }, + rate: 1_234_567.89, +})), 'size', undefined, false), /\(!\)/); +assert.match(analyzeScatter([ + { observation: 0, params: { method: 'a', size: 1 }, rate: 10 }, + { observation: 0, params: { method: 'b', size: 1 }, rate: 20 }, + { observation: 1, params: { method: 'a', size: 1 }, rate: 30 }, + { observation: 1, params: { method: 'b', size: 1 }, rate: 50 }, +], 'size', undefined, false), /\n\s*1\s+2\s+/); + +function run(script, args, options = undefined) { + return spawnSync(process.execPath, [script, ...args], { + encoding: 'utf8', + timeout: 30_000, + ...options, + }); +} + +{ + const pidLog = tmpdir.resolve('pids'); + const result = run(compare, [ + '--old', process.execPath, + '--new', process.execPath, + '--runs', '2', + '--', benchmark, + ], { + env: { __proto__: null, ...process.env, NODE_BENCH_PID_LOG: pidLog }, + }); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], + '"binary","filename","configuration","rate","time"'); + assert.strictEqual(lines.length, 9); + assert.strictEqual(lines.filter((line) => line.startsWith('"old",')).length, + 4); + assert.strictEqual(lines.filter((line) => line.startsWith('"new",')).length, + 4); + assert.deepStrictEqual(lines.slice(1).map((line) => line.slice(0, 5)), [ + '"old"', '"old"', '"new"', '"new"', + '"new"', '"new"', '"old"', '"old"', + ]); + assert(lines.slice(1).every( + (line) => line.includes('"tools/simple.js"'))); + const pids = fs.readFileSync(pidLog, 'utf8').trim().split('\n'); + assert.strictEqual(new Set(pids).size, 4); +} + +{ + const result = run(scatter, [ + '--node', process.execPath, + '--runs', '2', + '--', benchmark, + ]); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], + '"filename","method","size","rate","time"'); + assert.strictEqual(lines.length, 5); + assert(lines.slice(1).every( + (line) => line.startsWith('"tools/simple.js","loop",'))); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', fixtures.path('bench-runner/tools-no-params.cjs'), + ]); + assert.strictEqual(result.status, 0, result.stderr); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], '"filename","rate","time"'); + assert.strictEqual(lines.length, 2); + assert.strictEqual(lines[1].split(',').length, 3); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', fixtures.path('bench-runner/tools-reserved-param.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /parameter 'rate' is reserved/); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', fixtures.path('bench-runner/tools-collision.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /requires one logical benchmark name per file/); +} + +{ + const result = run(compare, [ + '--old', process.execPath, + '--new', process.execPath, + '--runs', '1', + '--', fixtures.path('bench-runner/tools-collision.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /Distinct benchmarks would share the CSV group/); +} + +{ + const result = run(scatter, [ + '--runs', '2', + '--', fixtures.path('bench-runner/a.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /set of reported benchmarks changed between runs/); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--name-pattern', 'missing', + '--', benchmark, + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /No benchmark samples were produced/); +} + +{ + const result = run(scatter, [ + '--runs', 'invalid', + '--', benchmark, + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /--runs must be an integer/); +} + +{ + const result = run(scatter, [ + '--runs', '2', + '--analyze', + '--xaxis', 'size', + '--category', 'method', + '--no-chart', + '--', benchmark, + ]); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + assert.match(result.stdout, + /size\s+method\s+samples\s+rate\s+confidence\.interval/); + assert.match(result.stdout, /Change between consecutive size values/); + assert.match(result.stdout, /Mann-Whitney U.*Cliff's delta/); + assert.doesNotMatch(result.stdout, /"filename","method"/); +} + +{ + const result = run(scatter, [ + '--analyze', + '--', benchmark, + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /--analyze requires --xaxis/); +} + +{ + const result = run(compare, [ + '--old', process.execPath, + '--new', process.execPath, + '--runs', '2', + '--max-regression', '100', + '--', fixtures.path('bench-runner/tools-no-params.cjs'), + ]); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + assert.match(result.stdout, /confidence\s+improvement\s+accuracy/); + assert.match(result.stdout, /Holm-Bonferroni correction/); + assert.match(result.stdout, /--max-regression uses the corrected values/); + assert.doesNotMatch(result.stdout, /"binary","filename"/); +} + +{ + const legacy = run(legacyScatter, [ + '--runs', '1', + path.resolve(__dirname, '../../benchmark/crypto/create-hash.js'), + ]); + const modern = run(scatter, [ + '--runs', '1', + '--', path.resolve( + __dirname, '../../benchmark/crypto/_create-hash.node-bench.js'), + ]); + assert.strictEqual(legacy.status, 0, legacy.stderr); + assert.strictEqual(modern.status, 0, modern.stderr); + const legacyLines = legacy.stdout.trim().split('\n'); + const modernLines = modern.stdout.trim().split('\n'); + assert.strictEqual(legacyLines[0].replaceAll(' ', ''), modernLines[0]); + const name = path.join('crypto', 'create-hash.js'); + assert(legacyLines[1].startsWith(`"${name}",`)); + assert(modernLines[1].startsWith(`"${name}",`)); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', path.resolve( + __dirname, + '../../benchmark/buffers/_buffer-compare-offset.node-bench.js', + ), + ]); + assert.strictEqual(result.status, 0, result.stderr); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], + '"filename","method","n","size","rate","time"'); + assert.strictEqual(lines.length, 9); +} From 0990a3212d5ee5b29d07e00ba3028e2c8f1919bc Mon Sep 17 00:00:00 2001 From: James M Snell Date: Fri, 28 Aug 2026 01:29:13 +0000 Subject: [PATCH 6/9] src: fixup histogram and options linting issues Signed-off-by: James M Snell --- src/histogram.cc | 4 ++-- src/node_options.h | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/histogram.cc b/src/histogram.cc index 1008c63c04d4..f2b93d3b8be4 100644 --- a/src/histogram.cc +++ b/src/histogram.cc @@ -501,8 +501,8 @@ Histogram::MeanCIResult Histogram::MeanCI(double confidence) const { static_cast(count - 1); double standard_error = std::sqrt(variance / static_cast(count)); double alpha = 1.0 - confidence; - double t_crit = StudentTUpperQuantile( - alpha / 2.0, static_cast(count - 1)); + double t_crit = + StudentTUpperQuantile(alpha / 2.0, static_cast(count - 1)); double margin = t_crit * standard_error; return {mean, mean - margin, mean + margin}; } diff --git a/src/node_options.h b/src/node_options.h index 2f399eac03cc..6b04cb3898ea 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -530,13 +530,12 @@ class OptionsParser { OptionEnvvarSettings env_setting = kDisallowedInEnvvar, bool default_is_true = false, OptionNamespaces namespace_id = OptionNamespaces::kNoNamespace); - void AddOption( - const char* name, - const char* help_text, - uint64_t Options::*field, - OptionEnvvarSettings env_setting = kDisallowedInEnvvar, - OptionNamespaces namespace_id = OptionNamespaces::kNoNamespace, - bool strict = false); + void AddOption(const char* name, + const char* help_text, + uint64_t Options::*field, + OptionEnvvarSettings env_setting = kDisallowedInEnvvar, + OptionNamespaces namespace_id = OptionNamespaces::kNoNamespace, + bool strict = false); void AddOption( const char* name, const char* help_text, From 8bbb4714dbd8cee802b863ab8b84a5322d847ea9 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Fri, 28 Aug 2026 16:03:14 +0000 Subject: [PATCH 7/9] lib: add `node:bench` explicit createRunner Makes it easier for benchmark tools to build on top of the bench runner primitives. Signed-off-by: James M Snell Assisted-by: Opencode --- doc/api/bench.md | 163 +++++++++++++++-- doc/api/cli.md | 5 +- doc/node.1 | 5 +- lib/bench.js | 6 +- lib/internal/bench_runner/benchmark.js | 141 +++++++++++++-- lib/internal/bench_runner/harness.js | 165 ++++++++++++++---- lib/internal/bench_runner/runner.js | 6 +- .../fixtures/bench-runner/recorded-detail.cjs | 17 ++ test/parallel/test-bench-cli.js | 19 ++ test/parallel/test-bench-context-control.js | 86 +++++++++ test/parallel/test-bench-context-errors.js | 82 +++++++++ test/parallel/test-bench-create-runner.js | 96 ++++++++++ test/parallel/test-bench-validation.js | 10 +- .../test-bench-yield-between-samples.js | 79 +++++++++ 14 files changed, 805 insertions(+), 75 deletions(-) create mode 100644 test/fixtures/bench-runner/recorded-detail.cjs create mode 100644 test/parallel/test-bench-context-control.js create mode 100644 test/parallel/test-bench-context-errors.js create mode 100644 test/parallel/test-bench-create-runner.js create mode 100644 test/parallel/test-bench-yield-between-samples.js diff --git a/doc/api/bench.md b/doc/api/bench.md index 4acea0f98c9d..7cb5720ffe28 100644 --- a/doc/api/bench.md +++ b/doc/api/bench.md @@ -63,18 +63,55 @@ exit code is set to `1`. ## Measurement model Each warmup and measured sample invokes the benchmark function once with a -fresh {BenchContext}. The function must call `context.start()` and -`context.end(operations)` exactly once. Setup before `start()` and cleanup after -`end()` are outside the measured region. Promise-returning functions are -awaited. - -An event loop turn occurs between sample invocations. The runner executes +fresh {BenchContext}. The function must either call `context.start()` and +`context.end(operations)` exactly once, or call `context.record(sample)` exactly +once to provide an externally measured sample. Setup before `start()` and +cleanup after `end()` are outside the measured region. Promise-returning +functions are awaited. + +By default, an event loop turn occurs between sample invocations. An embedded +runner can disable this using `yieldBetweenSamples`. The runner executes benchmarks serially, but it does not provide process isolation. Other work in the process, JIT compilation, garbage collection, CPU frequency changes, and system load can all affect results. Keep raw samples when comparing results and investigate noisy or skewed distributions rather than treating a confidence interval as a pass/fail threshold. +Calling `context.done()` during a measured sample completes the benchmark after +that sample. This allows a higher-level tool to treat `samples` as a maximum and +implement a dynamic sampling policy. + +## Reusable runners + +The module-level declaration functions use a shared runner and schedule it +automatically. Higher-level tools can create isolated, explicitly started +runners instead: + +```mjs +import { createRunner } from 'node:bench'; + +const runner = createRunner({ yieldBetweenSamples: false }); + +runner.bench('example', { samples: 100 }, (b) => { + const operations = chooseOperationCount(); + b.start(); + runOperations(operations); + const sample = b.end(operations); + + if (hasEnoughData(sample)) b.done(); +}); + +for await (const record of runner.run()) { + // Consume structured benchmark records. +} +``` + +Each runner has independent declarations, hooks, filtering, and output. Unlike +the module-level declarations, creating a benchmark on an explicit runner does +not schedule execution. This allows packages to collect declarations and start +them later. Calling the explicit runner's `run()` function prevents additional +declarations and a second call to `run()` is an error. + ## Command-line runner The `--bench` flag runs one or more explicit benchmark files or glob patterns: @@ -160,6 +197,29 @@ async function* names(source) { run().compose(names).pipe(process.stdout); ``` +## `createRunner([options])` + + + +* `options` {Object} + * `yieldBetweenSamples` {boolean} Schedule an event loop turn between sample + callbacks. Disabling this also prevents timer-based abort signals from + firing between synchronous callbacks. Benchmark timeouts continue to be + checked against a monotonic deadline. **Default:** `true`. +* Returns: {Object} An isolated benchmark runner with bound `after`, `afterEach`, + `before`, `beforeEach`, `bench`, `describe`, `run`, and `suite` functions. + +Creates an explicitly started benchmark runner. Declarations made through one +runner do not interact with declarations made through another runner or through +the module-level functions. Call the returned `run()` function to start the +runner and obtain its {BenchmarksStream}. + +Each runner can be started once. Its `run()` function accepts the same options +as the module-level [`run()`][]. `run({ yieldBetweenSamples })` overrides the +value passed to `createRunner()`. + ## `bench([name][, options], fn)` + +* {number} + +The zero-based invocation index within the current `context.phase`. Warmup and +measured samples have separate index sequences. + ### `context.name` + +* {string} + +The current sample phase. It is `'warmup'` for an unreported warmup invocation +and `'measurement'` for a measured invocation. + ### `context.signal` + +* `sample` {Object} + * `operations` {number} The number of completed operations. Must be a positive + safe integer. + * `duration_ns` {bigint} An externally measured positive duration in + nanoseconds no greater than `Number.MAX_SAFE_INTEGER`. + * `detail` {any} Additional structured-cloneable sample data. With CLI process + isolation, it must also be supported by advanced child process + serialization. +* Returns: {Object} The normalized sample, including its computed `rate` and + optional cloned `detail`. + +Records a measurement made by another clock or execution environment. This is +useful when a higher-level tool measures work in a worker and needs to exclude +message transport from the duration. `record()` is mutually exclusive with +`start()` and `end()` within one callback and must be called exactly once. + +### `context.done()` + + + +Requests successful benchmark completion after the current measured sample. +The callback must still call either `start()` and `end()`, or `record()`. +Calling `done()` during a warmup invocation is an error. The configured +`samples` value remains the maximum number of measured invocations if `done()` +is not called. ## Class: `BenchmarksStream` @@ -422,9 +551,10 @@ files. Each measured sample has the following properties: * `operations` {number} The positive operation count passed to - `context.end()`. + `context.end()` or `context.record()`. * `duration_ns` {bigint} The measured duration in nanoseconds. * `rate` {number} Operations per second. +* `detail` {any} The optional cloned sample detail. ## Benchmark result @@ -452,5 +582,6 @@ A completed benchmark result contains: interval for the median rate, with `lower` and `upper` properties. * `skewness` {number} The skewness of the scaled rate histogram. +[`run()`]: #runoptions [benchmark result]: #benchmark-result [command-line options documentation]: cli.md#--bench diff --git a/doc/api/cli.md b/doc/api/cli.md index cc88c901e817..cc8a664be7d7 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -539,8 +539,9 @@ added: REPLACEME > Stability: 1 - Experimental -Overrides the number of measured callback invocations for every selected -benchmark. `count` must be an integer between `1` and `4294967295`. +Overrides the maximum number of measured callback invocations for every +selected benchmark. A benchmark may finish earlier by calling +`context.done()`. `count` must be an integer between `1` and `4294967295`. ### `--bench-warmup=count` diff --git a/doc/node.1 b/doc/node.1 index e052606ab955..d73436902158 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -323,8 +323,9 @@ have a corresponding \fB--bench-reporter-destination\fR. The default reporter is \fBspec\fR. . .It Fl -bench-samples Ns = Ns Ar count -Overrides the number of measured callback invocations for every selected -benchmark. \fBcount\fR must be an integer between \fB1\fR and \fB4294967295\fR. +Overrides the maximum number of measured callback invocations for every +selected benchmark. A benchmark may finish earlier by calling +\fBcontext.done()\fR. \fBcount\fR must be an integer between \fB1\fR and \fB4294967295\fR. . .It Fl -bench-warmup Ns = Ns Ar count Overrides the number of unreported warmup callback invocations for every diff --git a/lib/bench.js b/lib/bench.js index cc3aa686ee57..9d1bd24fcebd 100644 --- a/lib/bench.js +++ b/lib/bench.js @@ -13,7 +13,10 @@ const { bench, suite, } = require('internal/bench_runner/harness'); -const { run } = require('internal/bench_runner/runner'); +const { + createRunner, + run, +} = require('internal/bench_runner/runner'); if (process.env.NODE_BENCH_CONTEXT !== 'child' || typeof process.send !== 'function') { @@ -27,6 +30,7 @@ ObjectAssign(module.exports, { before, beforeEach, bench, + createRunner, describe: suite, run, suite, diff --git a/lib/internal/bench_runner/benchmark.js b/lib/internal/bench_runner/benchmark.js index 27f372d7295f..d3c55c1b2ef5 100644 --- a/lib/internal/bench_runner/benchmark.js +++ b/lib/internal/bench_runner/benchmark.js @@ -43,6 +43,7 @@ const { validateString, validateUint32, } = require('internal/validators'); +const { structuredClone } = require('internal/worker/js_transferable'); const { bigint: hrtime } = process.hrtime; const kDefaultSamples = 30; @@ -138,6 +139,19 @@ function getNamePath(parent, name) { return path; } +function cloneSample(sample) { + const result = { + __proto__: null, + operations: sample.operations, + duration_ns: sample.duration_ns, + rate: sample.rate, + }; + if (sample.detail !== undefined) { + result.detail = structuredClone(sample.detail); + } + return result; +} + class Suite extends AsyncResource { constructor(harness, parent, name, options, fn, loc, isRoot = false) { super('BenchSuite'); @@ -219,16 +233,30 @@ class Bench extends AsyncResource { class BenchContext { #closed = false; + #done = false; #endCalled = false; + #index; #invalid = false; + #phase; + #recordCalled = false; #sample = null; #startCalled = false; #startTime; - constructor(bench, signal) { + constructor(bench, signal, phase, index) { this.name = bench.name; this.params = bench.params; this.signal = signal; + this.#phase = phase; + this.#index = index; + } + + get index() { + return this.#index; + } + + get phase() { + return this.#phase; } start() { @@ -236,6 +264,11 @@ class BenchContext { this.#invalid = true; throw new ERR_INVALID_STATE('benchmark sample is no longer active'); } + if (this.#recordCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'start() cannot be combined with record()'); + } if (this.#startCalled) { this.#invalid = true; throw new ERR_INVALID_STATE( @@ -245,12 +278,17 @@ class BenchContext { this.#startTime = hrtime(); } - end(operations) { + end(operations, options = kEmptyObject) { const endTime = hrtime(); if (this.#closed) { this.#invalid = true; throw new ERR_INVALID_STATE('benchmark sample is no longer active'); } + if (this.#recordCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'end() cannot be combined with record()'); + } if (this.#endCalled) { this.#invalid = true; throw new ERR_INVALID_STATE( @@ -263,24 +301,93 @@ class BenchContext { } try { + validateObject(options, 'options'); + const { detail } = options; validateInteger(operations, 'operations', 1, NumberMAX_SAFE_INTEGER); + const duration = endTime - this.#startTime; + if (duration === 0n) { + throw new ERR_INVALID_STATE( + 'insufficient clock precision for benchmark sample'); + } + this.#sample = { + __proto__: null, + operations, + duration_ns: duration, + rate: operations / (Number(duration) / 1e9), + }; + if (detail !== undefined) { + this.#sample.detail = structuredClone(detail); + } } catch (error) { this.#invalid = true; throw error; } + return cloneSample(this.#sample); + } - const duration = endTime - this.#startTime; - if (duration === 0n) { + record(sample) { + if (this.#closed) { + this.#invalid = true; + throw new ERR_INVALID_STATE('benchmark sample is no longer active'); + } + if (this.#recordCalled) { this.#invalid = true; throw new ERR_INVALID_STATE( - 'insufficient clock precision for benchmark sample'); + 'record() must be called exactly once per benchmark sample'); } - this.#sample = { - __proto__: null, - operations, - duration_ns: duration, - rate: operations / (Number(duration) / 1e9), - }; + if (this.#startCalled || this.#endCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'record() cannot be combined with start() or end()'); + } + this.#recordCalled = true; + + try { + validateObject(sample, 'sample'); + const { detail, duration_ns, operations } = sample; + validateInteger( + operations, 'sample.operations', 1, NumberMAX_SAFE_INTEGER); + if (typeof duration_ns !== 'bigint') { + throw new ERR_INVALID_ARG_TYPE( + 'sample.duration_ns', 'bigint', duration_ns); + } + if (duration_ns <= 0n) { + throw new ERR_OUT_OF_RANGE( + 'sample.duration_ns', 'a positive bigint', duration_ns); + } + if (duration_ns > 9_007_199_254_740_991n) { + throw new ERR_OUT_OF_RANGE( + 'sample.duration_ns', + 'less than or equal to Number.MAX_SAFE_INTEGER', + duration_ns); + } + this.#sample = { + __proto__: null, + operations, + duration_ns, + rate: operations / (Number(duration_ns) / 1e9), + }; + if (detail !== undefined) { + this.#sample.detail = structuredClone(detail); + } + } catch (error) { + this.#invalid = true; + throw error; + } + return cloneSample(this.#sample); + } + + done() { + if (this.#closed) { + this.#invalid = true; + throw new ERR_INVALID_STATE('benchmark sample is no longer active'); + } + if (this.#phase !== 'measurement') { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'done() can only be called during a measured sample'); + } + this.#done = true; } finish() { @@ -289,15 +396,19 @@ class BenchContext { throw new ERR_INVALID_STATE( 'benchmark sample violated the start()/end() contract'); } - if (!this.#startCalled) { + if (!this.#recordCalled && !this.#startCalled) { throw new ERR_INVALID_STATE( - 'benchmark callback did not call start()'); + 'benchmark callback did not call start() or record()'); } - if (!this.#endCalled || this.#sample === null) { + if (!this.#recordCalled && (!this.#endCalled || this.#sample === null)) { throw new ERR_INVALID_STATE( 'benchmark callback did not call end()'); } - return this.#sample; + return { + __proto__: null, + done: this.#done, + sample: this.#sample, + }; } close() { diff --git a/lib/internal/bench_runner/harness.js b/lib/internal/bench_runner/harness.js index 1e6dac738cb4..59b85860c2e9 100644 --- a/lib/internal/bench_runner/harness.js +++ b/lib/internal/bench_runner/harness.js @@ -4,7 +4,9 @@ const { ArrayPrototypePush, ArrayPrototypeReverse, ArrayPrototypeSlice, + BigInt, FunctionPrototypeCall, + MathCeil, Promise, PromisePrototypeThen, PromiseResolve, @@ -35,6 +37,7 @@ const { const { isRegExp } = require('internal/util/types'); const { validateAbortSignal, + validateBoolean, validateFunction, validateObject, validateUint32, @@ -65,16 +68,34 @@ function createAbortError(signal) { return new AbortError(undefined, { __proto__: null, cause: signal.reason }); } +function createTimeoutError(benchmark) { + return new ERR_OPERATION_FAILED( + `Benchmark timed out after ${benchmark.timeout}ms`); +} + class Harness { + #autoRun; #buildPromises = []; #duplicateErrors = new SafeMap(); #explicitRun = false; #hasOnly = false; #runPromise = null; #scheduled = false; + #starting = false; #storage = new AsyncLocalStorage(); + #yieldBetweenSamples; - constructor() { + constructor(options = kEmptyObject) { + validateObject(options, 'options'); + const { + autoRun = true, + yieldBetweenSamples = true, + } = options; + validateBoolean(autoRun, 'options.autoRun'); + validateBoolean( + yieldBetweenSamples, 'options.yieldBetweenSamples'); + this.#autoRun = autoRun; + this.#yieldBetweenSamples = yieldBetweenSamples; this.entryFile = process.argv?.[1]; this.stream = new BenchmarksStream(); this.state = 'collecting'; @@ -102,9 +123,10 @@ class Harness { } #ensureCollecting() { - if (this.state === 'collecting' || - (this.state === 'building' && - this.#storage.getStore() instanceof Suite)) return; + if (this.state === 'building' && + this.#storage.getStore() instanceof Suite) return; + if (!this.#explicitRun && !this.#starting && + this.state === 'collecting') return; throw new ERR_INVALID_STATE( 'benchmarks cannot be declared after execution has started'); } @@ -186,19 +208,23 @@ class Harness { configure(options = kEmptyObject) { validateObject(options, 'options'); - if (this.#runPromise !== null) { - if (options !== kEmptyObject) { - throw new ERR_INVALID_STATE('benchmark execution has already started'); - } - return; - } - const { namePattern, samples, signal, warmup } = options; + const { + namePattern, + samples, + signal, + warmup, + yieldBetweenSamples, + } = options; + let nextNamePattern = this.namePattern; + let nextSamples = this.samples; + let nextWarmup = this.warmup; + let nextYieldBetweenSamples = this.#yieldBetweenSamples; if (namePattern !== undefined) { if (typeof namePattern === 'string') { - this.namePattern = new RegExp(namePattern); + nextNamePattern = new RegExp(namePattern); } else if (isRegExp(namePattern)) { - this.namePattern = namePattern; + nextNamePattern = namePattern; } else { throw new ERR_INVALID_ARG_TYPE( 'options.namePattern', ['string', 'RegExp'], namePattern); @@ -206,15 +232,30 @@ class Harness { } if (samples !== undefined) { validateUint32(samples, 'options.samples', true); - this.samples = samples; + nextSamples = samples; } validateAbortSignal(signal, 'options.signal'); - this.outerSignal = signal; if (warmup !== undefined) { validateUint32(warmup, 'options.warmup'); - this.warmup = warmup; + nextWarmup = warmup; + } + if (yieldBetweenSamples !== undefined) { + validateBoolean( + yieldBetweenSamples, 'options.yieldBetweenSamples'); + nextYieldBetweenSamples = yieldBetweenSamples; + } + + this.namePattern = nextNamePattern; + this.samples = nextSamples; + this.outerSignal = signal; + this.warmup = nextWarmup; + this.#yieldBetweenSamples = nextYieldBetweenSamples; + } + + #ensureCanRun() { + if (this.#explicitRun || this.#starting || this.state !== 'collecting') { + throw new ERR_INVALID_STATE('benchmark execution has already started'); } - this.#explicitRun = true; } run(options = kEmptyObject, force = false) { @@ -222,13 +263,21 @@ class Harness { throw new ERR_INVALID_STATE( 'run() cannot be called from a file run with --bench'); } - this.configure(options); - this.#schedule(force); + this.#ensureCanRun(); + this.#starting = true; + try { + this.configure(options); + this.#explicitRun = true; + } finally { + this.#starting = false; + } + this.#schedule(force, true); return this.stream; } - #schedule(force = false) { + #schedule(force = false, explicit = false) { if (!force && kIsCliRunner) return; + if (!explicit && !this.#autoRun) return; if (this.#scheduled) return; this.#scheduled = true; queueMicrotask(() => { @@ -454,8 +503,7 @@ class Harness { } if (benchmark.timeout !== Infinity) { timer = setTimeout(() => { - const error = new ERR_OPERATION_FAILED( - `Benchmark timed out after ${benchmark.timeout}ms`); + const error = createTimeoutError(benchmark); controller.abort(error); stop.reject(error); }, benchmark.timeout); @@ -473,8 +521,9 @@ class Harness { } } - async #runSample(benchmark, signal) { - const context = new BenchContext(benchmark, signal); + async #runSample(benchmark, signal, phase, index) { + const context = new BenchContext( + benchmark, signal, phase, index); try { await this.#invoke( benchmark, benchmark, benchmark.fn, [context]); @@ -557,6 +606,15 @@ class Harness { }); const controller = new AbortController(); + const deadline = benchmark.timeout === Infinity ? + null : hrtime() + BigInt(MathCeil(benchmark.timeout * 1e6)); + const checkDeadline = () => { + if (deadline !== null && hrtime() >= deadline) { + const timeoutError = createTimeoutError(benchmark); + controller.abort(timeoutError); + throw timeoutError; + } + }; const samples = []; const hookContext = { __proto__: null, @@ -571,14 +629,18 @@ class Harness { try { await this.#runBenchHooks( benchmark, 'beforeEach', hookContext); + checkDeadline(); const warmup = this.warmup ?? benchmark.warmup; const total = warmup + (this.samples ?? benchmark.samples); for (let i = 0; i < total; i++) { if (controller.signal.aborted) { throw controller.signal.reason; } - const sample = await this.#runSample( - benchmark, controller.signal); + const phase = i < warmup ? 'warmup' : 'measurement'; + const index = phase === 'warmup' ? i : i - warmup; + const { done, sample } = await this.#runSample( + benchmark, controller.signal, phase, index); + checkDeadline(); if (controller.signal.aborted) { throw controller.signal.reason; } @@ -593,11 +655,15 @@ class Harness { ...sample, }); } - if (i + 1 < total) await eventLoopTurn(); + if (done) break; + if (i + 1 < total && this.#yieldBetweenSamples) { + await eventLoopTurn(); + } } } finally { await this.#runBenchHooks( benchmark, 'afterEach', hookContext); + checkDeadline(); } }); } catch (cause) { @@ -664,9 +730,9 @@ function lazyHarness() { return globalHarness; } -function runInParentContext(type) { +function createDeclaration(type, getHarness) { const declare = (name, options, fn, overrides = kEmptyObject) => { - const harness = lazyHarness(); + const harness = getHarness(); const loc = getCallerLocation(); const declarationOptions = { __proto__: null, ...overrides, loc }; return type === 'benchmark' ? @@ -683,12 +749,36 @@ function runInParentContext(type) { return declare; } -function hook(name) { - return (fn, options) => lazyHarness().createHook(name, fn, options); +function createHook(name, getHarness) { + return (fn, options) => getHarness().createHook(name, fn, options); } -const bench = runInParentContext('benchmark'); -const suite = runInParentContext('suite'); +const bench = createDeclaration('benchmark', lazyHarness); +const suite = createDeclaration('suite', lazyHarness); + +function createRunner(options = kEmptyObject) { + validateObject(options, 'options'); + const harness = new Harness({ + __proto__: null, + autoRun: false, + yieldBetweenSamples: options.yieldBetweenSamples === undefined ? + true : options.yieldBetweenSamples, + }); + const getHarness = () => harness; + const runnerBench = createDeclaration('benchmark', getHarness); + const runnerSuite = createDeclaration('suite', getHarness); + return { + __proto__: null, + after: createHook(kHookNames[0], getHarness), + afterEach: createHook(kHookNames[1], getHarness), + before: createHook(kHookNames[2], getHarness), + beforeEach: createHook(kHookNames[3], getHarness), + bench: runnerBench, + describe: runnerSuite, + run: (runOptions = kEmptyObject) => harness.run(runOptions), + suite: runnerSuite, + }; +} function runBenchmarks(options, force) { return lazyHarness().run(options, force); @@ -696,11 +786,12 @@ function runBenchmarks(options, force) { module.exports = { Harness, - after: hook(kHookNames[0]), - afterEach: hook(kHookNames[1]), - before: hook(kHookNames[2]), - beforeEach: hook(kHookNames[3]), + after: createHook(kHookNames[0], lazyHarness), + afterEach: createHook(kHookNames[1], lazyHarness), + before: createHook(kHookNames[2], lazyHarness), + beforeEach: createHook(kHookNames[3], lazyHarness), bench, + createRunner, runBenchmarks, suite, }; diff --git a/lib/internal/bench_runner/runner.js b/lib/internal/bench_runner/runner.js index ee55672b5f11..fc8595d2a35d 100644 --- a/lib/internal/bench_runner/runner.js +++ b/lib/internal/bench_runner/runner.js @@ -1,12 +1,16 @@ 'use strict'; const { kEmptyObject } = require('internal/util'); -const { runBenchmarks } = require('internal/bench_runner/harness'); +const { + createRunner, + runBenchmarks, +} = require('internal/bench_runner/harness'); function run(options = kEmptyObject) { return runBenchmarks(options); } module.exports = { + createRunner, run, }; diff --git a/test/fixtures/bench-runner/recorded-detail.cjs b/test/fixtures/bench-runner/recorded-detail.cjs new file mode 100644 index 000000000000..1643783225d0 --- /dev/null +++ b/test/fixtures/bench-runner/recorded-detail.cjs @@ -0,0 +1,17 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('recorded detail', { samples: 3 }, (b) => { + b.record({ + __proto__: null, + detail: { + index: b.index, + phase: b.phase, + value: 42n, + }, + duration_ns: 4n, + operations: 2, + }); + b.done(); +}); diff --git a/test/parallel/test-bench-cli.js b/test/parallel/test-bench-cli.js index 5a170ab9c674..8ed5afddad67 100644 --- a/test/parallel/test-bench-cli.js +++ b/test/parallel/test-bench-cli.js @@ -145,6 +145,25 @@ function parseOutput(output) { assert.strictEqual(records.at(-1).data.success, false); } +{ + const result = spawnBench([ + '--bench-reporter=json', + fixtures.path('bench-runner/recorded-detail.cjs'), + ]); + assert.strictEqual(result.status, 0); + const records = parseRecords(result); + const completion = records.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(completion.samples.length, 1); + const { rate, ...sample } = completion.samples[0]; + assert.deepStrictEqual(sample, { + detail: { index: 0, phase: 'measurement', value: '42' }, + duration_ns: '4', + operations: 2, + }); + assert(Math.abs(rate - 500_000_000) < 1); +} + for (const { file, status } of [ { file: 'a.cjs', status: 0 }, { file: 'error.cjs', status: 1 }, diff --git a/test/parallel/test-bench-context-control.js b/test/parallel/test-bench-context-control.js new file mode 100644 index 000000000000..ec28923f874f --- /dev/null +++ b/test/parallel/test-bench-context-control.js @@ -0,0 +1,86 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { createRunner } = require('node:bench'); + +(async () => { + const runner = createRunner({ yieldBetweenSamples: false }); + const invocations = []; + let closedContext; + + const controlledCompletion = runner.bench('controlled', { + samples: 5, + warmup: 2, + }, common.mustCall((b) => { + invocations.push(`${b.phase}:${b.index}`); + const detail = { index: b.index, phase: b.phase }; + b.start(); + process.hrtime.bigint(); + const sample = b.end(2, { detail }); + detail.index = -1; + + assert.strictEqual(sample.operations, 2); + assert.strictEqual(typeof sample.duration_ns, 'bigint'); + assert.strictEqual(sample.rate > 0, true); + assert.notStrictEqual(sample.detail, detail); + assert.notStrictEqual(sample.detail.index, -1); + sample.operations = 0; + sample.rate = NaN; + sample.detail.index = -2; + + if (b.phase === 'measurement' && b.index === 1) { + b.done(); + closedContext = b; + } + }, 4)); + + const recordedCompletion = runner.bench( + 'recorded', { samples: 3 }, common.mustCall((b) => { + const detail = { source: 'worker', value: 1n }; + const sample = b.record({ + __proto__: null, + detail, + duration_ns: 20n, + operations: 5, + }); + detail.source = 'changed'; + assert.deepStrictEqual(sample, { + __proto__: null, + detail: { source: 'worker', value: 1n }, + duration_ns: 20n, + operations: 5, + rate: 250_000_000, + }); + sample.operations = 0; + sample.rate = NaN; + sample.detail.source = 'returned value changed'; + b.done(); + })); + + const records = await runner.run().toArray(); + const [controlled, recorded] = await Promise.all([ + controlledCompletion, + recordedCompletion, + ]); + + assert.deepStrictEqual(invocations, [ + 'warmup:0', + 'warmup:1', + 'measurement:0', + 'measurement:1', + ]); + assert.strictEqual(controlled.samples.length, 2); + assert.deepStrictEqual( + controlled.samples.map(({ operations }) => operations), [2, 2]); + assert.deepStrictEqual( + controlled.samples.map(({ detail }) => detail.index), [0, 1]); + assert.strictEqual(controlled.samples.every(({ rate }) => rate > 0), true); + assert.strictEqual(recorded.samples.length, 1); + assert.deepStrictEqual(recorded.samples[0].detail, + { source: 'worker', value: 1n }); + assert.strictEqual( + records.filter(({ type }) => type === 'bench:sample').length, 3); + assert.throws(() => closedContext.done(), { code: 'ERR_INVALID_STATE' }); +})().then(common.mustCall()); diff --git a/test/parallel/test-bench-context-errors.js b/test/parallel/test-bench-context-errors.js new file mode 100644 index 000000000000..40aca29bfcc5 --- /dev/null +++ b/test/parallel/test-bench-context-errors.js @@ -0,0 +1,82 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { createRunner } = require('node:bench'); + +const runner = createRunner({ yieldBetweenSamples: false }); + +runner.bench('done during warmup', { samples: 1, warmup: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); + b.done(); +}); +runner.bench('invalid record', { samples: 1 }, (b) => b.record(null)); +runner.bench('invalid duration type', { samples: 1 }, (b) => b.record({ + duration_ns: 1, + operations: 1, +})); +runner.bench('invalid duration value', { samples: 1 }, (b) => b.record({ + duration_ns: 0n, + operations: 1, +})); +runner.bench('duration too large', { samples: 1 }, (b) => b.record({ + duration_ns: 9_007_199_254_740_992n, + operations: 1, +})); +runner.bench('invalid operations', { samples: 1 }, (b) => b.record({ + duration_ns: 1n, + operations: 0, +})); +runner.bench('mixed timing', { samples: 1 }, (b) => { + b.start(); + b.record({ duration_ns: 1n, operations: 1 }); +}); +runner.bench('duplicate record', { samples: 1 }, (b) => { + b.record({ duration_ns: 1n, operations: 1 }); + b.record({ duration_ns: 1n, operations: 1 }); +}); +runner.bench('reentrant record', { samples: 1 }, (b) => { + const sample = { duration_ns: 1n }; + Object.defineProperty(sample, 'operations', { + get() { + b.record({ duration_ns: 1n, operations: 1 }); + return 1; + }, + }); + b.record(sample); +}); +runner.bench('uncloneable detail', { samples: 1 }, (b) => { + b.record({ duration_ns: 1n, operations: 1, detail: () => {} }); +}); + +(async () => { + const records = await runner.run().toArray(); + const completions = records + .filter(({ type }) => type === 'bench:complete') + .map(({ data }) => data); + const byName = new Map(completions.map((result) => [result.name, result])); + + assert.strictEqual(byName.get('done during warmup').error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('invalid record').error.code, + 'ERR_INVALID_ARG_TYPE'); + assert.strictEqual(byName.get('invalid duration type').error.code, + 'ERR_INVALID_ARG_TYPE'); + assert.strictEqual(byName.get('invalid duration value').error.code, + 'ERR_OUT_OF_RANGE'); + assert.strictEqual(byName.get('duration too large').error.code, + 'ERR_OUT_OF_RANGE'); + assert.strictEqual(byName.get('invalid operations').error.code, + 'ERR_OUT_OF_RANGE'); + assert.strictEqual(byName.get('mixed timing').error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('duplicate record').error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('reentrant record').error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('uncloneable detail').error.name, + 'DataCloneError'); +})().then(common.mustCall()); diff --git a/test/parallel/test-bench-create-runner.js b/test/parallel/test-bench-create-runner.js new file mode 100644 index 000000000000..a8c25533ee63 --- /dev/null +++ b/test/parallel/test-bench-create-runner.js @@ -0,0 +1,96 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { createRunner } = require('node:bench'); + +(async () => { + const first = createRunner({ yieldBetweenSamples: false }); + const second = createRunner({ yieldBetweenSamples: false }); + let firstCalls = 0; + let secondCalls = 0; + + first.before(common.mustCall()); + second.before(common.mustCall()); + + const firstCompletion = first.bench( + 'same name', { samples: 2 }, common.mustCall((b) => { + firstCalls++; + b.start(); + process.hrtime.bigint(); + b.end(1); + }, 2)); + const secondCompletion = second.bench( + 'same name', { samples: 1 }, common.mustCall((b) => { + secondCalls++; + b.start(); + process.hrtime.bigint(); + b.end(1); + })); + + await new Promise((resolve) => setImmediate(resolve)); + assert.strictEqual(firstCalls, 0); + assert.strictEqual(secondCalls, 0); + + const firstStream = first.run(); + const secondStream = second.run(); + assert.throws(() => first.run(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => first.bench('late', common.mustNotCall()), + { code: 'ERR_INVALID_STATE' }); + const [firstRecords, secondRecords] = await Promise.all([ + firstStream.toArray(), + secondStream.toArray(), + ]); + const [firstResult, secondResult] = await Promise.all([ + firstCompletion, + secondCompletion, + ]); + + assert.strictEqual(firstCalls, 2); + assert.strictEqual(secondCalls, 1); + assert.strictEqual(firstResult.samples.length, 2); + assert.strictEqual(secondResult.samples.length, 1); + assert.strictEqual(firstResult.error, undefined); + assert.strictEqual(secondResult.error, undefined); + assert.strictEqual( + firstRecords.filter(({ type }) => type === 'bench:summary').length, 1); + assert.strictEqual( + secondRecords.filter(({ type }) => type === 'bench:summary').length, 1); + assert.strictEqual(typeof first.bench.skip, 'function'); + assert.strictEqual(typeof first.bench.only, 'function'); + assert.strictEqual(first.describe, first.suite); + + const retry = createRunner({ yieldBetweenSamples: false }); + retry.bench('not filtered', { samples: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); + }); + assert.throws(() => retry.run({ + namePattern: 'filtered', + samples: 0, + }), { code: 'ERR_OUT_OF_RANGE' }); + const retryRecords = await retry.run().toArray(); + const retryResult = retryRecords.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(retryResult.skip, undefined); + assert.strictEqual(retryResult.samples.length, 1); + + const reentrant = createRunner({ yieldBetweenSamples: false }); + reentrant.bench('reentrant options', { samples: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); + }); + const reentrantOptions = {}; + Object.defineProperty(reentrantOptions, 'samples', { + get: common.mustCall(() => reentrant.run()), + }); + assert.throws(() => reentrant.run(reentrantOptions), + { code: 'ERR_INVALID_STATE' }); + const reentrantRecords = await reentrant.run().toArray(); + const reentrantResult = reentrantRecords.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(reentrantResult.samples.length, 1); +})().then(common.mustCall()); diff --git a/test/parallel/test-bench-validation.js b/test/parallel/test-bench-validation.js index d276c4cfa650..cdbaee9e6ee6 100644 --- a/test/parallel/test-bench-validation.js +++ b/test/parallel/test-bench-validation.js @@ -3,7 +3,7 @@ const common = require('../common'); const assert = require('assert'); -const { bench, run } = require('node:bench'); +const { bench, createRunner, run } = require('node:bench'); const noop = () => {}; @@ -35,6 +35,14 @@ assert.throws(() => run({ samples: 0 }), { code: 'ERR_OUT_OF_RANGE' }); assert.throws(() => run({ warmup: -1 }), { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => run({ yieldBetweenSamples: 1 }), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => createRunner(null), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => createRunner({ yieldBetweenSamples: 1 }), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => createRunner({ yieldBetweenSamples: null }), + { code: 'ERR_INVALID_ARG_TYPE' }); bench('valid', { samples: 1 }, (b) => { b.start(); diff --git a/test/parallel/test-bench-yield-between-samples.js b/test/parallel/test-bench-yield-between-samples.js new file mode 100644 index 000000000000..f10ae6a2f6c4 --- /dev/null +++ b/test/parallel/test-bench-yield-between-samples.js @@ -0,0 +1,79 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { createRunner } = require('node:bench'); + +async function observe(factoryOptions, runOptions) { + const runner = createRunner(factoryOptions); + const observed = []; + let turnOccurred = false; + const turn = new Promise((resolve) => setImmediate(() => { + turnOccurred = true; + resolve(); + })); + + runner.bench('yielding', { samples: 2 }, (b) => { + observed.push(turnOccurred); + b.start(); + process.hrtime.bigint(); + b.end(1); + }); + + await runner.run(runOptions).toArray(); + await turn; + return observed; +} + +async function observeTimeout() { + const runner = createRunner({ yieldBetweenSamples: false }); + let invocations = 0; + const completion = runner.bench('timeout', { + samples: 10, + timeout: 5, + }, (b) => { + invocations++; + const until = process.hrtime.bigint() + 2_000_000n; + b.start(); + while (process.hrtime.bigint() < until) { /* Busy loop. */ } + b.end(1); + }); + await runner.run().toArray(); + const result = await completion; + assert.strictEqual(result.error.code, 'ERR_OPERATION_FAILED'); + assert.strictEqual(invocations < 10, true); +} + +async function observeAfterEachTimeout() { + const runner = createRunner({ yieldBetweenSamples: false }); + runner.afterEach(common.mustCall(() => { + const until = process.hrtime.bigint() + 10_000_000n; + while (process.hrtime.bigint() < until) { /* Busy loop. */ } + })); + const completion = runner.bench('afterEach timeout', { + samples: 1, + timeout: 5, + }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); + }); + await runner.run().toArray(); + const result = await completion; + assert.strictEqual(result.error.code, 'ERR_OPERATION_FAILED'); +} + +(async () => { + assert.deepStrictEqual(await observe(undefined, undefined), [false, true]); + assert.deepStrictEqual( + await observe({ yieldBetweenSamples: false }, undefined), [false, false]); + assert.deepStrictEqual(await observe( + { yieldBetweenSamples: false }, + { yieldBetweenSamples: true }), [false, true]); + assert.deepStrictEqual(await observe( + { yieldBetweenSamples: true }, + { yieldBetweenSamples: false }), [false, false]); + await observeTimeout(); + await observeAfterEachTimeout(); +})().then(common.mustCall()); From 2af297f74e35dcec8bf853f56257d432847434f3 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Fri, 28 Aug 2026 16:19:13 +0000 Subject: [PATCH 8/9] test: update bench tests to not fail on no-crypto Signed-off-by: James M Snell Assisted-by: Opencode --- test/parallel/test-bench-cli.js | 4 +-- .../test-benchmark-node-bench-tools.js | 35 ++++++++----------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/test/parallel/test-bench-cli.js b/test/parallel/test-bench-cli.js index 8ed5afddad67..3180dcfeab12 100644 --- a/test/parallel/test-bench-cli.js +++ b/test/parallel/test-bench-cli.js @@ -1,6 +1,6 @@ 'use strict'; -require('../common'); +const common = require('../common'); const assert = require('assert'); const { spawnSync } = require('child_process'); const fs = require('fs'); @@ -236,7 +236,7 @@ for (const isolation of ['process', 'none']) { fixtures.path('bench-runner/ipc.cjs')); } -{ +if (common.hasInspector) { const result = spawnBench([ '--inspect=0', '--bench-reporter=json', diff --git a/test/parallel/test-benchmark-node-bench-tools.js b/test/parallel/test-benchmark-node-bench-tools.js index 84b5f98e9eec..90dd02028de3 100644 --- a/test/parallel/test-benchmark-node-bench-tools.js +++ b/test/parallel/test-benchmark-node-bench-tools.js @@ -226,36 +226,31 @@ function run(script, args, options = undefined) { } { + const benchmark = path.resolve( + __dirname, '../../benchmark/buffers/buffer-compare-offset.js'); + const nodeBenchmark = path.resolve( + __dirname, '../../benchmark/buffers/_buffer-compare-offset.node-bench.js'); const legacy = run(legacyScatter, [ '--runs', '1', - path.resolve(__dirname, '../../benchmark/crypto/create-hash.js'), + benchmark, ]); const modern = run(scatter, [ '--runs', '1', - '--', path.resolve( - __dirname, '../../benchmark/crypto/_create-hash.node-bench.js'), + '--', nodeBenchmark, ]); assert.strictEqual(legacy.status, 0, legacy.stderr); assert.strictEqual(modern.status, 0, modern.stderr); const legacyLines = legacy.stdout.trim().split('\n'); const modernLines = modern.stdout.trim().split('\n'); - assert.strictEqual(legacyLines[0].replaceAll(' ', ''), modernLines[0]); - const name = path.join('crypto', 'create-hash.js'); + assert.deepStrictEqual( + legacyLines[0].replaceAll(' ', '').split(',').sort(), + modernLines[0].split(',').sort(), + ); + assert.strictEqual(modernLines[0], + '"filename","method","n","size","rate","time"'); + assert.strictEqual(legacyLines.length, 9); + assert.strictEqual(modernLines.length, 9); + const name = path.join('buffers', 'buffer-compare-offset.js'); assert(legacyLines[1].startsWith(`"${name}",`)); assert(modernLines[1].startsWith(`"${name}",`)); } - -{ - const result = run(scatter, [ - '--runs', '1', - '--', path.resolve( - __dirname, - '../../benchmark/buffers/_buffer-compare-offset.node-bench.js', - ), - ]); - assert.strictEqual(result.status, 0, result.stderr); - const lines = result.stdout.trim().split('\n'); - assert.strictEqual(lines[0], - '"filename","method","n","size","rate","time"'); - assert.strictEqual(lines.length, 9); -} From f5af4430e9377656c85874f51645874986ae5cba Mon Sep 17 00:00:00 2001 From: James M Snell Date: Fri, 28 Aug 2026 17:15:51 +0000 Subject: [PATCH 9/9] test: improve node:bench test coverage Signed-off-by: James M Snell Assisted-by: Opencode --- lib/internal/bench_runner/cli.js | 2 + test/fixtures/bench-runner/abrupt-exit.cjs | 18 ++ .../bench-runner/destroying-reporter.cjs | 8 + test/fixtures/bench-runner/fake-ipc.cjs | 3 + test/fixtures/bench-runner/inspector.cjs | 15 ++ .../bench-runner/malformed-record.cjs | 15 ++ test/fixtures/bench-runner/many-records.cjs | 10 + test/fixtures/bench-runner/send-error.cjs | 17 ++ test/fixtures/bench-runner/slow-reporter.cjs | 35 +++ test/fixtures/bench-runner/throws-null.cjs | 3 + test/fixtures/bench-runner/v8-option.cjs | 13 ++ test/parallel/test-bench-cli.js | 210 +++++++++++++++++- test/parallel/test-bench-clock-precision.js | 22 ++ test/parallel/test-bench-context-control.js | 6 + test/parallel/test-bench-context-errors.js | 20 ++ test/parallel/test-bench-create-runner.js | 3 +- test/parallel/test-bench-errors.js | 7 +- test/parallel/test-bench-harness-errors.js | 113 ++++++++++ test/parallel/test-bench-hook-errors.js | 3 +- test/parallel/test-bench-reporters.js | 62 ++++++ test/parallel/test-bench-run.js | 7 +- test/parallel/test-bench-validation.js | 24 ++ .../test-bench-yield-between-samples.js | 6 +- 23 files changed, 603 insertions(+), 19 deletions(-) create mode 100644 test/fixtures/bench-runner/abrupt-exit.cjs create mode 100644 test/fixtures/bench-runner/destroying-reporter.cjs create mode 100644 test/fixtures/bench-runner/fake-ipc.cjs create mode 100644 test/fixtures/bench-runner/inspector.cjs create mode 100644 test/fixtures/bench-runner/malformed-record.cjs create mode 100644 test/fixtures/bench-runner/many-records.cjs create mode 100644 test/fixtures/bench-runner/send-error.cjs create mode 100644 test/fixtures/bench-runner/slow-reporter.cjs create mode 100644 test/fixtures/bench-runner/throws-null.cjs create mode 100644 test/fixtures/bench-runner/v8-option.cjs create mode 100644 test/parallel/test-bench-clock-precision.js create mode 100644 test/parallel/test-bench-harness-errors.js diff --git a/lib/internal/bench_runner/cli.js b/lib/internal/bench_runner/cli.js index 5a1bb2cbf3a5..ce54be0e1e22 100644 --- a/lib/internal/bench_runner/cli.js +++ b/lib/internal/bench_runner/cli.js @@ -428,6 +428,8 @@ function getChildArgs(path, options) { }, ); ArrayPrototypePushApply(args, unknownExecArgv); + // Option serialization omits port 0, which would otherwise become 9229. + if (process.debugPort === 0) ArrayPrototypePush(args, '--inspect-port=0'); ArrayPrototypePush(args, '--bench', '--bench-isolation=none'); if (options.namePatternSource.length > 0) { ArrayPrototypePush( diff --git a/test/fixtures/bench-runner/abrupt-exit.cjs b/test/fixtures/bench-runner/abrupt-exit.cjs new file mode 100644 index 000000000000..04be350b7259 --- /dev/null +++ b/test/fixtures/bench-runner/abrupt-exit.cjs @@ -0,0 +1,18 @@ +'use strict'; + +const mode = process.env.NODE_BENCH_EXIT_MODE; + +if (mode === 'code') process.exit(2); +if (mode === 'signal') process.kill(process.pid, 'SIGTERM'); + +const { bench } = require('node:bench'); + +if (mode === 'late') { + process.on('beforeExit', () => { process.exitCode = 2; }); +} + +bench('abrupt exit', { samples: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/destroying-reporter.cjs b/test/fixtures/bench-runner/destroying-reporter.cjs new file mode 100644 index 000000000000..9e3c04fb1fb0 --- /dev/null +++ b/test/fixtures/bench-runner/destroying-reporter.cjs @@ -0,0 +1,8 @@ +'use strict'; + +module.exports = async function* destroyingReporter(source) { + source.once('bench:start', () => { + source.destroy(new Error('benchmark reporter closed the stream')); + }); + yield* source; +}; diff --git a/test/fixtures/bench-runner/fake-ipc.cjs b/test/fixtures/bench-runner/fake-ipc.cjs new file mode 100644 index 000000000000..7b2afa8dda93 --- /dev/null +++ b/test/fixtures/bench-runner/fake-ipc.cjs @@ -0,0 +1,3 @@ +'use strict'; + +process.send = () => {}; diff --git a/test/fixtures/bench-runner/inspector.cjs b/test/fixtures/bench-runner/inspector.cjs new file mode 100644 index 000000000000..6ea7a71a2264 --- /dev/null +++ b/test/fixtures/bench-runner/inspector.cjs @@ -0,0 +1,15 @@ +'use strict'; + +const { bench } = require('node:bench'); + +const inspectPort = process.execArgv.filter( + (arg) => arg.startsWith('--inspect-port=')).at(-1); + +bench('inspector option', { + params: { inspectPort }, + samples: 1, +}, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/malformed-record.cjs b/test/fixtures/bench-runner/malformed-record.cjs new file mode 100644 index 000000000000..5744bf1ea4b0 --- /dev/null +++ b/test/fixtures/bench-runner/malformed-record.cjs @@ -0,0 +1,15 @@ +'use strict'; + +const common = require('../../common'); + +const record = process.env.NODE_BENCH_MALFORMED_RECORD === 'summary' ? { + type: 'bench:summary', + data: { + counts: { completed: 0, failed: 0, skipped: 0, total: -1 }, + duration_ns: 1n, + success: true, + }, +} : null; + +process.send?.({ type: 'node:bench:record', record }); +setTimeout(() => process.exit(2), common.platformTimeout(10_000)); diff --git a/test/fixtures/bench-runner/many-records.cjs b/test/fixtures/bench-runner/many-records.cjs new file mode 100644 index 000000000000..e841c39458f7 --- /dev/null +++ b/test/fixtures/bench-runner/many-records.cjs @@ -0,0 +1,10 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('many records', { samples: 30 }, (b) => { + process.stdout.write(`${b.index}\n`); + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/send-error.cjs b/test/fixtures/bench-runner/send-error.cjs new file mode 100644 index 000000000000..a647305fca76 --- /dev/null +++ b/test/fixtures/bench-runner/send-error.cjs @@ -0,0 +1,17 @@ +'use strict'; + +if (process.env.NODE_BENCH_SEND_ERROR === 'callback') { + process.send = (_message, _handle, _options, callback) => { + callback(new Error('benchmark send callback failed')); + }; +} else { + process.send = () => { throw new Error('benchmark send threw'); }; +} + +const { bench } = require('node:bench'); + +bench('send error', { samples: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/slow-reporter.cjs b/test/fixtures/bench-runner/slow-reporter.cjs new file mode 100644 index 000000000000..b5670d28ea70 --- /dev/null +++ b/test/fixtures/bench-runner/slow-reporter.cjs @@ -0,0 +1,35 @@ +'use strict'; + +const { setTimeout } = require('timers/promises'); + +module.exports = async function* slowReporter(source) { + const { promise, resolve } = Promise.withResolvers(); + let emitted = 0; + const onRecord = () => { + if (++emitted === source.readableHighWaterMark) resolve(); + }; + for (const type of [ + 'bench:start', + 'bench:sample', + 'bench:complete', + 'bench:diagnostic', + 'bench:summary', + ]) { + source.on(type, onRecord); + } + await promise; + + let samples = 0; + let stdout = ''; + for await (const record of source) { + await setTimeout(2); + if (record.type === 'bench:sample') samples++; + if (record.type === 'bench:diagnostic' && + record.data.stream === 'stdout') { + stdout += record.data.message; + } + if (record.type === 'bench:summary') { + yield `${JSON.stringify({ samples, stdout })}\n`; + } + } +}; diff --git a/test/fixtures/bench-runner/throws-null.cjs b/test/fixtures/bench-runner/throws-null.cjs new file mode 100644 index 000000000000..562e969ac350 --- /dev/null +++ b/test/fixtures/bench-runner/throws-null.cjs @@ -0,0 +1,3 @@ +'use strict'; + +throw null; diff --git a/test/fixtures/bench-runner/v8-option.cjs b/test/fixtures/bench-runner/v8-option.cjs new file mode 100644 index 000000000000..a4e60ddf738f --- /dev/null +++ b/test/fixtures/bench-runner/v8-option.cjs @@ -0,0 +1,13 @@ +'use strict'; + +const assert = require('assert'); +const { bench } = require('node:bench'); + +assert.strictEqual(Error.stackTraceLimit, 17); +assert(process.execArgv.includes('--random-seed=17')); + +bench('V8 option', { samples: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/parallel/test-bench-cli.js b/test/parallel/test-bench-cli.js index 3180dcfeab12..28dcc8b2dbb7 100644 --- a/test/parallel/test-bench-cli.js +++ b/test/parallel/test-bench-cli.js @@ -8,15 +8,24 @@ const fixtures = require('../common/fixtures'); const tmpdir = require('../common/tmpdir'); const basicPattern = fixtures.path('bench-runner/[ab].*'); +const spawnTimeout = common.platformTimeout(30_000); tmpdir.refresh(); +function spawnNode(args, options = undefined) { + const result = spawnSync(process.execPath, args, { + __proto__: null, + encoding: 'utf8', + timeout: spawnTimeout, + ...options, + }); + assert.ifError(result.error); + assert.strictEqual(result.signal, null); + return result; +} + function spawnBench(args, options = undefined) { - return spawnSync(process.execPath, [ - '--no-warnings', - '--bench', - ...args, - ], { __proto__: null, encoding: 'utf8', ...options }); + return spawnNode(['--no-warnings', '--bench', ...args], options); } function parseRecords(result) { @@ -42,6 +51,42 @@ function parseOutput(output) { assert.match(result.stderr, /^Could not find/); } +if (common.canCreateSymLink()) { + const dangling = tmpdir.resolve('dangling.cjs'); + fs.symlinkSync(tmpdir.resolve('missing.cjs'), dangling); + const result = spawnBench([dangling]); + assert.strictEqual(result.status, 1); + assert.strictEqual(result.stdout, ''); + assert.match(result.stderr, /^Could not find/); +} + +for (const { patterns, message } of [ + { + patterns: [ + fixtures.path('bench-runner/a.cjs'), + fixtures.path('bench-runner/b.mjs'), + ], + message: /benchmark child process requires exactly one file/, + }, + { + patterns: [fixtures.path('bench-runner/missing.cjs')], + message: /^Could not find/, + }, +]) { + const result = spawnNode([ + '--no-warnings', + '--require', fixtures.path('bench-runner/fake-ipc.cjs'), + '--bench', + ...patterns, + ], { + __proto__: null, + env: { __proto__: null, ...process.env, NODE_BENCH_CONTEXT: 'child' }, + }); + assert.strictEqual(result.status, 1); + assert.strictEqual(result.stdout, ''); + assert.match(result.stderr, message); +} + { const result = spawnBench(['--bench-reporter=json', basicPattern]); assert.strictEqual(result.status, 0); @@ -145,6 +190,20 @@ function parseOutput(output) { assert.strictEqual(records.at(-1).data.success, false); } +{ + const result = spawnBench([ + '--bench-isolation=none', + '--bench-reporter=json', + fixtures.path('bench-runner/throws-null.cjs'), + ]); + assert.strictEqual(result.status, 1); + const records = parseRecords(result); + const diagnostic = records.find( + ({ type }) => type === 'bench:diagnostic').data; + assert.strictEqual(diagnostic.message, 'null'); + assert.strictEqual(records.at(-1).data.success, false); +} + { const result = spawnBench([ '--bench-reporter=json', @@ -236,11 +295,89 @@ for (const isolation of ['process', 'none']) { fixtures.path('bench-runner/ipc.cjs')); } +for (const { kind, message } of [ + { kind: 'record', message: /not a valid benchmark record/ }, + { kind: 'summary', message: /not a valid benchmark summary/ }, +]) { + const result = spawnBench([ + '--bench-reporter=json', + fixtures.path('bench-runner/malformed-record.cjs'), + ], { + env: { + __proto__: null, + ...process.env, + NODE_BENCH_MALFORMED_RECORD: kind, + }, + }); + assert.strictEqual(result.status, 1); + const records = parseRecords(result); + const diagnostics = records.filter( + ({ type }) => type === 'bench:diagnostic'); + assert(diagnostics.some(({ data }) => message.test(data.message))); + assert.strictEqual(records.at(-1).data.success, false); +} + +for (const { mode, message } of [ + { mode: 'code', message: /failed with exit code 2/ }, + { mode: 'late', message: /failed with exit code 2/ }, + ...common.isWindows ? [] : [ + { mode: 'signal', message: /failed with signal SIGTERM/ }, + ], +]) { + const result = spawnBench([ + '--bench-reporter=json', + fixtures.path('bench-runner/abrupt-exit.cjs'), + ], { + env: { + __proto__: null, + ...process.env, + NODE_BENCH_EXIT_MODE: mode, + }, + }); + assert.strictEqual(result.status, 1); + const records = parseRecords(result); + const diagnostics = records.filter( + ({ type }) => type === 'bench:diagnostic'); + assert(diagnostics.some(({ data }) => message.test(data.message))); +} + +for (const mode of ['callback', 'throw']) { + const result = spawnBench([ + '--bench-reporter=json', + fixtures.path('bench-runner/send-error.cjs'), + ], { + env: { + __proto__: null, + ...process.env, + NODE_BENCH_SEND_ERROR: mode, + }, + }); + assert.strictEqual(result.status, 1); + const records = parseRecords(result); + const diagnostics = records.filter( + ({ type }) => type === 'bench:diagnostic'); + const messages = diagnostics.map(({ data }) => data.message).join(''); + assert.match(messages, /benchmark send/); +} + +{ + const result = spawnBench([ + '--stack-trace-limit=17', + '--random-seed=17', + '--bench-reporter=json', + fixtures.path('bench-runner/v8-option.cjs'), + ]); + assert.strictEqual(result.status, 0); + const records = parseRecords(result); + assert.strictEqual(records.find( + ({ type }) => type === 'bench:complete').data.name, 'V8 option'); +} + if (common.hasInspector) { const result = spawnBench([ '--inspect=0', '--bench-reporter=json', - fixtures.path('bench-runner/a.cjs'), + fixtures.path('bench-runner/inspector.cjs'), ]); assert.strictEqual(result.status, 0); const records = parseRecords(result); @@ -248,6 +385,9 @@ if (common.hasInspector) { ({ type }) => type === 'bench:diagnostic') .map(({ data }) => data.message).join(''); assert.match(diagnostics, /Debugger listening on ws:\/\//); + const completion = records.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(completion.params.inspectPort, '--inspect-port=0'); } { @@ -307,6 +447,40 @@ if (common.hasInspector) { assert.match(result.stderr, /benchmark reporter failed/); } +{ + const result = spawnBench([ + `--bench-reporter=${fixtures.fileURL('bench-runner/slow-reporter.cjs')}`, + fixtures.path('bench-runner/many-records.cjs'), + ]); + assert.strictEqual(result.status, 0); + assert.strictEqual(result.stderr, ''); + const report = JSON.parse(result.stdout); + assert.strictEqual(report.samples, 30); + assert.strictEqual(report.stdout, + Array.from({ length: 30 }, (_, i) => `${i}\n`).join('')); +} + +{ + const result = spawnBench([ + `--bench-reporter=${fixtures.fileURL('bench-runner/destroying-reporter.cjs')}`, + fixtures.path('bench-runner/a.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /benchmark reporter closed the stream/); +} + +{ + const result = spawnBench([ + '--bench-reporter=json', + '--bench-reporter-destination=stdout', + '--bench-reporter=data:text/javascript,export default 0', + '--bench-reporter-destination=stderr', + fixtures.path('bench-runner/a.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /is not a valid reporter/); +} + { const result = spawnBench([ '--bench-reporter=json', @@ -318,11 +492,11 @@ if (common.hasInspector) { } { - const result = spawnSync(process.execPath, [ + const result = spawnNode([ '--no-warnings', `--experimental-config-file=${fixtures.path('bench-runner/node.config.json')}`, fixtures.path('bench-runner/a.cjs'), - ], { __proto__: null, encoding: 'utf8' }); + ]); assert.strictEqual(result.status, 0); const records = parseRecords(result); assert.strictEqual(records.find( @@ -385,6 +559,26 @@ for (const { args, message } of [ args: ['--bench-warmup=abc', 'unused.js'], message: /invalid value for --bench-warmup/, }, + { + args: ['--bench-name-pattern=[', 'unused.js'], + message: /invalid regular expression/, + }, + { + args: ['--eval=1', 'unused.js'], + message: /either --bench or --eval can be used, not both/, + }, + { + args: ['--interactive', 'unused.js'], + message: /either --bench or --interactive can be used, not both/, + }, + { + args: ['--watch', 'unused.js'], + message: /either --bench or --watch can be used, not both/, + }, + { + args: ['--watch-path=.', 'unused.js'], + message: /either --bench or --watch can be used, not both/, + }, { args: ['--check', 'unused.js'], message: /either --bench or --check can be used, not both/, diff --git a/test/parallel/test-bench-clock-precision.js b/test/parallel/test-bench-clock-precision.js new file mode 100644 index 000000000000..93158e51319b --- /dev/null +++ b/test/parallel/test-bench-clock-precision.js @@ -0,0 +1,22 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); + +const originalHrtimeBigint = process.hrtime.bigint; +process.hrtime.bigint = () => 1n; +const { bench, run } = require('node:bench'); +process.hrtime.bigint = originalHrtimeBigint; + +const completion = bench('zero duration', { samples: 1 }, (b) => { + b.start(); + b.end(1); +}); + +(async () => { + await run().toArray(); + const result = await completion; + assert.strictEqual(result.error.code, 'ERR_INVALID_STATE'); + assert.match(result.error.message, /insufficient clock precision/); +})().then(common.mustCall()); diff --git a/test/parallel/test-bench-context-control.js b/test/parallel/test-bench-context-control.js index ec28923f874f..b9adb9f2f9f1 100644 --- a/test/parallel/test-bench-context-control.js +++ b/test/parallel/test-bench-context-control.js @@ -82,5 +82,11 @@ const { createRunner } = require('node:bench'); { source: 'worker', value: 1n }); assert.strictEqual( records.filter(({ type }) => type === 'bench:sample').length, 3); + assert.throws(() => closedContext.start(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => closedContext.end(1), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => closedContext.record({ + duration_ns: 1n, + operations: 1, + }), { code: 'ERR_INVALID_STATE' }); assert.throws(() => closedContext.done(), { code: 'ERR_INVALID_STATE' }); })().then(common.mustCall()); diff --git a/test/parallel/test-bench-context-errors.js b/test/parallel/test-bench-context-errors.js index 40aca29bfcc5..072b9b3e2a9d 100644 --- a/test/parallel/test-bench-context-errors.js +++ b/test/parallel/test-bench-context-errors.js @@ -34,6 +34,14 @@ runner.bench('mixed timing', { samples: 1 }, (b) => { b.start(); b.record({ duration_ns: 1n, operations: 1 }); }); +runner.bench('start after record', { samples: 1 }, (b) => { + b.record({ duration_ns: 1n, operations: 1 }); + b.start(); +}); +runner.bench('end after record', { samples: 1 }, (b) => { + b.record({ duration_ns: 1n, operations: 1 }); + b.end(1); +}); runner.bench('duplicate record', { samples: 1 }, (b) => { b.record({ duration_ns: 1n, operations: 1 }); b.record({ duration_ns: 1n, operations: 1 }); @@ -51,6 +59,12 @@ runner.bench('reentrant record', { samples: 1 }, (b) => { runner.bench('uncloneable detail', { samples: 1 }, (b) => { b.record({ duration_ns: 1n, operations: 1, detail: () => {} }); }); +runner.bench('caught contract violation', { samples: 1 }, + common.mustCall((b) => { + b.start(); + assert.throws(() => b.start(), { code: 'ERR_INVALID_STATE' }); + b.end(1); + })); (async () => { const records = await runner.run().toArray(); @@ -73,10 +87,16 @@ runner.bench('uncloneable detail', { samples: 1 }, (b) => { 'ERR_OUT_OF_RANGE'); assert.strictEqual(byName.get('mixed timing').error.code, 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('start after record').error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('end after record').error.code, + 'ERR_INVALID_STATE'); assert.strictEqual(byName.get('duplicate record').error.code, 'ERR_INVALID_STATE'); assert.strictEqual(byName.get('reentrant record').error.code, 'ERR_INVALID_STATE'); assert.strictEqual(byName.get('uncloneable detail').error.name, 'DataCloneError'); + assert.match(byName.get('caught contract violation').error.message, + /violated the start\(\)\/end\(\) contract/); })().then(common.mustCall()); diff --git a/test/parallel/test-bench-create-runner.js b/test/parallel/test-bench-create-runner.js index a8c25533ee63..a7194d002d55 100644 --- a/test/parallel/test-bench-create-runner.js +++ b/test/parallel/test-bench-create-runner.js @@ -4,6 +4,7 @@ const common = require('../common'); const assert = require('assert'); const { createRunner } = require('node:bench'); +const { setImmediate } = require('timers/promises'); (async () => { const first = createRunner({ yieldBetweenSamples: false }); @@ -29,7 +30,7 @@ const { createRunner } = require('node:bench'); b.end(1); })); - await new Promise((resolve) => setImmediate(resolve)); + await setImmediate(); assert.strictEqual(firstCalls, 0); assert.strictEqual(secondCalls, 0); diff --git a/test/parallel/test-bench-errors.js b/test/parallel/test-bench-errors.js index 7a6f7aec7e16..b0774965a26f 100644 --- a/test/parallel/test-bench-errors.js +++ b/test/parallel/test-bench-errors.js @@ -4,6 +4,7 @@ const common = require('../common'); const assert = require('assert'); const { bench, run } = require('node:bench'); +const { setTimeout } = require('timers/promises'); const options = { samples: 1 }; @@ -31,7 +32,7 @@ bench('timeout', { samples: 1, timeout: 10 }, async () => { }); bench('late timeout', { samples: 1, timeout: 5 }, async (b) => { b.start(); - await new Promise((resolve) => setTimeout(resolve, 30)); + await setTimeout(30); b.end(1); }); @@ -97,8 +98,8 @@ stream.on('end', common.mustCall(() => { assert.strictEqual(duplicates[0].error, undefined); assert.match(duplicates[1].error.message, /duplicate benchmark identity/); assert.strictEqual(byName.get('continues')[0].error, undefined); - setTimeout(common.mustCall(() => { + setTimeout(40).then(common.mustCall(() => { assert.strictEqual(sampleNames.includes('late timeout'), false); - }), 40); + })); })); stream.resume(); diff --git a/test/parallel/test-bench-harness-errors.js b/test/parallel/test-bench-harness-errors.js new file mode 100644 index 000000000000..3e4ebefd2ebf --- /dev/null +++ b/test/parallel/test-bench-harness-errors.js @@ -0,0 +1,113 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { createRunner } = require('node:bench'); +const { setImmediate } = require('timers/promises'); + +function complete(b) { + b.start(); + process.hrtime.bigint(); + b.end(1); +} + +async function testSynchronousSuiteFailure() { + const runner = createRunner({ yieldBetweenSamples: false }); + const completion = runner.suite('outer', () => { + runner.suite('nested', () => { + runner.bench('blocked', { samples: 1 }, common.mustNotCall()); + }); + throw new Error('synchronous suite failure'); + }); + const records = await runner.run().toArray(); + await completion; + const result = records.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(result.name, 'blocked'); + assert.strictEqual(result.error.message, 'synchronous suite failure'); +} + +async function testRunSignal() { + const runner = createRunner(); + const controller = new AbortController(); + let invocations = 0; + let abortPromise; + const completion = runner.bench('aborted between samples', { + samples: 3, + }, (b) => { + invocations++; + complete(b); + if (b.index === 0) { + abortPromise = setImmediate().then(() => { + controller.abort(new Error('run aborted')); + }); + } + }); + await runner.run({ signal: controller.signal }).toArray(); + const result = await completion; + await abortPromise; + await setImmediate(); + assert.strictEqual(result.error.code, 'ABORT_ERR'); + assert.strictEqual(result.error.cause.message, 'run aborted'); + assert.strictEqual(invocations, 1); +} + +async function testRunSignalAfterSample() { + const runner = createRunner({ yieldBetweenSamples: false }); + const controller = new AbortController(); + const completion = runner.bench('aborted after sample', { + samples: 1, + }, (b) => { + complete(b); + controller.abort(new Error('sample aborted')); + }); + await runner.run({ signal: controller.signal }).toArray(); + const result = await completion; + await setImmediate(); + assert.strictEqual(result.error.code, 'ABORT_ERR'); + assert.strictEqual(result.error.cause.message, 'sample aborted'); +} + +async function testStringNamePattern() { + const runner = createRunner({ yieldBetweenSamples: false }); + runner.bench('included', { samples: 1 }, complete); + runner.bench('excluded', { samples: 1 }, common.mustNotCall()); + const records = await runner.run({ namePattern: 'included' }).toArray(); + const excluded = records.find( + ({ type, data }) => type === 'bench:complete' && + data.name === 'excluded').data; + const included = records.find( + ({ type, data }) => type === 'bench:complete' && + data.name === 'included').data; + assert.strictEqual(included.error, undefined); + assert.strictEqual(included.samples.length, 1); + assert.strictEqual(excluded.skip, 'name pattern'); +} + +async function testTopLevelRecovery() { + const runner = createRunner({ yieldBetweenSamples: false }); + runner.bench('listener failure', { samples: 1 }, complete); + const stream = runner.run(); + const failure = new Error(); + failure.message = undefined; + stream.on('bench:start', common.mustCall(() => { throw failure; })); + const records = await stream.toArray(); + const diagnostic = records.find( + ({ type }) => type === 'bench:diagnostic').data; + const summary = records.find(({ type }) => type === 'bench:summary').data; + assert.strictEqual(diagnostic.message, 'Error'); + assert.strictEqual(diagnostic.file, undefined); + assert.strictEqual(diagnostic.line, undefined); + assert.strictEqual(diagnostic.column, undefined); + assert.strictEqual(summary.duration_ns, 0n); + assert.strictEqual(summary.success, false); +} + +(async () => { + await testSynchronousSuiteFailure(); + await testRunSignal(); + await testRunSignalAfterSample(); + await testStringNamePattern(); + await testTopLevelRecovery(); +})().then(common.mustCall()); diff --git a/test/parallel/test-bench-hook-errors.js b/test/parallel/test-bench-hook-errors.js index 72de600b8abd..c4615419674a 100644 --- a/test/parallel/test-bench-hook-errors.js +++ b/test/parallel/test-bench-hook-errors.js @@ -3,6 +3,7 @@ const common = require('../common'); const assert = require('assert'); +const { setImmediate } = require('timers/promises'); const { after, afterEach, @@ -37,7 +38,7 @@ suite('after failure', () => { }); suite('build failure', async () => { - await new Promise((resolve) => setImmediate(resolve)); + await setImmediate(); throw new Error('build failure'); }); diff --git a/test/parallel/test-bench-reporters.js b/test/parallel/test-bench-reporters.js index c5a3b0000e9c..ed6039d9da3a 100644 --- a/test/parallel/test-bench-reporters.js +++ b/test/parallel/test-bench-reporters.js @@ -111,4 +111,66 @@ bench('json failed', { samples: 1 }, () => { 'broken | 0 | - | - | - | error: boom\n' + 'diagnostic: suite problem\n\n' + '1 completed, 1 failed, 1 skipped\n'); + + const circular = {}; + circular.self = circular; + const aggregate = new AggregateError([1n], 'aggregate failure'); + const jsonEdgeChunks = await Readable.from([{ + type: 'bench:diagnostic', + data: { aggregate, circular }, + }]).compose(json).toArray(); + const jsonEdge = JSON.parse(jsonEdgeChunks.join('')); + assert.deepStrictEqual(jsonEdge.data.aggregate.errors, ['1']); + assert.strictEqual(jsonEdge.data.circular.self, '[Circular]'); + + async function* undefinedRecord() { + yield undefined; + } + const undefinedChunks = []; + for await (const chunk of json(undefinedRecord())) { + undefinedChunks.push(chunk); + } + assert.strictEqual(undefinedChunks.join(''), 'null\n'); + + function result(name, rate) { + return { + type: 'bench:complete', + data: { + name, + params: {}, + samples: [{}], + summary: { + mean: rate, + median: rate, + coefficientOfVariation: 0, + confidenceInterval: { lower: rate, upper: rate }, + skewness: 0, + }, + }, + }; + } + + const specEdgeChunks = await Readable.from([ + result('giga', 1_500_000_000), + result('mega', 1_500_000), + result('fractional', 0.5), + { + type: 'bench:complete', + data: { name: 'skip', params: {}, samples: [], skip: true }, + }, + { + type: 'bench:complete', + data: { name: 'error', params: {}, samples: [], error: 'failure' }, + }, + ]).compose(spec).toArray(); + const specEdgeOutput = specEdgeChunks.join(''); + assert.match(specEdgeOutput, /giga \| 1 \| 1\.50G ops\/s/); + assert.match(specEdgeOutput, /mega \| 1 \| 1\.50M ops\/s/); + assert.match(specEdgeOutput, /fractional \| 1 \| 0\.500 ops\/s/); + assert.match(specEdgeOutput, /skip \| 0 \| - \| - \| - \| skipped\n/); + assert.match(specEdgeOutput, + /error \| 0 \| - \| - \| - \| error: failure/); + + const emptySpecChunks = await Readable.from([]).compose(spec).toArray(); + assert.deepStrictEqual(emptySpecChunks, []); })().then(common.mustCall()); diff --git a/test/parallel/test-bench-run.js b/test/parallel/test-bench-run.js index fafc9e3309ad..5b7edb4e6418 100644 --- a/test/parallel/test-bench-run.js +++ b/test/parallel/test-bench-run.js @@ -3,6 +3,7 @@ const common = require('../common'); const assert = require('assert'); +const { setImmediate } = require('timers/promises'); const { after, afterEach, @@ -23,7 +24,7 @@ beforeEach(() => calls.push('root beforeEach')); afterEach(() => calls.push('root afterEach')); const suiteCompletion = suite('group', { tags: ['Group'] }, async () => { - await new Promise((resolve) => setImmediate(resolve)); + await setImmediate(); before(() => calls.push('suite before')); after(() => calls.push('suite after')); @@ -52,11 +53,11 @@ const suiteCompletion = suite('group', { tags: ['Group'] }, async () => { active = true; contexts.add(b); calls.push('async sample'); - await new Promise((resolve) => setImmediate(resolve)); + await setImmediate(); b.start(); process.hrtime.bigint(); b.end(1); - await new Promise((resolve) => setImmediate(resolve)); + await setImmediate(); active = false; }, 2)); diff --git a/test/parallel/test-bench-validation.js b/test/parallel/test-bench-validation.js index cdbaee9e6ee6..7071dd2d915f 100644 --- a/test/parallel/test-bench-validation.js +++ b/test/parallel/test-bench-validation.js @@ -6,6 +6,23 @@ const assert = require('assert'); const { bench, createRunner, run } = require('node:bench'); const noop = () => {}; +let functionOverloadCalls = 0; +let objectOverloadCalls = 0; + +function functionOverload(b) { + functionOverloadCalls++; + b.start(); + process.hrtime.bigint(); + b.end(1); + b.done(); +} + +function objectOverload(b) { + objectOverloadCalls++; + b.start(); + process.hrtime.bigint(); + b.end(1); +} assert.throws(() => bench('', noop), { code: 'ERR_INVALID_ARG_VALUE' }); assert.throws(() => bench('name', null), { code: 'ERR_INVALID_ARG_TYPE' }); @@ -44,6 +61,9 @@ assert.throws(() => createRunner({ yieldBetweenSamples: 1 }), assert.throws(() => createRunner({ yieldBetweenSamples: null }), { code: 'ERR_INVALID_ARG_TYPE' }); +bench(functionOverload); +bench({ samples: 1 }, objectOverload); + bench('valid', { samples: 1 }, (b) => { b.start(); process.hrtime.bigint(); @@ -53,5 +73,9 @@ bench('valid', { samples: 1 }, (b) => { const stream = run(); stream.on('bench:start', common.mustCall(() => { assert.throws(() => bench('late', noop), { code: 'ERR_INVALID_STATE' }); +}, 3)); +stream.on('end', common.mustCall(() => { + assert.strictEqual(functionOverloadCalls, 1); + assert.strictEqual(objectOverloadCalls, 1); })); stream.resume(); diff --git a/test/parallel/test-bench-yield-between-samples.js b/test/parallel/test-bench-yield-between-samples.js index f10ae6a2f6c4..7d08318a027a 100644 --- a/test/parallel/test-bench-yield-between-samples.js +++ b/test/parallel/test-bench-yield-between-samples.js @@ -4,15 +4,15 @@ const common = require('../common'); const assert = require('assert'); const { createRunner } = require('node:bench'); +const { setImmediate } = require('timers/promises'); async function observe(factoryOptions, runOptions) { const runner = createRunner(factoryOptions); const observed = []; let turnOccurred = false; - const turn = new Promise((resolve) => setImmediate(() => { + const turn = setImmediate().then(() => { turnOccurred = true; - resolve(); - })); + }); runner.bench('yielding', { samples: 2 }, (b) => { observed.push(turnOccurred);