From b4c993e6990ec3a4d00a8337d869bfb37a8676a7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Mon, 3 Nov 2025 21:57:17 +0100 Subject: [PATCH] refactor: migrate benchmarks to tinybench MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- .github/workflows/internal-benchmark.yml | 2 +- .vscode/launch.json | 4 +- benchmarks/README.md | 2 +- benchmarks/benchmarks-utils.mjs | 109 ++++++++++++-------- benchmarks/internal/bench.mjs | 17 ++- benchmarks/worker-selection/least.mjs | 43 ++++---- benchmarks/worker-selection/round-robin.mjs | 39 +++---- docs/media/README.md | 2 +- package.json | 8 +- pnpm-lock.yaml | 23 ++--- 10 files changed, 130 insertions(+), 119 deletions(-) diff --git a/.github/workflows/internal-benchmark.yml b/.github/workflows/internal-benchmark.yml index a219b5f88..79d59246a 100644 --- a/.github/workflows/internal-benchmark.yml +++ b/.github/workflows/internal-benchmark.yml @@ -49,4 +49,4 @@ jobs: --file benchmark-report.json \ --err \ --github-actions ${{ secrets.GITHUB_TOKEN }} \ - "pnpm benchmark:tatami-ng:prod" + "pnpm benchmark:tinybench:prod" diff --git a/.vscode/launch.json b/.vscode/launch.json index c750ba521..cba82b980 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,10 +17,10 @@ { "type": "node", "request": "launch", - "name": "Launch Tatami NG Benchmark Debug", + "name": "Launch Tinybench Benchmark Debug", "cwd": "${workspaceFolder}", "runtimeExecutable": "pnpm", - "runtimeArgs": ["run", "benchmark:tatami-ng:debug"], + "runtimeArgs": ["run", "benchmark:tinybench:debug"], "skipFiles": ["/**"], "stopOnEntry": true } diff --git a/benchmarks/README.md b/benchmarks/README.md index 319cafcf4..74ec138ce 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -24,6 +24,6 @@ See the dedicated repository [README.md](https://github.com/poolifier/benchmark# To run the internal benchmark, you just need to navigate to the root of poolifier cloned repository and run: -- `pnpm benchmark:tatami-ng` +- `pnpm benchmark:tinybench` ### [Results](https://bencher.dev/perf/poolifier) diff --git a/benchmarks/benchmarks-utils.mjs b/benchmarks/benchmarks-utils.mjs index 0743037da..d0500928b 100644 --- a/benchmarks/benchmarks-utils.mjs +++ b/benchmarks/benchmarks-utils.mjs @@ -1,5 +1,5 @@ import { strictEqual } from 'node:assert' -import { bench, group, run } from 'tatami-ng' +import { Bench } from 'tinybench' import { DynamicClusterPool, @@ -58,57 +58,27 @@ const runPoolifierPool = async (pool, { taskExecutions, workerData }) => { } } -export const runPoolifierBenchmarkTatamiNg = async ( +export const runPoolifierBenchmarkTinyBench = async ( name, workerType, poolType, poolSize, - benchmarkReporter, { taskExecutions, workerData } ) => { try { + const bench = new Bench() const pool = buildPoolifierPool(workerType, poolType, poolSize) + for (const workerChoiceStrategy of Object.values(WorkerChoiceStrategies)) { for (const enableTasksQueue of [false, true]) { if (workerChoiceStrategy === WorkerChoiceStrategies.FAIR_SHARE) { for (const measurement of [Measurements.runTime, Measurements.elu]) { - group(name, () => { - bench( - `${name} with ${workerChoiceStrategy}, with ${measurement} and ${ - enableTasksQueue ? 'with' : 'without' - } tasks queue`, - async () => { - await runPoolifierPool(pool, { - taskExecutions, - workerData, - }) - }, - { - before: () => { - pool.setWorkerChoiceStrategy(workerChoiceStrategy, { - measurement, - }) - pool.enableTasksQueue(enableTasksQueue) - strictEqual( - pool.opts.workerChoiceStrategy, - workerChoiceStrategy - ) - strictEqual(pool.opts.enableTasksQueue, enableTasksQueue) - strictEqual( - pool.opts.workerChoiceStrategyOptions.measurement, - measurement - ) - }, - } - ) - }) - } - } else { - group(name, () => { - bench( - `${name} with ${workerChoiceStrategy} and ${ - enableTasksQueue ? 'with' : 'without' - } tasks queue`, + const taskName = `${name} with ${workerChoiceStrategy}, with ${measurement} and ${ + enableTasksQueue ? 'with' : 'without' + } tasks queue` + + bench.add( + taskName, async () => { await runPoolifierPool(pool, { taskExecutions, @@ -116,24 +86,73 @@ export const runPoolifierBenchmarkTatamiNg = async ( }) }, { - before: () => { - pool.setWorkerChoiceStrategy(workerChoiceStrategy) + beforeAll: () => { + pool.setWorkerChoiceStrategy(workerChoiceStrategy, { + measurement, + }) pool.enableTasksQueue(enableTasksQueue) strictEqual( pool.opts.workerChoiceStrategy, workerChoiceStrategy ) strictEqual(pool.opts.enableTasksQueue, enableTasksQueue) + strictEqual( + pool.opts.workerChoiceStrategyOptions.measurement, + measurement + ) }, } ) - }) + } + } else { + const taskName = `${name} with ${workerChoiceStrategy} and ${ + enableTasksQueue ? 'with' : 'without' + } tasks queue` + + bench.add( + taskName, + async () => { + await runPoolifierPool(pool, { + taskExecutions, + workerData, + }) + }, + { + beforeAll: () => { + pool.setWorkerChoiceStrategy(workerChoiceStrategy) + pool.enableTasksQueue(enableTasksQueue) + strictEqual( + pool.opts.workerChoiceStrategy, + workerChoiceStrategy + ) + strictEqual(pool.opts.enableTasksQueue, enableTasksQueue) + }, + } + ) } } } - const report = await run({ reporter: benchmarkReporter }) + + const tasks = await bench.run() + console.table(bench.table()) await pool.destroy() - return report + + const bmfResults = {} + for (const task of tasks) { + bmfResults[task.name] = { + latency: { + lower_value: task.result.latency.mean - task.result.latency.sd, + upper_value: task.result.latency.mean + task.result.latency.sd, + value: task.result.latency.mean, + }, + throughput: { + lower_value: task.result.throughput.mean - task.result.throughput.sd, + upper_value: task.result.throughput.mean + task.result.throughput.sd, + value: task.result.throughput.mean, + }, + } + } + return bmfResults } catch (error) { console.error(error) } diff --git a/benchmarks/internal/bench.mjs b/benchmarks/internal/bench.mjs index e24240c6e..49f434e3b 100644 --- a/benchmarks/internal/bench.mjs +++ b/benchmarks/internal/bench.mjs @@ -1,7 +1,6 @@ import { writeFileSync } from 'node:fs' import { env } from 'node:process' import { parseArgs } from 'node:util' -import { bmf } from 'tatami-ng' import { availableParallelism, @@ -9,7 +8,7 @@ import { WorkerTypes, } from '../../lib/index.mjs' import { TaskFunctions } from '../benchmarks-types.cjs' -import { runPoolifierBenchmarkTatamiNg } from '../benchmarks-utils.mjs' +import { runPoolifierBenchmarkTinyBench } from '../benchmarks-utils.mjs' const poolSize = availableParallelism() const taskExecutions = 1 @@ -33,14 +32,13 @@ switch ( strict: true, }).values.type ) { - case 'tatami-ng': + case 'tinybench': default: - benchmarkReport = await runPoolifierBenchmarkTatamiNg( + benchmarkReport = await runPoolifierBenchmarkTinyBench( 'FixedThreadPool', WorkerTypes.thread, PoolTypes.fixed, poolSize, - bmf, { taskExecutions, workerData, @@ -48,12 +46,11 @@ switch ( ) benchmarkReport = { ...benchmarkReport, - ...(await runPoolifierBenchmarkTatamiNg( + ...(await runPoolifierBenchmarkTinyBench( 'DynamicThreadPool', WorkerTypes.thread, PoolTypes.dynamic, poolSize, - bmf, { taskExecutions, workerData, @@ -62,12 +59,11 @@ switch ( } benchmarkReport = { ...benchmarkReport, - ...(await runPoolifierBenchmarkTatamiNg( + ...(await runPoolifierBenchmarkTinyBench( 'FixedClusterPool', WorkerTypes.cluster, PoolTypes.fixed, poolSize, - bmf, { taskExecutions, workerData, @@ -76,12 +72,11 @@ switch ( } benchmarkReport = { ...benchmarkReport, - ...(await runPoolifierBenchmarkTatamiNg( + ...(await runPoolifierBenchmarkTinyBench( 'DynamicClusterPool', WorkerTypes.cluster, PoolTypes.dynamic, poolSize, - bmf, { taskExecutions, workerData, diff --git a/benchmarks/worker-selection/least.mjs b/benchmarks/worker-selection/least.mjs index e5ac16257..1b9dbe441 100644 --- a/benchmarks/worker-selection/least.mjs +++ b/benchmarks/worker-selection/least.mjs @@ -1,5 +1,5 @@ import { randomInt } from 'node:crypto' -import { bench, group, run } from 'tatami-ng' +import { Bench } from 'tinybench' /** * Generates a random tasks map for benchmarking. @@ -241,25 +241,26 @@ function swap (array, index1, index2) { array[index2] = tmp } -group('Least used worker tasks distribution', () => { - bench('Loop select', () => { - loopSelect(tasksMap) - }) - bench('Array sort select', () => { - arraySortSelect(tasksMap) - }) - bench('Quick select loop', () => { - quickSelectLoop(tasksMap) - }) - bench('Quick select loop with random pivot', () => { - quickSelectLoopRandomPivot(tasksMap) - }) - bench('Quick select recursion', () => { - quickSelectRecursion(tasksMap) - }) - bench('Quick select recursion with random pivot', () => { - quickSelectRecursionRandomPivot(tasksMap) - }) +const bench = new Bench() + +bench.add('Loop select', () => { + loopSelect(tasksMap) +}) +bench.add('Array sort select', () => { + arraySortSelect(tasksMap) +}) +bench.add('Quick select loop', () => { + quickSelectLoop(tasksMap) +}) +bench.add('Quick select loop with random pivot', () => { + quickSelectLoopRandomPivot(tasksMap) +}) +bench.add('Quick select recursion', () => { + quickSelectRecursion(tasksMap) +}) +bench.add('Quick select recursion with random pivot', () => { + quickSelectRecursionRandomPivot(tasksMap) }) -await run({ units: true }) +await bench.run() +console.table(bench.table()) diff --git a/benchmarks/worker-selection/round-robin.mjs b/benchmarks/worker-selection/round-robin.mjs index 675fc98f8..57053cc8d 100644 --- a/benchmarks/worker-selection/round-robin.mjs +++ b/benchmarks/worker-selection/round-robin.mjs @@ -1,4 +1,4 @@ -import { bench, group, run } from 'tatami-ng' +import { Bench } from 'tinybench' /** * Generates an array of worker indices. @@ -57,23 +57,24 @@ function roundRobinTernaryWithPreChoosing () { return chosenWorker } -group('Round robin tasks distribution', () => { - bench('Ternary off by one', () => { - nextWorkerIndex = 0 - roundRobinTernaryOffByOne() - }) - bench('Ternary with negation', () => { - nextWorkerIndex = 0 - roundRobinTernaryWithNegation() - }) - bench('Ternary with pre-choosing', () => { - nextWorkerIndex = 0 - roundRobinTernaryWithPreChoosing() - }) - bench('Increment+Modulo', () => { - nextWorkerIndex = 0 - roundRobinIncrementModulo() - }) +const bench = new Bench() + +bench.add('Ternary off by one', () => { + nextWorkerIndex = 0 + roundRobinTernaryOffByOne() +}) +bench.add('Ternary with negation', () => { + nextWorkerIndex = 0 + roundRobinTernaryWithNegation() +}) +bench.add('Ternary with pre-choosing', () => { + nextWorkerIndex = 0 + roundRobinTernaryWithPreChoosing() +}) +bench.add('Increment+Modulo', () => { + nextWorkerIndex = 0 + roundRobinIncrementModulo() }) -await run({ units: true }) +await bench.run() +console.table(bench.table()) diff --git a/docs/media/README.md b/docs/media/README.md index 319cafcf4..74ec138ce 100644 --- a/docs/media/README.md +++ b/docs/media/README.md @@ -24,6 +24,6 @@ See the dedicated repository [README.md](https://github.com/poolifier/benchmark# To run the internal benchmark, you just need to navigate to the root of poolifier cloned repository and run: -- `pnpm benchmark:tatami-ng` +- `pnpm benchmark:tinybench` ### [Results](https://bencher.dev/perf/poolifier) diff --git a/package.json b/package.json index 6ea390588..e907d81a6 100644 --- a/package.json +++ b/package.json @@ -24,9 +24,9 @@ "build:prod": "rollup --config", "build:typedoc": "rollup --config --environment DOCUMENTATION,BUILD:development", "build:analyze": "rollup --config --environment ANALYZE,BUILD:development", - "benchmark:tatami-ng": "pnpm build && node --enable-source-maps benchmarks/internal/bench.mjs -t tatami-ng", - "benchmark:tatami-ng:prod": "pnpm build:prod && node --enable-source-maps benchmarks/internal/bench.mjs -t tatami-ng", - "benchmark:tatami-ng:debug": "pnpm build && node --enable-source-maps --inspect benchmarks/internal/bench.mjs -t tatami-ng", + "benchmark:tinybench": "pnpm build && node --enable-source-maps benchmarks/internal/bench.mjs -t tinybench", + "benchmark:tinybench:prod": "pnpm build:prod && node --enable-source-maps benchmarks/internal/bench.mjs -t tinybench", + "benchmark:tinybench:debug": "pnpm build && node --enable-source-maps --inspect benchmarks/internal/bench.mjs -t tinybench", "test": "pnpm build --environment SOURCEMAP:false && cross-env NODE_ENV=test c8 mocha 'tests/**/*.test.mjs'", "test:parallel": "pnpm build --environment SOURCEMAP:false && cross-env NODE_ENV=test c8 mocha --parallel 'tests/**/*.test.mjs'", "test:debug": "pnpm build && cross-env NODE_ENV=test mocha --inspect 'tests/**/*.test.mjs'", @@ -135,7 +135,7 @@ "rollup-plugin-delete": "^3.0.1", "rollup-plugin-dts": "^6.2.3", "sinon": "^21.0.0", - "tatami-ng": "^0.8.18", + "tinybench": "^5.1.0", "typedoc": "^0.28.14", "typescript": "~5.9.3" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6f291057e..b7ac564f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -95,9 +95,9 @@ importers: sinon: specifier: ^21.0.0 version: 21.0.0 - tatami-ng: - specifier: ^0.8.18 - version: 0.8.18(typescript@5.9.3) + tinybench: + specifier: ^5.1.0 + version: 5.1.0 typedoc: specifier: ^0.28.14 version: 0.28.14(typescript@5.9.3) @@ -2601,12 +2601,6 @@ packages: resolution: {integrity: sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==} engines: {node: '>=6'} - tatami-ng@0.8.18: - resolution: {integrity: sha512-Q22ZpW/yPXP1Hb4e2s1JQcTtoMaVHZLCt8AjAyBjARiXcorgHyvuWyIPFJOvmrTglXU2qQPLqL+7HEE0tIHdiA==} - hasBin: true - peerDependencies: - typescript: ^5.4.3 - tcomb-validation@3.4.1: resolution: {integrity: sha512-urVVMQOma4RXwiVCa2nM2eqrAomHROHvWPuj6UkDGz/eb5kcy0x6P0dVt6kzpUZtYMNoAqJLWmz1BPtxrtjtrA==} @@ -2629,6 +2623,10 @@ packages: through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} + tinybench@5.1.0: + resolution: {integrity: sha512-LXKNtFualiKOm6gADe1UXPtf8+Nfn1CtPMEHAT33Fd2YjQatrujkDcK0+4wRC1X6t7fxUDXUs6BsvuIgfkDgDg==} + engines: {node: '>=20.0.0'} + tinyexec@1.0.1: resolution: {integrity: sha512-5uC6DDlmeqiOwCPmK9jMSdOuZTh8bU39Ys6yidB+UTt5hfZUPGAypSgFRiEp+jbi9qH40BLDvy85jIU88wKSqw==} @@ -5570,11 +5568,6 @@ snapshots: tapable@2.3.0: {} - tatami-ng@0.8.18(typescript@5.9.3): - dependencies: - peowly: 1.3.2 - typescript: 5.9.3 - tcomb-validation@3.4.1: dependencies: tcomb: 3.2.29 @@ -5598,6 +5591,8 @@ snapshots: through@2.3.8: {} + tinybench@5.1.0: {} + tinyexec@1.0.1: {} tinyglobby@0.2.15: -- 2.43.0