From 8f01ffbe12f90c234fe5acb12932b752c56905ef Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Mon, 13 May 2024 16:50:34 +0200 Subject: [PATCH] perf: fix continuous benchmarking MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- .eslintrc.cjs | 1 + .github/workflows/internal-benchmark.yml | 1 + .gitignore | 1 + benchmarks/README.md | 1 - benchmarks/benchmarks-utils.mjs | 203 +++-------------------- benchmarks/internal/bench.mjs | 146 +++++++--------- package.json | 6 +- pnpm-lock.yaml | 26 +-- 8 files changed, 92 insertions(+), 293 deletions(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index af3c350b..80b400b6 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -35,6 +35,7 @@ module.exports = defineConfig({ 'argv', 'axios', 'benoit', + 'bmf', 'browserslist', 'builtins', 'christopher', diff --git a/.github/workflows/internal-benchmark.yml b/.github/workflows/internal-benchmark.yml index 936f0ad7..75b97cff 100644 --- a/.github/workflows/internal-benchmark.yml +++ b/.github/workflows/internal-benchmark.yml @@ -38,6 +38,7 @@ jobs: --else-if-branch "$GITHUB_BASE_REF" \ --else-if-branch master \ --hash "$GITHUB_SHA" \ + --file benchmark-report.json \ --err \ --github-actions ${{ secrets.GITHUB_TOKEN }} \ "pnpm benchmark:tatami-ng:prod" diff --git a/.gitignore b/.gitignore index d014f1b3..9f9e3aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ dist tmp .history/ reports/ +benchmark-report.json diff --git a/benchmarks/README.md b/benchmarks/README.md index 27467511..319cafcf 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -24,7 +24,6 @@ See the dedicated repository [README.md](https://github.com/poolifier/benchmark# To run the internal benchmark, you just need to navigate to the root of poolifier cloned repository and run: -- `pnpm benchmark:benchmark.js` or - `pnpm benchmark:tatami-ng` ### [Results](https://bencher.dev/perf/poolifier) diff --git a/benchmarks/benchmarks-utils.mjs b/benchmarks/benchmarks-utils.mjs index 7d395d96..8620dd68 100644 --- a/benchmarks/benchmarks-utils.mjs +++ b/benchmarks/benchmarks-utils.mjs @@ -1,7 +1,5 @@ import { strictEqual } from 'node:assert' -import { env } from 'node:process' -import Benchmark from 'benchmark' import { bench, clear, group, run } from 'tatami-ng' import { @@ -76,180 +74,6 @@ const runPoolifierPool = async (pool, { taskExecutions, workerData }) => { }) } -export const runPoolifierBenchmarkBenchmarkJs = async ( - name, - workerType, - poolType, - poolSize, - poolOptions, - { taskExecutions, workerData } -) => { - return await new Promise((resolve, reject) => { - const pool = buildPoolifierPool(workerType, poolType, poolSize, poolOptions) - let workerChoiceStrategy - let enableTasksQueue - let workerChoiceStrategyOptions - if (poolOptions != null) { - ({ - workerChoiceStrategy, - enableTasksQueue, - workerChoiceStrategyOptions - } = poolOptions) - } - const measurement = workerChoiceStrategyOptions?.measurement - new Benchmark( - `${name} with ${workerChoiceStrategy ?? pool.opts.workerChoiceStrategy}${ - measurement != null ? `, with ${measurement}` : '' - } and ${enableTasksQueue ? 'with' : 'without'} tasks queue`, - async () => { - await runPoolifierPool(pool, { - taskExecutions, - workerData - }) - }, - { - onStart: () => { - if (workerChoiceStrategy != null) { - strictEqual(pool.opts.workerChoiceStrategy, workerChoiceStrategy) - } - if (enableTasksQueue != null) { - strictEqual(pool.opts.enableTasksQueue, enableTasksQueue) - } - if (measurement != null) { - strictEqual( - pool.opts.workerChoiceStrategyOptions.measurement, - measurement - ) - } - }, - onComplete: event => { - console.info(event.target.toString()) - if (pool.started && !pool.destroying) { - pool.destroy().then(resolve).catch(reject) - } else { - resolve() - } - }, - onError: event => { - if (pool.started && !pool.destroying) { - pool - .destroy() - .then(() => { - return reject(event.target.error) - }) - .catch(() => {}) - } else { - reject(event.target.error) - } - } - } - ).run({ async: true }) - }) -} - -export const runPoolifierBenchmarkBenchmarkJsSuite = async ( - name, - workerType, - poolType, - poolSize, - { taskExecutions, workerData } -) => { - return await new Promise((resolve, reject) => { - const pool = buildPoolifierPool(workerType, poolType, poolSize) - const suite = new Benchmark.Suite(name, { - onComplete: () => { - if (pool.started && !pool.destroying) { - pool.destroy().then(resolve).catch(reject) - } else { - resolve() - } - }, - onCycle: event => { - console.info(event.target.toString()) - }, - onError: event => { - if (pool.started && !pool.destroying) { - pool - .destroy() - .then(() => { - return reject(event.target.error) - }) - .catch(() => {}) - } else { - reject(event.target.error) - } - } - }) - for (const workerChoiceStrategy of Object.values(WorkerChoiceStrategies)) { - for (const enableTasksQueue of [false, true]) { - if (workerChoiceStrategy === WorkerChoiceStrategies.FAIR_SHARE) { - for (const measurement of [Measurements.runTime, Measurements.elu]) { - suite.add( - `${name} with ${workerChoiceStrategy}, with ${measurement} and ${ - enableTasksQueue ? 'with' : 'without' - } tasks queue`, - async () => { - await runPoolifierPool(pool, { - taskExecutions, - workerData - }) - }, - { - onStart: () => { - pool.setWorkerChoiceStrategy(workerChoiceStrategy, { - measurement - }) - pool.enableTasksQueue(enableTasksQueue) - strictEqual( - pool.opts.workerChoiceStrategy, - workerChoiceStrategy - ) - strictEqual(pool.opts.enableTasksQueue, enableTasksQueue) - strictEqual( - pool.opts.workerChoiceStrategyOptions.measurement, - measurement - ) - } - } - ) - } - } else { - suite.add( - `${name} with ${workerChoiceStrategy} and ${ - enableTasksQueue ? 'with' : 'without' - } tasks queue`, - async () => { - await runPoolifierPool(pool, { - taskExecutions, - workerData - }) - }, - { - onStart: () => { - pool.setWorkerChoiceStrategy(workerChoiceStrategy) - pool.enableTasksQueue(enableTasksQueue) - strictEqual( - pool.opts.workerChoiceStrategy, - workerChoiceStrategy - ) - strictEqual(pool.opts.enableTasksQueue, enableTasksQueue) - } - } - ) - } - } - } - suite - .on('complete', function () { - console.info( - 'Fastest is ' + - LIST_FORMATTER.format(this.filter('fastest').map('name')) - ) - }) - .run({ async: true }) - }) -} - export const runPoolifierBenchmarkTatamiNg = async ( name, workerType, @@ -322,19 +146,32 @@ export const runPoolifierBenchmarkTatamiNg = async ( } } } - await run({ - json: env.CI != null ? 'bmf' : false - }) + const report = await run() clear() await pool.destroy() + return report } catch (error) { console.error(error) } } -const LIST_FORMATTER = new Intl.ListFormat('en-US', { - style: 'long', - type: 'conjunction' -}) +export const convertTatamiNgToBmf = report => { + return report.benchmarks + .map(({ name, stats }) => { + return { + [name]: { + latency: { + value: stats?.avg, + lower_value: stats?.min, + upper_value: stats?.max + }, + throughput: { + value: stats?.iter + } + } + } + }) + .reduce((obj, item) => Object.assign(obj, item), {}) +} export { executeTaskFunction } diff --git a/benchmarks/internal/bench.mjs b/benchmarks/internal/bench.mjs index da0c023d..0e3ee055 100644 --- a/benchmarks/internal/bench.mjs +++ b/benchmarks/internal/bench.mjs @@ -1,4 +1,5 @@ -import { exit } from 'node:process' +import { writeFileSync } from 'node:fs' +import { env, exit } from 'node:process' // eslint-disable-next-line n/no-unsupported-features/node-builtins import { parseArgs } from 'node:util' @@ -9,7 +10,7 @@ import { } from '../../lib/index.mjs' import { TaskFunctions } from '../benchmarks-types.cjs' import { - runPoolifierBenchmarkBenchmarkJsSuite, + convertTatamiNgToBmf, runPoolifierBenchmarkTatamiNg } from '../benchmarks-utils.mjs' @@ -19,6 +20,8 @@ const workerData = { function: TaskFunctions.factorial, taskSize: 1000 } +const benchmarkReportFile = 'benchmark-report.json' +let benchmarkReport switch ( parseArgs({ @@ -34,89 +37,66 @@ switch ( }).values.type ) { case 'tatami-ng': - await runPoolifierBenchmarkTatamiNg( - 'FixedThreadPool', - WorkerTypes.thread, - PoolTypes.fixed, - poolSize, - { - taskExecutions, - workerData - } - ) - await runPoolifierBenchmarkTatamiNg( - 'DynamicThreadPool', - WorkerTypes.thread, - PoolTypes.dynamic, - poolSize, - { - taskExecutions, - workerData - } - ) - await runPoolifierBenchmarkTatamiNg( - 'FixedClusterPool', - WorkerTypes.cluster, - PoolTypes.fixed, - poolSize, - { - taskExecutions, - workerData - } - ) - await runPoolifierBenchmarkTatamiNg( - 'DynamicClusterPool', - WorkerTypes.cluster, - PoolTypes.dynamic, - poolSize, - { - taskExecutions, - workerData - } - ) - break - case 'benchmark.js': default: - await runPoolifierBenchmarkBenchmarkJsSuite( - 'FixedThreadPool', - WorkerTypes.thread, - PoolTypes.fixed, - poolSize, - { - taskExecutions, - workerData - } - ) - await runPoolifierBenchmarkBenchmarkJsSuite( - 'DynamicThreadPool', - WorkerTypes.thread, - PoolTypes.dynamic, - poolSize, - { - taskExecutions, - workerData - } - ) - await runPoolifierBenchmarkBenchmarkJsSuite( - 'FixedClusterPool', - WorkerTypes.cluster, - PoolTypes.fixed, - poolSize, - { - taskExecutions, - workerData - } - ) - await runPoolifierBenchmarkBenchmarkJsSuite( - 'DynamicClusterPool', - WorkerTypes.cluster, - PoolTypes.dynamic, - poolSize, - { - taskExecutions, - workerData - } + benchmarkReport = convertTatamiNgToBmf( + await runPoolifierBenchmarkTatamiNg( + 'FixedThreadPool', + WorkerTypes.thread, + PoolTypes.fixed, + poolSize, + { + taskExecutions, + workerData + } + ) ) + benchmarkReport = { + ...benchmarkReport, + ...convertTatamiNgToBmf( + await runPoolifierBenchmarkTatamiNg( + 'DynamicThreadPool', + WorkerTypes.thread, + PoolTypes.dynamic, + poolSize, + { + taskExecutions, + workerData + } + ) + ) + } + benchmarkReport = { + ...benchmarkReport, + ...convertTatamiNgToBmf( + await runPoolifierBenchmarkTatamiNg( + 'FixedClusterPool', + WorkerTypes.cluster, + PoolTypes.fixed, + poolSize, + { + taskExecutions, + workerData + } + ) + ) + } + benchmarkReport = { + ...benchmarkReport, + ...convertTatamiNgToBmf( + await runPoolifierBenchmarkTatamiNg( + 'DynamicClusterPool', + WorkerTypes.cluster, + PoolTypes.dynamic, + poolSize, + { + taskExecutions, + workerData + } + ) + ) + } + env.CI != null && + writeFileSync(benchmarkReportFile, JSON.stringify(benchmarkReport)) break } diff --git a/package.json b/package.json index bf763edc..ff866a52 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,6 @@ "build:prod": "rollup --config", "build:typedoc": "rollup --config --environment DOCUMENTATION,BUILD:development", "build:analyze": "rollup --config --environment ANALYZE,BUILD:development", - "benchmark:benchmark.js": "pnpm build && node --enable-source-maps benchmarks/internal/bench.mjs -t benchmark.js", - "benchmark:benchmark.js:prod": "pnpm build:prod && node --enable-source-maps benchmarks/internal/bench.mjs -t benchmark.js", - "benchmark:benchmark.js:debug": "pnpm build && node --enable-source-maps --inspect benchmarks/internal/bench.mjs -t benchmark.js", "benchmark:tatami-ng": "pnpm build && node --enable-source-maps benchmarks/internal/bench.mjs -t tatami-ng", "benchmark:tatami-ng:prod": "pnpm build:prod && node --enable-source-maps benchmarks/internal/bench.mjs -t tatami-ng", "benchmark:tatami-ng:debug": "pnpm build && node --enable-source-maps --inspect benchmarks/internal/bench.mjs -t tatami-ng", @@ -114,7 +111,6 @@ "@types/node": "^20.12.11", "@typescript-eslint/eslint-plugin": "^7.8.0", "@typescript-eslint/parser": "^7.8.0", - "benchmark": "^2.1.4", "c8": "^9.1.0", "cross-env": "^7.0.3", "eslint": "^8.57.0", @@ -143,7 +139,7 @@ "rollup-plugin-delete": "^2.0.0", "rollup-plugin-dts": "^6.1.0", "sinon": "^17.0.2", - "tatami-ng": "^0.4.6", + "tatami-ng": "^0.4.8", "typedoc": "^0.25.13", "typescript": "~5.4.5" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f9cbefb1..65b9a37a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,9 +41,6 @@ importers: '@typescript-eslint/parser': specifier: ^7.8.0 version: 7.8.0(eslint@8.57.0)(typescript@5.4.5) - benchmark: - specifier: ^2.1.4 - version: 2.1.4 c8: specifier: ^9.1.0 version: 9.1.0 @@ -129,8 +126,8 @@ importers: specifier: ^17.0.2 version: 17.0.2 tatami-ng: - specifier: ^0.4.6 - version: 0.4.6(typescript@5.4.5) + specifier: ^0.4.8 + version: 0.4.8(typescript@5.4.5) typedoc: specifier: ^0.25.13 version: 0.25.13(typescript@5.4.5) @@ -851,9 +848,6 @@ packages: before-after-hook@2.2.3: resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==} - benchmark@2.1.4: - resolution: {integrity: sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ==} - binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} @@ -2509,9 +2503,6 @@ packages: engines: {node: '>=0.10'} hasBin: true - platform@1.3.6: - resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} - possible-typed-array-names@1.0.0: resolution: {integrity: sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==} engines: {node: '>= 0.4'} @@ -2914,8 +2905,8 @@ packages: resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} engines: {node: '>=6'} - tatami-ng@0.4.6: - resolution: {integrity: sha512-DR59y8nC6r+u9YbYY5objx8o+DJah0NgxktO9Ib6jMk/tETzsCfqaPOOMt6GhWi/msZ2i6mFV6D2pur5HTBkzw==} + tatami-ng@0.4.8: + resolution: {integrity: sha512-RoAvJYBdweev0gMbphpea/kC+ht9hcatUzTj4XPP5kRpkjrsRUu9oAGAxVFPISl/s+iVvC76ChiDaDNHl5sVJg==} peerDependencies: typescript: ^5.0.0 @@ -3944,11 +3935,6 @@ snapshots: before-after-hook@2.2.3: {} - benchmark@2.1.4: - dependencies: - lodash: 4.17.21 - platform: 1.3.6 - binary-extensions@2.3.0: {} bl@4.1.0: @@ -5784,8 +5770,6 @@ snapshots: pidtree@0.6.0: {} - platform@1.3.6: {} - possible-typed-array-names@1.0.0: {} prelude-ls@1.2.1: {} @@ -6242,7 +6226,7 @@ snapshots: tapable@2.2.1: {} - tatami-ng@0.4.6(typescript@5.4.5): + tatami-ng@0.4.8(typescript@5.4.5): dependencies: typescript: 5.4.5 -- 2.34.1