fix(benchmark): correct Bencher reporting and benchmark failures (#3310)

* fix(benchmark): correct Bencher reporting and benchmark failures

* refactor(benchmark): use Tinybench unit conversion helper

diff --git a/.github/workflows/internal-benchmark.yml b/.github/workflows/internal-benchmark.yml
index 1116e176b7cab320d5a5d3ee869b4eb97278ed5c..8c7a81bbbd22e27106d6e2c425e8a50ba0517b2b 100644 (file)
--- a/.github/workflows/internal-benchmark.yml
+++ b/.github/workflows/internal-benchmark.yml
@@ -33,9 +33,17 @@ jobs:
         run: pnpm install --ignore-scripts --frozen-lockfile
       - uses: bencherdev/bencher@main
       - name: Run production benchmark
+        run: pnpm benchmark:tinybench:prod
+      - name: Publish benchmark
+        shell: bash
         run: |
+          args=()
+          if [[ "$GITHUB_REF" != refs/heads/master ]]; then
+            args+=(--dry-run)
+          fi
           bencher run \
-          --branch master \
+          "${args[@]}" \
+          --branch "$GITHUB_REF_NAME" \
           --threshold-measure latency \
           --threshold-test t_test \
           --threshold-max-sample-size 64 \
@@ -48,6 +56,4 @@ jobs:
           --threshold-upper-boundary _ \
           --thresholds-reset \
           --file benchmark-report.json \
-          --err \
-          --github-actions ${{ secrets.GITHUB_TOKEN }} \
-          "pnpm benchmark:tinybench:prod"
+          --err

diff --git a/benchmarks/benchmarks-utils.mjs b/benchmarks/benchmarks-utils.mjs
index 4cf11681732ae48f67c88b15444f869a508b71bf..53755f8befc120d73e1891f575de48f6b8d2ad25 100644 (file)
--- a/benchmarks/benchmarks-utils.mjs
+++ b/benchmarks/benchmarks-utils.mjs
@@ -1,5 +1,5 @@
 import { strictEqual } from 'node:assert'
-import { Bench } from 'tinybench'
+import { Bench, mToNs } from 'tinybench'
 
 import {
   DynamicClusterPool,
@@ -68,7 +68,7 @@ export const runPoolifierBenchmarkTinyBench = async (
   const bmfResults = {}
   let pool
   try {
-    const bench = new Bench()
+    const bench = new Bench({ throws: true })
     pool = buildPoolifierPool(workerType, poolType, poolSize)
 
     for (const workerChoiceStrategy of Object.values(WorkerChoiceStrategies)) {
@@ -139,30 +139,24 @@ export const runPoolifierBenchmarkTinyBench = async (
     console.table(bench.table())
 
     for (const task of tasks) {
-      if (
-        task.result?.state === 'completed' ||
-        task.result?.state === 'aborted-with-statistics'
-      ) {
-        bmfResults[task.name] = {
-          latency: {
-            lower_value: task.result.latency.mean - task.result.latency.sd,
-            upper_value: task.result.latency.mean + task.result.latency.sd,
-            value: task.result.latency.mean,
-          },
-          throughput: {
-            lower_value:
-              task.result.throughput.mean - task.result.throughput.sd,
-            upper_value:
-              task.result.throughput.mean + task.result.throughput.sd,
-            value: task.result.throughput.mean,
-          },
-        }
+      if (task.result?.state !== 'completed') {
+        throw new Error(`Benchmark did not complete: ${task.name}`)
+      }
+      bmfResults[task.name] = {
+        latency: {
+          // Tinybench reports milliseconds; Bencher latency uses nanoseconds.
+          lower_value: mToNs(task.result.latency.mean - task.result.latency.sd),
+          upper_value: mToNs(task.result.latency.mean + task.result.latency.sd),
+          value: mToNs(task.result.latency.mean),
+        },
+        throughput: {
+          lower_value: task.result.throughput.mean - task.result.throughput.sd,
+          upper_value: task.result.throughput.mean + task.result.throughput.sd,
+          value: task.result.throughput.mean,
+        },
       }
     }
     return bmfResults
-  } catch (error) {
-    console.error(error)
-    return bmfResults
   } finally {
     if (pool != null) {
       await pool.destroy()