From: Jérôme Benoit <jerome.benoit@sap.com>
Date: Mon, 17 Oct 2022 22:15:45 +0000 (+0200)
Subject: Improve benchmarks: add IO intensive task workload, add task size option, integrate... 
X-Git-Tag: v2.3.5~18
X-Git-Url: https://git.piment-noir.org/?a=commitdiff_plain;h=7a6a0a967f5d0978d7ad0714616194fe7592f69a;p=poolifier.git

Improve benchmarks: add IO intensive task workload, add task size option, integrate into eslint.

Signed-off-by: Jérôme Benoit <jerome.benoit@sap.com>
---

diff --git a/.eslintignore b/.eslintignore
index 0786c823..f94ab6fc 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,5 +1,3 @@
-examples/typescript/**/*.ts
-benchmarks/versus-external-pools/
 docs/
 outputs/
 lib/
diff --git a/.eslintrc.js b/.eslintrc.js
index f9453f90..bd39bc7e 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -37,23 +37,33 @@ module.exports = defineConfig({
         skipWords: [
           'browserslist',
           'christopher',
+          'cjs',
           'comparator',
           'cpu',
           'cpus',
+          'ctx',
           'ecma',
           'enum',
           'fibonacci',
+          'fs',
           'inheritDoc',
           'jsdoc',
+          'microjob',
           'num',
           'os',
+          'piscina',
           'poolifier',
+          'poolify',
           'readonly',
           'serializable',
           'sinon',
+          'threadjs',
+          'threadwork',
           'tsconfig',
           'typedoc',
+          'unlink',
           'unregister',
+          'utf8',
           'workerpool'
         ],
         skipIfMatch: ['^@.*', '^plugin:.*']
@@ -117,6 +127,16 @@ module.exports = defineConfig({
         'jsdoc/require-returns-type': 'off'
       }
     },
+    {
+      files: ['examples/typescript/**/*.ts'],
+      rules: {
+        'import/no-unresolved': 'off',
+        'jsdoc/require-jsdoc': 'off',
+        '@typescript-eslint/no-unsafe-argument': 'off',
+        '@typescript-eslint/no-unsafe-call': 'off',
+        '@typescript-eslint/no-unsafe-assignment': 'off'
+      }
+    },
     {
       files: ['**/*.js'],
       extends: 'plugin:node/recommended'
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 00e9d33e..a2fb8aeb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Improve benchmarks: add IO intensive task workload, add task size option, integrate into eslint.
+
 ## [2.3.4] - 2022-10-17
 
 ### Added
diff --git a/benchmarks/versus-external-pools/bench.sh b/benchmarks/versus-external-pools/bench.sh
index f78500f7..34ebb788 100755
--- a/benchmarks/versus-external-pools/bench.sh
+++ b/benchmarks/versus-external-pools/bench.sh
@@ -1,20 +1,30 @@
 #!/usr/bin/env bash
 
-### The -t argument is needed to specify the type of task that you want to benchmark.
-### Supported values are CPU_INTENSIVE
+### The -t argument permit to specify the type of task that you want to benchmark.
+### The -s argument permit to specify the size of task that you want to benchmark.
+### Supported values are CPU_INTENSIVE, IO_INTENSIVE
 
 taskType='CPU_INTENSIVE'
-while getopts t: flag
+taskSize=5000
+while getopts "t:s:h" option
 do
-  case "${flag}" in
+  case "${option}" in
     t)
       taskType=${OPTARG}
       ;;
+    s)
+      taskSize=${OPTARG}
+      ;;
+    *|h)
+      echo "Usage: $0 [-t taskType] [-s taskSize]"
+      exit 1
+      ;;
   esac
 done
 
-echo 'Running bench for task type:' $taskType
-export TASK_TYPE=$taskType
+echo 'Running benchmarks for task type:' ${taskType} 'and task size:' ${taskSize}
+export TASK_TYPE=${taskType}
+export TASK_SIZE=${taskSize}
 # Execute bench
 export NODE_ENV=production
 export POOL_SIZE=10
diff --git a/benchmarks/versus-external-pools/dynamic-piscina.js b/benchmarks/versus-external-pools/dynamic-piscina.js
index 61681dab..f31bb0bb 100644
--- a/benchmarks/versus-external-pools/dynamic-piscina.js
+++ b/benchmarks/versus-external-pools/dynamic-piscina.js
@@ -5,7 +5,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const piscina = new Piscina({
@@ -21,6 +22,7 @@ async function run () {
     promises.push(piscina.run(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/dynamic-poolifier.js b/benchmarks/versus-external-pools/dynamic-poolifier.js
index f466d28a..fe2a158f 100644
--- a/benchmarks/versus-external-pools/dynamic-poolifier.js
+++ b/benchmarks/versus-external-pools/dynamic-poolifier.js
@@ -5,7 +5,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const dynamicPool = new DynamicThreadPool(
@@ -20,6 +21,7 @@ async function run () {
     promises.push(dynamicPool.execute(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/dynamic-suchmokuo-node-worker-threads-pool.js b/benchmarks/versus-external-pools/dynamic-suchmokuo-node-worker-threads-pool.js
index fd0b7bd1..f6e32414 100644
--- a/benchmarks/versus-external-pools/dynamic-suchmokuo-node-worker-threads-pool.js
+++ b/benchmarks/versus-external-pools/dynamic-suchmokuo-node-worker-threads-pool.js
@@ -8,7 +8,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const pool = new DynamicPool(size)
@@ -24,6 +25,7 @@ async function run () {
     )
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/dynamic-worker-nodes.js b/benchmarks/versus-external-pools/dynamic-worker-nodes.js
index 6b860d5f..11a2b2d4 100644
--- a/benchmarks/versus-external-pools/dynamic-worker-nodes.js
+++ b/benchmarks/versus-external-pools/dynamic-worker-nodes.js
@@ -5,7 +5,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const workerNodes = new WorkerNodes(
@@ -23,6 +24,7 @@ async function run () {
     promises.push(workerNodes.call.functionToBench(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/dynamic-workerpool.js b/benchmarks/versus-external-pools/dynamic-workerpool.js
index 9b926936..d168633f 100644
--- a/benchmarks/versus-external-pools/dynamic-workerpool.js
+++ b/benchmarks/versus-external-pools/dynamic-workerpool.js
@@ -3,7 +3,7 @@ const workerpool = require('workerpool')
 // FINISH IMPORT LIBRARIES
 const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
-const dataArray = ['MYBENCH', process.env['TASK_TYPE']]
+const dataArray = ['MYBENCH', process.env.TASK_TYPE, process.env.TASK_SIZE]
 
 const workerPool = workerpool.pool(
   './workers/workerpool/function-to-bench-worker.js',
@@ -20,6 +20,7 @@ async function run () {
     promises.push(workerPool.exec('functionToBench', dataArray))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/fixed-microjob.js b/benchmarks/versus-external-pools/fixed-microjob.js
index 287d8a6f..2ab167aa 100644
--- a/benchmarks/versus-external-pools/fixed-microjob.js
+++ b/benchmarks/versus-external-pools/fixed-microjob.js
@@ -8,7 +8,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 async function run () {
@@ -20,11 +21,12 @@ async function run () {
         data => {
           functionToBench(data)
         },
-        { data: data, ctx: { functionToBench } }
+        { data, ctx: { functionToBench } }
       )
     )
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/fixed-piscina.js b/benchmarks/versus-external-pools/fixed-piscina.js
index 861fe4f6..b8785733 100644
--- a/benchmarks/versus-external-pools/fixed-piscina.js
+++ b/benchmarks/versus-external-pools/fixed-piscina.js
@@ -5,7 +5,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const piscina = new Piscina({
@@ -20,6 +21,7 @@ async function run () {
     promises.push(piscina.run(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/fixed-poolifier.js b/benchmarks/versus-external-pools/fixed-poolifier.js
index 28aa7e3a..38682040 100644
--- a/benchmarks/versus-external-pools/fixed-poolifier.js
+++ b/benchmarks/versus-external-pools/fixed-poolifier.js
@@ -5,7 +5,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const fixedPool = new FixedThreadPool(
@@ -19,6 +20,7 @@ async function run () {
     promises.push(fixedPool.execute(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/fixed-threadwork.js b/benchmarks/versus-external-pools/fixed-threadwork.js
index eb316b53..8a036fbd 100644
--- a/benchmarks/versus-external-pools/fixed-threadwork.js
+++ b/benchmarks/versus-external-pools/fixed-threadwork.js
@@ -4,7 +4,8 @@ const threadPool = require('./pool-threadwork')
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 async function run () {
@@ -13,6 +14,7 @@ async function run () {
     promises.push(threadPool.run(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/fixed-worker-nodes.js b/benchmarks/versus-external-pools/fixed-worker-nodes.js
index 58965d21..2d4e7c22 100644
--- a/benchmarks/versus-external-pools/fixed-worker-nodes.js
+++ b/benchmarks/versus-external-pools/fixed-worker-nodes.js
@@ -5,7 +5,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const workerNodes = new WorkerNodes(
@@ -23,6 +24,7 @@ async function run () {
     promises.push(workerNodes.call.functionToBench(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/fixed-workerpool.js b/benchmarks/versus-external-pools/fixed-workerpool.js
index cf60a366..95f44d0b 100644
--- a/benchmarks/versus-external-pools/fixed-workerpool.js
+++ b/benchmarks/versus-external-pools/fixed-workerpool.js
@@ -3,7 +3,7 @@ const workerpool = require('workerpool')
 // FINISH IMPORT LIBRARIES
 const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
-const dataArray = ['MYBENCH', process.env['TASK_TYPE']]
+const dataArray = ['MYBENCH', process.env.TASK_TYPE, process.env.TASK_SIZE]
 
 const workerPool = workerpool.pool(
   './workers/workerpool/function-to-bench-worker.js',
@@ -20,6 +20,7 @@ async function run () {
     promises.push(workerPool.exec('functionToBench', dataArray))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/functions/function-to-bench.js b/benchmarks/versus-external-pools/functions/function-to-bench.js
index b9054db2..7ecc7e12 100644
--- a/benchmarks/versus-external-pools/functions/function-to-bench.js
+++ b/benchmarks/versus-external-pools/functions/function-to-bench.js
@@ -1,9 +1,18 @@
+const fs = require('fs')
+
+const TaskTypes = {
+  CPU_INTENSIVE: 'CPU_INTENSIVE',
+  IO_INTENSIVE: 'IO_INTENSIVE'
+}
+
 module.exports = function (data) {
+  console.log('functionToBench', data)
   data = data || {}
-  data.taskType = data.taskType || 'CPU_INTENSIVE'
+  data.taskType = data.taskType || TaskTypes.CPU_INTENSIVE
   data.taskSize = data.taskSize || 5000
+  const benchmarksFilePath = '/tmp/poolifier-benchmarks'
   switch (data.taskType) {
-    case 'CPU_INTENSIVE':
+    case TaskTypes.CPU_INTENSIVE:
       // CPU intensive task
       for (let i = 0; i < data.taskSize; i++) {
         const o = {
@@ -12,6 +21,14 @@ module.exports = function (data) {
         JSON.stringify(o)
       }
       return { ok: 1 }
+    case TaskTypes.IO_INTENSIVE:
+      // IO intensive task
+      for (let i = 0; i < data.taskSize; i++) {
+        fs.writeFileSync(benchmarksFilePath, i.toString(), 'utf8')
+        fs.readFileSync(benchmarksFilePath, 'utf8')
+        fs.unlinkSync(benchmarksFilePath)
+      }
+      return { ok: 1 }
     default:
       throw new Error(`Unknown task type: ${data.taskType}`)
   }
diff --git a/benchmarks/versus-external-pools/pool-threadwork.js b/benchmarks/versus-external-pools/pool-threadwork.js
index 5e5e04e3..36baad1f 100644
--- a/benchmarks/versus-external-pools/pool-threadwork.js
+++ b/benchmarks/versus-external-pools/pool-threadwork.js
@@ -6,4 +6,4 @@ const functionToBench = require('./functions/function-to-bench')
 // FINISH IMPORT FUNCTION TO BENCH
 const size = Number(process.env.POOL_SIZE)
 
-module.exports = new ThreadPool({ task: functionToBench, size: size })
+module.exports = new ThreadPool({ task: functionToBench, size })
diff --git a/benchmarks/versus-external-pools/static-suchmokuo-node-worker-threads-pool.js b/benchmarks/versus-external-pools/static-suchmokuo-node-worker-threads-pool.js
index ded55b88..398c38ed 100644
--- a/benchmarks/versus-external-pools/static-suchmokuo-node-worker-threads-pool.js
+++ b/benchmarks/versus-external-pools/static-suchmokuo-node-worker-threads-pool.js
@@ -8,11 +8,12 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 const pool = new StaticPool({
-  size: size,
+  size,
   task: functionToBench
 })
 
@@ -22,6 +23,7 @@ async function run () {
     promises.push(pool.exec(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/threadjs.js b/benchmarks/versus-external-pools/threadjs.js
index 18a01cfd..5d7d64ed 100644
--- a/benchmarks/versus-external-pools/threadjs.js
+++ b/benchmarks/versus-external-pools/threadjs.js
@@ -5,7 +5,8 @@ const size = Number(process.env.POOL_SIZE)
 const iterations = Number(process.env.NUM_ITERATIONS)
 const data = {
   test: 'MYBENCH',
-  taskType: process.env['TASK_TYPE']
+  taskType: process.env.TASK_TYPE,
+  taskSize: process.env.TASK_SIZE
 }
 
 // Threads.js is not really a pool so we need to write few additional code
@@ -27,6 +28,7 @@ async function run () {
     promises.push(worker.exposedFunction(data))
   }
   await Promise.all(promises)
+  // eslint-disable-next-line no-process-exit
   process.exit()
 }
 
diff --git a/benchmarks/versus-external-pools/workers/workerpool/function-to-bench-worker.js b/benchmarks/versus-external-pools/workers/workerpool/function-to-bench-worker.js
index 02f777f7..a883ce43 100644
--- a/benchmarks/versus-external-pools/workers/workerpool/function-to-bench-worker.js
+++ b/benchmarks/versus-external-pools/workers/workerpool/function-to-bench-worker.js
@@ -2,8 +2,12 @@
 const workerpool = require('workerpool')
 const functionToBench = require('../../functions/function-to-bench')
 
-function workerPoolWrapperFunctionToBench (testName, taskType) {
-  return functionToBench({ test: testName, taskType: taskType })
+function workerPoolWrapperFunctionToBench (testName, taskType, taskSize) {
+  return functionToBench({
+    test: testName,
+    taskType,
+    taskSize
+  })
 }
 
 workerpool.worker({
diff --git a/examples/dynamicExample.js b/examples/dynamicExample.js
index 94c665a6..ed1bddeb 100644
--- a/examples/dynamicExample.js
+++ b/examples/dynamicExample.js
@@ -12,7 +12,7 @@ const iterations = 1000
 for (let i = 1; i <= iterations; i++) {
   pool
     .execute({})
-    .then(res => {
+    .then(() => {
       resolved++
       if (resolved === iterations) {
         console.log('Time take is ' + (Date.now() - start))
diff --git a/examples/fixedExample.js b/examples/fixedExample.js
index 07a57c3e..2c36a703 100644
--- a/examples/fixedExample.js
+++ b/examples/fixedExample.js
@@ -10,7 +10,7 @@ const iterations = 1000
 for (let i = 1; i <= iterations; i++) {
   pool
     .execute({})
-    .then(res => {
+    .then(() => {
       resolved++
       if (resolved === iterations) {
         return console.log('Time take is ' + (Date.now() - start))
diff --git a/examples/typescript/pool.ts b/examples/typescript/pool.ts
index 96b861c4..8b29e3be 100644
--- a/examples/typescript/pool.ts
+++ b/examples/typescript/pool.ts
@@ -1,6 +1,6 @@
 import { join } from 'path'
 import { DynamicThreadPool, FixedThreadPool } from 'poolifier'
-import { MyData, MyResponse } from './worker'
+import type { MyData, MyResponse } from './worker'
 
 export const fixedPool = new FixedThreadPool<MyData, Promise<MyResponse>>(
   8,
diff --git a/examples/yourWorker.js b/examples/yourWorker.js
index 3e59f346..5bd1ad65 100644
--- a/examples/yourWorker.js
+++ b/examples/yourWorker.js
@@ -4,7 +4,7 @@ const { isMainThread } = require('worker_threads')
 
 const debug = false
 
-function yourFunction (data) {
+function yourFunction () {
   for (let i = 0; i <= 1000; i++) {
     const o = {
       a: i
diff --git a/tsconfig.json b/tsconfig.json
index a0ad5d4c..442a4745 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -8,6 +8,6 @@
     "strict": true,
     "importsNotUsedAsValues": "error"
   },
-  "include": ["src/**/*.ts"],
+  "include": ["**/*.ts"],
   "exclude": ["node_modules"]
 }