X-Git-Url: https://git.piment-noir.org/?a=blobdiff_plain;f=src%2Fpools%2Fabstract-pool.ts;h=6081902720554c2bcdb809d37f04d5f93ed89da7;hb=db746922cea3decbb34b5878d56a468d659d33f5;hp=41e7bb24cd4f69653e0e11b7d74b636bbda8a25c;hpb=20c6f652269cbc14fea53910c8066900f56c2f25;p=poolifier.git

diff --git a/src/pools/abstract-pool.ts b/src/pools/abstract-pool.ts
index 41e7bb24..60819027 100644
--- a/src/pools/abstract-pool.ts
+++ b/src/pools/abstract-pool.ts
@@ -5,12 +5,14 @@ import { type TransferListItem } from 'node:worker_threads'
 import type {
   MessageValue,
   PromiseResponseWrapper,
-  Task
+  Task,
+  Writable
 } from '../utility-types'
 import {
   DEFAULT_TASK_NAME,
   DEFAULT_WORKER_CHOICE_STRATEGY_OPTIONS,
   EMPTY_FUNCTION,
+  average,
   isKillBehavior,
   isPlainObject,
   median,
@@ -93,6 +95,10 @@ export abstract class AbstractPool<
    * Whether the pool is starting or not.
    */
   private readonly starting: boolean
+  /**
+   * Whether the pool is started or not.
+   */
+  private started: boolean
   /**
    * The start timestamp of the pool.
    */
@@ -141,6 +147,7 @@ export abstract class AbstractPool<
     this.starting = true
     this.startPool()
     this.starting = false
+    this.started = true
 
     this.startTimestamp = performance.now()
   }
@@ -285,7 +292,7 @@ export abstract class AbstractPool<
   }
 
   private checkValidTasksQueueOptions (
-    tasksQueueOptions: TasksQueueOptions
+    tasksQueueOptions: Writable<TasksQueueOptions>
   ): void {
     if (tasksQueueOptions != null && !isPlainObject(tasksQueueOptions)) {
       throw new TypeError('Invalid tasks queue options: must be a plain object')
@@ -308,18 +315,26 @@ export abstract class AbstractPool<
     }
     if (
       tasksQueueOptions?.queueMaxSize != null &&
-      !Number.isSafeInteger(tasksQueueOptions.queueMaxSize)
+      tasksQueueOptions?.size != null
     ) {
-      throw new TypeError(
-        'Invalid worker node tasks queue max size: must be an integer'
+      throw new Error(
+        'Invalid tasks queue options: cannot specify both queueMaxSize and size'
       )
     }
+    if (tasksQueueOptions?.queueMaxSize != null) {
+      tasksQueueOptions.size = tasksQueueOptions.queueMaxSize
+    }
     if (
-      tasksQueueOptions?.queueMaxSize != null &&
-      tasksQueueOptions.queueMaxSize <= 0
+      tasksQueueOptions?.size != null &&
+      !Number.isSafeInteger(tasksQueueOptions.size)
     ) {
+      throw new TypeError(
+        'Invalid worker node tasks queue max size: must be an integer'
+      )
+    }
+    if (tasksQueueOptions?.size != null && tasksQueueOptions.size <= 0) {
       throw new RangeError(
-        `Invalid worker node tasks queue max size: ${tasksQueueOptions.queueMaxSize} is a negative integer or zero`
+        `Invalid worker node tasks queue max size: ${tasksQueueOptions.size} is a negative integer or zero`
       )
     }
   }
@@ -390,6 +405,13 @@ export abstract class AbstractPool<
       ...(this.opts.enableTasksQueue === true && {
         backPressure: this.hasBackPressure()
       }),
+      ...(this.opts.enableTasksQueue === true && {
+        stolenTasks: this.workerNodes.reduce(
+          (accumulator, workerNode) =>
+            accumulator + workerNode.usage.tasks.stolen,
+          0
+        )
+      }),
       failedTasks: this.workerNodes.reduce(
         (accumulator, workerNode) =>
           accumulator + workerNode.usage.tasks.failed,
@@ -412,24 +434,26 @@ export abstract class AbstractPool<
               )
             )
           ),
-          average: round(
-            this.workerNodes.reduce(
-              (accumulator, workerNode) =>
-                accumulator + (workerNode.usage.runTime?.aggregate ?? 0),
-              0
-            ) /
-              this.workerNodes.reduce(
-                (accumulator, workerNode) =>
-                  accumulator + (workerNode.usage.tasks?.executed ?? 0),
-                0
+          ...(this.workerChoiceStrategyContext.getTaskStatisticsRequirements()
+            .runTime.average && {
+            average: round(
+              average(
+                this.workerNodes.reduce<number[]>(
+                  (accumulator, workerNode) =>
+                    accumulator.concat(workerNode.usage.runTime.history),
+                  []
+                )
               )
-          ),
+            )
+          }),
           ...(this.workerChoiceStrategyContext.getTaskStatisticsRequirements()
             .runTime.median && {
             median: round(
               median(
-                this.workerNodes.map(
-                  (workerNode) => workerNode.usage.runTime?.median ?? 0
+                this.workerNodes.reduce<number[]>(
+                  (accumulator, workerNode) =>
+                    accumulator.concat(workerNode.usage.runTime.history),
+                  []
                 )
               )
             )
@@ -453,24 +477,26 @@ export abstract class AbstractPool<
               )
             )
           ),
-          average: round(
-            this.workerNodes.reduce(
-              (accumulator, workerNode) =>
-                accumulator + (workerNode.usage.waitTime?.aggregate ?? 0),
-              0
-            ) /
-              this.workerNodes.reduce(
-                (accumulator, workerNode) =>
-                  accumulator + (workerNode.usage.tasks?.executed ?? 0),
-                0
+          ...(this.workerChoiceStrategyContext.getTaskStatisticsRequirements()
+            .waitTime.average && {
+            average: round(
+              average(
+                this.workerNodes.reduce<number[]>(
+                  (accumulator, workerNode) =>
+                    accumulator.concat(workerNode.usage.waitTime.history),
+                  []
+                )
               )
-          ),
+            )
+          }),
           ...(this.workerChoiceStrategyContext.getTaskStatisticsRequirements()
             .waitTime.median && {
             median: round(
               median(
-                this.workerNodes.map(
-                  (workerNode) => workerNode.usage.waitTime?.median ?? 0
+                this.workerNodes.reduce<number[]>(
+                  (accumulator, workerNode) =>
+                    accumulator.concat(workerNode.usage.waitTime.history),
+                  []
                 )
               )
             )
@@ -636,17 +662,15 @@ export abstract class AbstractPool<
       this.checkValidTasksQueueOptions(tasksQueueOptions)
       this.opts.tasksQueueOptions =
         this.buildTasksQueueOptions(tasksQueueOptions)
-      this.setTasksQueueMaxSize(
-        this.opts.tasksQueueOptions.queueMaxSize as number
-      )
+      this.setTasksQueueMaxSize(this.opts.tasksQueueOptions.size as number)
     } else if (this.opts.tasksQueueOptions != null) {
       delete this.opts.tasksQueueOptions
     }
   }
 
-  private setTasksQueueMaxSize (queueMaxSize: number): void {
+  private setTasksQueueMaxSize (size: number): void {
     for (const workerNode of this.workerNodes) {
-      workerNode.tasksQueueBackPressureSize = queueMaxSize
+      workerNode.tasksQueueBackPressureSize = size
     }
   }
 
@@ -655,7 +679,7 @@ export abstract class AbstractPool<
   ): TasksQueueOptions {
     return {
       ...{
-        queueMaxSize: Math.pow(this.maxSize, 2),
+        size: Math.pow(this.maxSize, 2),
         concurrency: 1
       },
       ...tasksQueueOptions
@@ -723,6 +747,9 @@ export abstract class AbstractPool<
     transferList?: TransferListItem[]
   ): Promise<Response> {
     return await new Promise<Response>((resolve, reject) => {
+      if (!this.started) {
+        reject(new Error('Cannot execute a task on destroyed pool'))
+      }
       if (name != null && typeof name !== 'string') {
         reject(new TypeError('name argument must be a string'))
       }
@@ -765,6 +792,7 @@ export abstract class AbstractPool<
       if (
         this.opts.enableTasksQueue === false ||
         (this.opts.enableTasksQueue === true &&
+          this.tasksQueueSize(workerNodeKey) === 0 &&
           this.workerNodes[workerNodeKey].usage.tasks.executing <
             (this.opts.tasksQueueOptions?.concurrency as number))
       ) {
@@ -783,6 +811,7 @@ export abstract class AbstractPool<
       })
     )
     this.emitter?.emit(PoolEvents.destroy, this.info)
+    this.started = false
   }
 
   protected async sendKillMessageToWorker (
@@ -815,7 +844,7 @@ export abstract class AbstractPool<
    * @virtual
    */
   protected setupHook (): void {
-    // Intentionally empty
+    /** Intentionally empty */
   }
 
   /**
@@ -879,7 +908,7 @@ export abstract class AbstractPool<
       const taskFunctionWorkerUsage = this.workerNodes[
         workerNodeKey
       ].getTaskFunctionWorkerUsage(
-        message.taskPerformance?.name ?? DEFAULT_TASK_NAME
+        message.taskPerformance?.name as string
       ) as WorkerUsage
       this.updateTaskStatisticsWorkerUsage(taskFunctionWorkerUsage, message)
       this.updateRunTimeWorkerUsage(taskFunctionWorkerUsage, message)
@@ -912,13 +941,6 @@ export abstract class AbstractPool<
       workerTaskStatistics.executing > 0
     ) {
       --workerTaskStatistics.executing
-    } else if (
-      workerTaskStatistics.executing != null &&
-      workerTaskStatistics.executing < 0
-    ) {
-      throw new Error(
-        'Worker usage statistic for tasks executing cannot be negative'
-      )
     }
     if (message.taskError == null) {
       ++workerTaskStatistics.executed
@@ -931,11 +953,13 @@ export abstract class AbstractPool<
     workerUsage: WorkerUsage,
     message: MessageValue<Response>
   ): void {
+    if (message.taskError != null) {
+      return
+    }
     updateMeasurementStatistics(
       workerUsage.runTime,
       this.workerChoiceStrategyContext.getTaskStatisticsRequirements().runTime,
-      message.taskPerformance?.runTime ?? 0,
-      workerUsage.tasks.executed
+      message.taskPerformance?.runTime ?? 0
     )
   }
 
@@ -948,8 +972,7 @@ export abstract class AbstractPool<
     updateMeasurementStatistics(
       workerUsage.waitTime,
       this.workerChoiceStrategyContext.getTaskStatisticsRequirements().waitTime,
-      taskWaitTime,
-      workerUsage.tasks.executed
+      taskWaitTime
     )
   }
 
@@ -957,19 +980,20 @@ export abstract class AbstractPool<
     workerUsage: WorkerUsage,
     message: MessageValue<Response>
   ): void {
+    if (message.taskError != null) {
+      return
+    }
     const eluTaskStatisticsRequirements: MeasurementStatisticsRequirements =
       this.workerChoiceStrategyContext.getTaskStatisticsRequirements().elu
     updateMeasurementStatistics(
       workerUsage.elu.active,
       eluTaskStatisticsRequirements,
-      message.taskPerformance?.elu?.active ?? 0,
-      workerUsage.tasks.executed
+      message.taskPerformance?.elu?.active ?? 0
     )
     updateMeasurementStatistics(
       workerUsage.elu.idle,
       eluTaskStatisticsRequirements,
-      message.taskPerformance?.elu?.idle ?? 0,
-      workerUsage.tasks.executed
+      message.taskPerformance?.elu?.idle ?? 0
     )
     if (eluTaskStatisticsRequirements.aggregate) {
       if (message.taskPerformance?.elu != null) {
@@ -1050,7 +1074,11 @@ export abstract class AbstractPool<
       workerInfo.ready = false
       this.workerNodes[workerNodeKey].closeChannel()
       this.emitter?.emit(PoolEvents.error, error)
-      if (this.opts.restartWorkerOnError === true && !this.starting) {
+      if (
+        this.opts.restartWorkerOnError === true &&
+        !this.starting &&
+        this.started
+      ) {
         if (workerInfo.dynamic) {
           this.createAndSetupDynamicWorkerNode()
         } else {
@@ -1142,6 +1170,12 @@ export abstract class AbstractPool<
     this.sendStartupMessageToWorker(workerNodeKey)
     // Send the statistics message to worker.
     this.sendStatisticsMessageToWorker(workerNodeKey)
+    if (this.opts.enableTasksQueue === true) {
+      this.workerNodes[workerNodeKey].onEmptyQueue =
+        this.taskStealingOnEmptyQueue.bind(this)
+      this.workerNodes[workerNodeKey].onBackPressure =
+        this.tasksStealingOnBackPressure.bind(this)
+    }
   }
 
   /**
@@ -1171,44 +1205,132 @@ export abstract class AbstractPool<
 
   private redistributeQueuedTasks (workerNodeKey: number): void {
     while (this.tasksQueueSize(workerNodeKey) > 0) {
-      let targetWorkerNodeKey: number = workerNodeKey
+      let destinationWorkerNodeKey!: number
       let minQueuedTasks = Infinity
-      let executeTask = false
       for (const [workerNodeId, workerNode] of this.workerNodes.entries()) {
-        const workerInfo = this.getWorkerInfo(workerNodeId) as WorkerInfo
+        if (workerNode.info.ready && workerNodeId !== workerNodeKey) {
+          if (workerNode.usage.tasks.queued === 0) {
+            destinationWorkerNodeKey = workerNodeId
+            break
+          }
+          if (workerNode.usage.tasks.queued < minQueuedTasks) {
+            minQueuedTasks = workerNode.usage.tasks.queued
+            destinationWorkerNodeKey = workerNodeId
+          }
+        }
+      }
+      if (destinationWorkerNodeKey != null) {
+        const destinationWorkerNode = this.workerNodes[destinationWorkerNodeKey]
+        const task = {
+          ...(this.dequeueTask(workerNodeKey) as Task<Data>),
+          workerId: destinationWorkerNode.info.id as number
+        }
         if (
-          workerNodeId !== workerNodeKey &&
-          workerInfo.ready &&
-          workerNode.usage.tasks.queued === 0
+          this.tasksQueueSize(destinationWorkerNodeKey) === 0 &&
+          destinationWorkerNode.usage.tasks.executing <
+            (this.opts.tasksQueueOptions?.concurrency as number)
         ) {
-          if (
-            this.workerNodes[workerNodeId].usage.tasks.executing <
+          this.executeTask(destinationWorkerNodeKey, task)
+        } else {
+          this.enqueueTask(destinationWorkerNodeKey, task)
+        }
+      }
+    }
+  }
+
+  private taskStealingOnEmptyQueue (workerId: number): void {
+    const destinationWorkerNodeKey = this.getWorkerNodeKeyByWorkerId(workerId)
+    const destinationWorkerNode = this.workerNodes[destinationWorkerNodeKey]
+    const workerNodes = this.workerNodes
+      .slice()
+      .sort(
+        (workerNodeA, workerNodeB) =>
+          workerNodeB.usage.tasks.queued - workerNodeA.usage.tasks.queued
+      )
+    for (const sourceWorkerNode of workerNodes) {
+      if (sourceWorkerNode.usage.tasks.queued === 0) {
+        break
+      }
+      if (
+        sourceWorkerNode.info.ready &&
+        sourceWorkerNode.info.id !== workerId &&
+        sourceWorkerNode.usage.tasks.queued > 0
+      ) {
+        const task = {
+          ...(sourceWorkerNode.popTask() as Task<Data>),
+          workerId: destinationWorkerNode.info.id as number
+        }
+        if (
+          this.tasksQueueSize(destinationWorkerNodeKey) === 0 &&
+          destinationWorkerNode.usage.tasks.executing <
             (this.opts.tasksQueueOptions?.concurrency as number)
-          ) {
-            executeTask = true
-          }
-          targetWorkerNodeKey = workerNodeId
-          break
+        ) {
+          this.executeTask(destinationWorkerNodeKey, task)
+        } else {
+          this.enqueueTask(destinationWorkerNodeKey, task)
+        }
+        if (destinationWorkerNode?.usage != null) {
+          ++destinationWorkerNode.usage.tasks.stolen
         }
         if (
-          workerNodeId !== workerNodeKey &&
-          workerInfo.ready &&
-          workerNode.usage.tasks.queued < minQueuedTasks
+          this.shallUpdateTaskFunctionWorkerUsage(destinationWorkerNodeKey) &&
+          destinationWorkerNode.getTaskFunctionWorkerUsage(
+            task.name as string
+          ) != null
         ) {
-          minQueuedTasks = workerNode.usage.tasks.queued
-          targetWorkerNodeKey = workerNodeId
+          const taskFunctionWorkerUsage =
+            destinationWorkerNode.getTaskFunctionWorkerUsage(
+              task.name as string
+            ) as WorkerUsage
+          ++taskFunctionWorkerUsage.tasks.stolen
         }
+        break
       }
-      if (executeTask) {
-        this.executeTask(
-          targetWorkerNodeKey,
-          this.dequeueTask(workerNodeKey) as Task<Data>
-        )
-      } else {
-        this.enqueueTask(
-          targetWorkerNodeKey,
-          this.dequeueTask(workerNodeKey) as Task<Data>
-        )
+    }
+  }
+
+  private tasksStealingOnBackPressure (workerId: number): void {
+    const sourceWorkerNode =
+      this.workerNodes[this.getWorkerNodeKeyByWorkerId(workerId)]
+    const workerNodes = this.workerNodes
+      .slice()
+      .sort(
+        (workerNodeA, workerNodeB) =>
+          workerNodeA.usage.tasks.queued - workerNodeB.usage.tasks.queued
+      )
+    for (const [workerNodeKey, workerNode] of workerNodes.entries()) {
+      if (
+        sourceWorkerNode.usage.tasks.queued > 0 &&
+        workerNode.info.ready &&
+        workerNode.info.id !== workerId &&
+        workerNode.usage.tasks.queued <
+          (this.opts.tasksQueueOptions?.size as number) - 1
+      ) {
+        const task = {
+          ...(sourceWorkerNode.popTask() as Task<Data>),
+          workerId: workerNode.info.id as number
+        }
+        if (
+          this.tasksQueueSize(workerNodeKey) === 0 &&
+          workerNode.usage.tasks.executing <
+            (this.opts.tasksQueueOptions?.concurrency as number)
+        ) {
+          this.executeTask(workerNodeKey, task)
+        } else {
+          this.enqueueTask(workerNodeKey, task)
+        }
+        if (workerNode?.usage != null) {
+          ++workerNode.usage.tasks.stolen
+        }
+        if (
+          this.shallUpdateTaskFunctionWorkerUsage(workerNodeKey) &&
+          workerNode.getTaskFunctionWorkerUsage(task.name as string) != null
+        ) {
+          const taskFunctionWorkerUsage = workerNode.getTaskFunctionWorkerUsage(
+            task.name as string
+          ) as WorkerUsage
+          ++taskFunctionWorkerUsage.tasks.stolen
+        }
       }
     }
   }
@@ -1321,7 +1443,7 @@ export abstract class AbstractPool<
     const workerNode = new WorkerNode<Worker, Data>(
       worker,
       this.worker,
-      this.opts.tasksQueueOptions?.queueMaxSize ?? Math.pow(this.maxSize, 2)
+      this.opts.tasksQueueOptions?.size ?? Math.pow(this.maxSize, 2)
     )
     // Flag the worker node as ready at pool startup.
     if (this.starting) {