X-Git-Url: https://git.piment-noir.org/?a=blobdiff_plain;f=src%2Fpools%2Fabstract-pool.ts;h=631434f37618e4727b2a8dcdbee119c309965da1;hb=47aacbaa07c16cb4a11fd5d46323db47cb948b41;hp=661eb7b6ec363c228ad0c540f43b44361b76e75e;hpb=875a7c37808f9dd37369348b5b4574881d5f9c32;p=poolifier.git

diff --git a/src/pools/abstract-pool.ts b/src/pools/abstract-pool.ts
index 661eb7b6..631434f3 100644
--- a/src/pools/abstract-pool.ts
+++ b/src/pools/abstract-pool.ts
@@ -1,31 +1,45 @@
 import crypto from 'node:crypto'
 import type { MessageValue, PromiseResponseWrapper } from '../utility-types'
-import { EMPTY_FUNCTION, median } from '../utils'
+import {
+  DEFAULT_WORKER_CHOICE_STRATEGY_OPTIONS,
+  EMPTY_FUNCTION,
+  isPlainObject,
+  median
+} from '../utils'
 import { KillBehaviors, isKillBehavior } from '../worker/worker-options'
-import { PoolEvents, type PoolOptions } from './pool'
-import { PoolEmitter } from './pool'
-import type { IPoolInternal } from './pool-internal'
-import { PoolType } from './pool-internal'
+import { CircularArray } from '../circular-array'
+import { Queue } from '../queue'
+import {
+  type IPool,
+  PoolEmitter,
+  PoolEvents,
+  type PoolInfo,
+  type PoolOptions,
+  type PoolType,
+  PoolTypes,
+  type TasksQueueOptions,
+  type WorkerType
+} from './pool'
 import type { IWorker, Task, TasksUsage, WorkerNode } from './worker'
 import {
   WorkerChoiceStrategies,
-  type WorkerChoiceStrategy
+  type WorkerChoiceStrategy,
+  type WorkerChoiceStrategyOptions
 } from './selection-strategies/selection-strategies-types'
 import { WorkerChoiceStrategyContext } from './selection-strategies/worker-choice-strategy-context'
-import { CircularArray } from '../circular-array'
 
 /**
  * Base class that implements some shared logic for all poolifier pools.
  *
  * @typeParam Worker - Type of worker which manages this pool.
  * @typeParam Data - Type of data sent to the worker. This can only be serializable data.
- * @typeParam Response - Type of response of execution. This can only be serializable data.
+ * @typeParam Response - Type of execution response. This can only be serializable data.
  */
 export abstract class AbstractPool<
   Worker extends IWorker,
   Data = unknown,
   Response = unknown
-> implements IPoolInternal<Worker, Data, Response> {
+> implements IPool<Worker, Data, Response> {
   /** @inheritDoc */
   public readonly workerNodes: Array<WorkerNode<Worker, Data>> = []
 
@@ -60,7 +74,7 @@ export abstract class AbstractPool<
    * Constructs a new poolifier pool.
    *
    * @param numberOfWorkers - Number of workers that this pool should manage.
-   * @param filePath - Path to the worker-file.
+   * @param filePath - Path to the worker file.
    * @param opts - Options for the pool.
    */
   public constructor (
@@ -75,10 +89,10 @@ export abstract class AbstractPool<
     this.checkFilePath(this.filePath)
     this.checkPoolOptions(this.opts)
 
-    this.chooseWorkerNode.bind(this)
-    this.executeTask.bind(this)
-    this.enqueueTask.bind(this)
-    this.checkAndEmitEvents.bind(this)
+    this.chooseWorkerNode = this.chooseWorkerNode.bind(this)
+    this.executeTask = this.executeTask.bind(this)
+    this.enqueueTask = this.enqueueTask.bind(this)
+    this.checkAndEmitEvents = this.checkAndEmitEvents.bind(this)
 
     this.setupHook()
 
@@ -116,25 +130,42 @@ export abstract class AbstractPool<
       )
     } else if (!Number.isSafeInteger(numberOfWorkers)) {
       throw new TypeError(
-        'Cannot instantiate a pool with a non integer number of workers'
+        'Cannot instantiate a pool with a non safe integer number of workers'
       )
     } else if (numberOfWorkers < 0) {
       throw new RangeError(
         'Cannot instantiate a pool with a negative number of workers'
       )
-    } else if (this.type === PoolType.FIXED && numberOfWorkers === 0) {
+    } else if (this.type === PoolTypes.fixed && numberOfWorkers === 0) {
       throw new Error('Cannot instantiate a fixed pool with no worker')
     }
   }
 
   private checkPoolOptions (opts: PoolOptions<Worker>): void {
-    this.opts.workerChoiceStrategy =
-      opts.workerChoiceStrategy ?? WorkerChoiceStrategies.ROUND_ROBIN
-    this.checkValidWorkerChoiceStrategy(this.opts.workerChoiceStrategy)
-    this.opts.workerChoiceStrategyOptions =
-      opts.workerChoiceStrategyOptions ?? { medRunTime: false }
-    this.opts.enableEvents = opts.enableEvents ?? true
-    this.opts.enableTasksQueue = opts.enableTasksQueue ?? false
+    if (isPlainObject(opts)) {
+      this.opts.workerChoiceStrategy =
+        opts.workerChoiceStrategy ?? WorkerChoiceStrategies.ROUND_ROBIN
+      this.checkValidWorkerChoiceStrategy(this.opts.workerChoiceStrategy)
+      this.opts.workerChoiceStrategyOptions =
+        opts.workerChoiceStrategyOptions ??
+        DEFAULT_WORKER_CHOICE_STRATEGY_OPTIONS
+      this.checkValidWorkerChoiceStrategyOptions(
+        this.opts.workerChoiceStrategyOptions
+      )
+      this.opts.restartWorkerOnError = opts.restartWorkerOnError ?? true
+      this.opts.enableEvents = opts.enableEvents ?? true
+      this.opts.enableTasksQueue = opts.enableTasksQueue ?? false
+      if (this.opts.enableTasksQueue) {
+        this.checkValidTasksQueueOptions(
+          opts.tasksQueueOptions as TasksQueueOptions
+        )
+        this.opts.tasksQueueOptions = this.buildTasksQueueOptions(
+          opts.tasksQueueOptions as TasksQueueOptions
+        )
+      }
+    } else {
+      throw new TypeError('Invalid pool options: must be a plain object')
+    }
   }
 
   private checkValidWorkerChoiceStrategy (
@@ -147,31 +178,91 @@ export abstract class AbstractPool<
     }
   }
 
+  private checkValidWorkerChoiceStrategyOptions (
+    workerChoiceStrategyOptions: WorkerChoiceStrategyOptions
+  ): void {
+    if (!isPlainObject(workerChoiceStrategyOptions)) {
+      throw new TypeError(
+        'Invalid worker choice strategy options: must be a plain object'
+      )
+    }
+    if (
+      workerChoiceStrategyOptions.weights != null &&
+      Object.keys(workerChoiceStrategyOptions.weights).length !== this.maxSize
+    ) {
+      throw new Error(
+        'Invalid worker choice strategy options: must have a weight for each worker node'
+      )
+    }
+  }
+
+  private checkValidTasksQueueOptions (
+    tasksQueueOptions: TasksQueueOptions
+  ): void {
+    if (tasksQueueOptions != null && !isPlainObject(tasksQueueOptions)) {
+      throw new TypeError('Invalid tasks queue options: must be a plain object')
+    }
+    if ((tasksQueueOptions?.concurrency as number) <= 0) {
+      throw new Error(
+        `Invalid worker tasks concurrency '${
+          tasksQueueOptions.concurrency as number
+        }'`
+      )
+    }
+  }
+
   /** @inheritDoc */
   public abstract get type (): PoolType
 
+  /** @inheritDoc */
+  public get info (): PoolInfo {
+    return {
+      type: this.type,
+      worker: this.worker,
+      minSize: this.minSize,
+      maxSize: this.maxSize,
+      workerNodes: this.workerNodes.length,
+      idleWorkerNodes: this.workerNodes.reduce(
+        (accumulator, workerNode) =>
+          workerNode.tasksUsage.running === 0 ? accumulator + 1 : accumulator,
+        0
+      ),
+      busyWorkerNodes: this.workerNodes.reduce(
+        (accumulator, workerNode) =>
+          workerNode.tasksUsage.running > 0 ? accumulator + 1 : accumulator,
+        0
+      ),
+      runningTasks: this.workerNodes.reduce(
+        (accumulator, workerNode) =>
+          accumulator + workerNode.tasksUsage.running,
+        0
+      ),
+      queuedTasks: this.workerNodes.reduce(
+        (accumulator, workerNode) => accumulator + workerNode.tasksQueue.size,
+        0
+      ),
+      maxQueuedTasks: this.workerNodes.reduce(
+        (accumulator, workerNode) =>
+          accumulator + workerNode.tasksQueue.maxSize,
+        0
+      )
+    }
+  }
+
   /**
-   * Number of tasks running in the pool.
+   * Gets the worker type.
    */
-  private get numberOfRunningTasks (): number {
-    return this.workerNodes.reduce(
-      (accumulator, workerNode) => accumulator + workerNode.tasksUsage.running,
-      0
-    )
-  }
+  protected abstract get worker (): WorkerType
 
   /**
-   * Number of tasks queued in the pool.
+   * Pool minimum size.
    */
-  private get numberOfQueuedTasks (): number {
-    if (this.opts.enableTasksQueue === false) {
-      return 0
-    }
-    return this.workerNodes.reduce(
-      (accumulator, workerNode) => accumulator + workerNode.tasksQueue.length,
-      0
-    )
-  }
+  protected abstract get minSize (): number
+
+  /**
+   * Pool maximum size.
+   */
+  protected abstract get maxSize (): number
 
   /**
    * Gets the given worker its worker node key.
@@ -187,7 +278,8 @@ export abstract class AbstractPool<
 
   /** @inheritDoc */
   public setWorkerChoiceStrategy (
-    workerChoiceStrategy: WorkerChoiceStrategy
+    workerChoiceStrategy: WorkerChoiceStrategy,
+    workerChoiceStrategyOptions?: WorkerChoiceStrategyOptions
   ): void {
     this.checkValidWorkerChoiceStrategy(workerChoiceStrategy)
     this.opts.workerChoiceStrategy = workerChoiceStrategy
@@ -199,57 +291,115 @@ export abstract class AbstractPool<
         runTimeHistory: new CircularArray(),
         avgRunTime: 0,
         medRunTime: 0,
+        waitTime: 0,
+        waitTimeHistory: new CircularArray(),
+        avgWaitTime: 0,
+        medWaitTime: 0,
         error: 0
       })
     }
     this.workerChoiceStrategyContext.setWorkerChoiceStrategy(
-      workerChoiceStrategy
+      this.opts.workerChoiceStrategy
     )
+    if (workerChoiceStrategyOptions != null) {
+      this.setWorkerChoiceStrategyOptions(workerChoiceStrategyOptions)
+    }
   }
 
   /** @inheritDoc */
-  public abstract get full (): boolean
+  public setWorkerChoiceStrategyOptions (
+    workerChoiceStrategyOptions: WorkerChoiceStrategyOptions
+  ): void {
+    this.checkValidWorkerChoiceStrategyOptions(workerChoiceStrategyOptions)
+    this.opts.workerChoiceStrategyOptions = workerChoiceStrategyOptions
+    this.workerChoiceStrategyContext.setOptions(
+      this.opts.workerChoiceStrategyOptions
+    )
+  }
 
   /** @inheritDoc */
-  public abstract get busy (): boolean
+  public enableTasksQueue (
+    enable: boolean,
+    tasksQueueOptions?: TasksQueueOptions
+  ): void {
+    if (this.opts.enableTasksQueue === true && !enable) {
+      this.flushTasksQueues()
+    }
+    this.opts.enableTasksQueue = enable
+    this.setTasksQueueOptions(tasksQueueOptions as TasksQueueOptions)
+  }
+
+  /** @inheritDoc */
+  public setTasksQueueOptions (tasksQueueOptions: TasksQueueOptions): void {
+    if (this.opts.enableTasksQueue === true) {
+      this.checkValidTasksQueueOptions(tasksQueueOptions)
+      this.opts.tasksQueueOptions =
+        this.buildTasksQueueOptions(tasksQueueOptions)
+    } else {
+      delete this.opts.tasksQueueOptions
+    }
+  }
+
+  private buildTasksQueueOptions (
+    tasksQueueOptions: TasksQueueOptions
+  ): TasksQueueOptions {
+    return {
+      concurrency: tasksQueueOptions?.concurrency ?? 1
+    }
+  }
+
+  /**
+   * Whether the pool is full or not.
+   *
+   * The pool filling boolean status.
+   */
+  protected abstract get full (): boolean
+
+  /**
+   * Whether the pool is busy or not.
+   *
+   * The pool busyness boolean status.
+   */
+  protected abstract get busy (): boolean
 
   protected internalBusy (): boolean {
     return (
-      this.numberOfRunningTasks >= this.numberOfWorkers &&
-      this.findFreeWorkerNodeKey() === -1
+      this.workerNodes.findIndex(workerNode => {
+        return workerNode.tasksUsage.running === 0
+      }) === -1
     )
   }
 
   /** @inheritDoc */
-  public findFreeWorkerNodeKey (): number {
-    return this.workerNodes.findIndex(workerNode => {
-      return workerNode.tasksUsage?.running === 0
-    })
-  }
-
-  /** @inheritDoc */
-  public async execute (data: Data): Promise<Response> {
-    const [workerNodeKey, workerNode] = this.chooseWorkerNode()
+  public async execute (data?: Data, name?: string): Promise<Response> {
+    const submissionTimestamp = performance.now()
+    const workerNodeKey = this.chooseWorkerNode()
     const submittedTask: Task<Data> = {
+      name,
       // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
       data: data ?? ({} as Data),
+      submissionTimestamp,
       id: crypto.randomUUID()
     }
     const res = new Promise<Response>((resolve, reject) => {
-      this.promiseResponseMap.set(submittedTask.id, {
+      this.promiseResponseMap.set(submittedTask.id as string, {
         resolve,
         reject,
-        worker: workerNode.worker
+        worker: this.workerNodes[workerNodeKey].worker
       })
     })
     if (
       this.opts.enableTasksQueue === true &&
-      (this.busy || this.workerNodes[workerNodeKey].tasksUsage.running > 0)
+      (this.busy ||
+        this.workerNodes[workerNodeKey].tasksUsage.running >=
+          ((this.opts.tasksQueueOptions as TasksQueueOptions)
+            .concurrency as number))
     ) {
       this.enqueueTask(workerNodeKey, submittedTask)
     } else {
       this.executeTask(workerNodeKey, submittedTask)
     }
+    this.workerChoiceStrategyContext.update(workerNodeKey)
     this.checkAndEmitEvents()
     // eslint-disable-next-line @typescript-eslint/return-await
     return res
@@ -260,6 +410,7 @@ export abstract class AbstractPool<
     await Promise.all(
       this.workerNodes.map(async (workerNode, workerNodeKey) => {
         this.flushTasksQueue(workerNodeKey)
+        // FIXME: wait for tasks to be finished
         await this.destroyWorker(workerNode.worker)
       })
     )
@@ -308,12 +459,21 @@ export abstract class AbstractPool<
     worker: Worker,
     message: MessageValue<Response>
   ): void {
-    const workerTasksUsage = this.getWorkerTasksUsage(worker) as TasksUsage
+    const workerTasksUsage =
+      this.workerNodes[this.getWorkerNodeKey(worker)].tasksUsage
     --workerTasksUsage.running
     ++workerTasksUsage.run
     if (message.error != null) {
       ++workerTasksUsage.error
     }
+    this.updateRunTimeTasksUsage(workerTasksUsage, message)
+    this.updateWaitTimeTasksUsage(workerTasksUsage, message)
+  }
+
+  private updateRunTimeTasksUsage (
+    workerTasksUsage: TasksUsage,
+    message: MessageValue<Response>
+  ): void {
     if (this.workerChoiceStrategyContext.getRequiredStatistics().runTime) {
       workerTasksUsage.runTime += message.runTime ?? 0
       if (
@@ -323,44 +483,68 @@ export abstract class AbstractPool<
         workerTasksUsage.avgRunTime =
           workerTasksUsage.runTime / workerTasksUsage.run
       }
-      if (this.workerChoiceStrategyContext.getRequiredStatistics().medRunTime) {
-        workerTasksUsage.runTimeHistory.push(message.runTime ?? 0)
+      if (
+        this.workerChoiceStrategyContext.getRequiredStatistics().medRunTime &&
+        message.runTime != null
+      ) {
+        workerTasksUsage.runTimeHistory.push(message.runTime)
         workerTasksUsage.medRunTime = median(workerTasksUsage.runTimeHistory)
       }
     }
   }
 
+  private updateWaitTimeTasksUsage (
+    workerTasksUsage: TasksUsage,
+    message: MessageValue<Response>
+  ): void {
+    if (this.workerChoiceStrategyContext.getRequiredStatistics().waitTime) {
+      workerTasksUsage.waitTime += message.waitTime ?? 0
+      if (
+        this.workerChoiceStrategyContext.getRequiredStatistics().avgWaitTime &&
+        workerTasksUsage.run !== 0
+      ) {
+        workerTasksUsage.avgWaitTime =
+          workerTasksUsage.waitTime / workerTasksUsage.run
+      }
+      if (
+        this.workerChoiceStrategyContext.getRequiredStatistics().medWaitTime &&
+        message.waitTime != null
+      ) {
+        workerTasksUsage.waitTimeHistory.push(message.waitTime)
+        workerTasksUsage.medWaitTime = median(workerTasksUsage.waitTimeHistory)
+      }
+    }
+  }
+
   /**
    * Chooses a worker node for the next task.
    *
-   * The default uses a round robin algorithm to distribute the load.
+   * The default worker choice strategy uses a round robin algorithm to distribute the load.
    *
-   * @returns [worker node key, worker node].
+   * @returns The worker node key
    */
-  protected chooseWorkerNode (): [number, WorkerNode<Worker, Data>] {
+  protected chooseWorkerNode (): number {
     let workerNodeKey: number
-    if (
-      this.type === PoolType.DYNAMIC &&
-      !this.full &&
-      this.findFreeWorkerNodeKey() === -1
-    ) {
+    if (this.type === PoolTypes.dynamic && !this.full && this.internalBusy()) {
       const workerCreated = this.createAndSetupWorker()
       this.registerWorkerMessageListener(workerCreated, message => {
+        const currentWorkerNodeKey = this.getWorkerNodeKey(workerCreated)
         if (
           isKillBehavior(KillBehaviors.HARD, message.kill) ||
           (message.kill != null &&
-            this.getWorkerTasksUsage(workerCreated)?.running === 0)
+            this.workerNodes[currentWorkerNodeKey].tasksUsage.running === 0)
         ) {
           // Kill message received from the worker: no new tasks are submitted to that worker for a while ( > maxInactiveTime)
-          this.flushTasksQueueByWorker(workerCreated)
-          void this.destroyWorker(workerCreated)
+          this.flushTasksQueue(currentWorkerNodeKey)
+          // FIXME: wait for tasks to be finished
+          void (this.destroyWorker(workerCreated) as Promise<void>)
         }
       })
       workerNodeKey = this.getWorkerNodeKey(workerCreated)
     } else {
       workerNodeKey = this.workerChoiceStrategyContext.execute()
     }
-    return [workerNodeKey, this.workerNodes[workerNodeKey]]
+    return workerNodeKey
   }
 
   /**
@@ -408,6 +592,16 @@ export abstract class AbstractPool<
 
     worker.on('message', this.opts.messageHandler ?? EMPTY_FUNCTION)
     worker.on('error', this.opts.errorHandler ?? EMPTY_FUNCTION)
+    worker.on('error', error => {
+      if (this.emitter != null) {
+        this.emitter.emit(PoolEvents.error, error)
+      }
+    })
+    if (this.opts.restartWorkerOnError === true) {
+      worker.on('error', () => {
+        this.createAndSetupWorker()
+      })
+    }
     worker.on('online', this.opts.onlineHandler ?? EMPTY_FUNCTION)
     worker.on('exit', this.opts.exitHandler ?? EMPTY_FUNCTION)
     worker.once('exit', () => {
@@ -434,6 +628,12 @@ export abstract class AbstractPool<
         if (promiseResponse != null) {
           if (message.error != null) {
             promiseResponse.reject(message.error)
+            if (this.emitter != null) {
+              this.emitter.emit(PoolEvents.taskError, {
+                error: message.error,
+                errorData: message.errorData
+              })
+            }
           } else {
             promiseResponse.resolve(message.data as Response)
           }
@@ -455,12 +655,12 @@ export abstract class AbstractPool<
   }
 
   private checkAndEmitEvents (): void {
-    if (this.opts.enableEvents === true) {
+    if (this.emitter != null) {
       if (this.busy) {
-        this.emitter?.emit(PoolEvents.busy)
+        this.emitter?.emit(PoolEvents.busy, this.info)
       }
-      if (this.type === PoolType.DYNAMIC && this.full) {
-        this.emitter?.emit(PoolEvents.full)
+      if (this.type === PoolTypes.dynamic && this.full) {
+        this.emitter?.emit(PoolEvents.full, this.info)
       }
     }
   }
@@ -478,20 +678,6 @@ export abstract class AbstractPool<
     workerNode.tasksUsage = tasksUsage
   }
 
-  /**
-   * Gets the given worker its tasks usage in the pool.
-   *
-   * @param worker - The worker.
-   * @returns The worker tasks usage.
-   */
-  private getWorkerTasksUsage (worker: Worker): TasksUsage | undefined {
-    const workerNodeKey = this.getWorkerNodeKey(worker)
-    if (workerNodeKey !== -1) {
-      return this.workerNodes[workerNodeKey].tasksUsage
-    }
-    throw new Error('Worker could not be found in the pool worker nodes')
-  }
-
   /**
    * Pushes the given worker in the pool worker nodes.
    *
@@ -508,9 +694,13 @@ export abstract class AbstractPool<
         runTimeHistory: new CircularArray(),
         avgRunTime: 0,
         medRunTime: 0,
+        waitTime: 0,
+        waitTimeHistory: new CircularArray(),
+        avgWaitTime: 0,
+        medWaitTime: 0,
         error: 0
       },
-      tasksQueue: []
+      tasksQueue: new Queue<Task<Data>>()
     })
   }
 
@@ -526,7 +716,7 @@ export abstract class AbstractPool<
     workerNodeKey: number,
     worker: Worker,
     tasksUsage: TasksUsage,
-    tasksQueue: Array<Task<Data>>
+    tasksQueue: Queue<Task<Data>>
   ): void {
     this.workerNodes[workerNodeKey] = {
       worker,
@@ -542,8 +732,10 @@ export abstract class AbstractPool<
    */
   private removeWorkerNode (worker: Worker): void {
     const workerNodeKey = this.getWorkerNodeKey(worker)
-    this.workerNodes.splice(workerNodeKey, 1)
-    this.workerChoiceStrategyContext.remove(workerNodeKey)
+    if (workerNodeKey !== -1) {
+      this.workerNodes.splice(workerNodeKey, 1)
+      this.workerChoiceStrategyContext.remove(workerNodeKey)
+    }
   }
 
   private executeTask (workerNodeKey: number, task: Task<Data>): void {
@@ -551,28 +743,32 @@ export abstract class AbstractPool<
     this.sendToWorker(this.workerNodes[workerNodeKey].worker, task)
   }
 
-  private enqueueTask (workerNodeKey: number, task: Task<Data>): void {
-    this.workerNodes[workerNodeKey].tasksQueue.push(task)
+  private enqueueTask (workerNodeKey: number, task: Task<Data>): number {
+    return this.workerNodes[workerNodeKey].tasksQueue.enqueue(task)
   }
 
   private dequeueTask (workerNodeKey: number): Task<Data> | undefined {
-    return this.workerNodes[workerNodeKey].tasksQueue.shift()
+    return this.workerNodes[workerNodeKey].tasksQueue.dequeue()
   }
 
   private tasksQueueSize (workerNodeKey: number): number {
-    return this.workerNodes[workerNodeKey].tasksQueue.length
+    return this.workerNodes[workerNodeKey].tasksQueue.size
   }
 
   private flushTasksQueue (workerNodeKey: number): void {
     if (this.tasksQueueSize(workerNodeKey) > 0) {
-      for (const task of this.workerNodes[workerNodeKey].tasksQueue) {
-        this.executeTask(workerNodeKey, task)
+      for (let i = 0; i < this.tasksQueueSize(workerNodeKey); i++) {
+        this.executeTask(
+          workerNodeKey,
+          this.dequeueTask(workerNodeKey) as Task<Data>
+        )
       }
     }
   }
 
-  private flushTasksQueueByWorker (worker: Worker): void {
-    const workerNodeKey = this.getWorkerNodeKey(worker)
-    this.flushTasksQueue(workerNodeKey)
+  private flushTasksQueues (): void {
+    for (const [workerNodeKey] of this.workerNodes.entries()) {
+      this.flushTasksQueue(workerNodeKey)
+    }
   }
 }