From 7d2e3f590a08e6dd5ef104cd3b704e7d5d1c2dcd Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Thu, 6 Jul 2023 02:21:07 +0200 Subject: [PATCH] refactor: factor out code to redistribute queued tasks MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- package.json | 4 +- pnpm-lock.yaml | 136 +++++++++++++++++++------------------ src/pools/abstract-pool.ts | 54 ++++++++------- 3 files changed, 101 insertions(+), 93 deletions(-) diff --git a/package.json b/package.json index 2f39ab9c..ea3b707b 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "@release-it/keep-a-changelog": "^3.1.0", "@rollup/plugin-terser": "^0.4.3", "@rollup/plugin-typescript": "^11.1.2", - "@types/node": "^20.3.3", + "@types/node": "^20.4.0", "@typescript-eslint/eslint-plugin": "^5.61.0", "@typescript-eslint/parser": "^5.61.0", "benny": "^3.7.1", @@ -129,7 +129,7 @@ "mocha": "^10.2.0", "mochawesome": "^7.1.3", "prettier": "^2.8.8", - "release-it": "^15.11.0", + "release-it": "^16.0.0", "rollup": "^3.26.1", "rollup-plugin-analyzer": "^4.0.0", "rollup-plugin-command": "^1.1.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8ba4194..d90d0222 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,10 +13,10 @@ devDependencies: version: 17.6.6 '@release-it/bumper': specifier: ^4.0.2 - version: 4.0.2(release-it@15.11.0) + version: 4.0.2(release-it@16.0.0) '@release-it/keep-a-changelog': specifier: ^3.1.0 - version: 3.1.0(release-it@15.11.0) + version: 3.1.0(release-it@16.0.0) '@rollup/plugin-terser': specifier: ^0.4.3 version: 0.4.3(rollup@3.26.1) @@ -24,8 +24,8 @@ devDependencies: specifier: ^11.1.2 version: 11.1.2(rollup@3.26.1)(typescript@5.1.6) '@types/node': - specifier: ^20.3.3 - version: 20.3.3 + specifier: ^20.4.0 + version: 20.4.0 '@typescript-eslint/eslint-plugin': specifier: ^5.61.0 version: 5.61.0(@typescript-eslint/parser@5.61.0)(eslint@8.44.0)(typescript@5.1.6) @@ -93,8 +93,8 @@ devDependencies: specifier: ^2.8.8 version: 2.8.8 release-it: - specifier: ^15.11.0 - version: 15.11.0 + specifier: ^16.0.0 + version: 16.0.0 rollup: specifier: ^3.26.1 version: 3.26.1 @@ -270,15 +270,15 @@ packages: '@commitlint/execute-rule': 17.4.0 '@commitlint/resolve-extends': 17.4.4 '@commitlint/types': 17.4.4 - '@types/node': 20.3.3 + '@types/node': 20.4.0 chalk: 4.1.2 cosmiconfig: 8.2.0 - cosmiconfig-typescript-loader: 4.3.0(@types/node@20.3.3)(cosmiconfig@8.2.0)(ts-node@10.9.1)(typescript@5.1.6) + cosmiconfig-typescript-loader: 4.3.0(@types/node@20.4.0)(cosmiconfig@8.2.0)(ts-node@10.9.1)(typescript@5.1.6) lodash.isplainobject: 4.0.6 lodash.merge: 4.6.2 lodash.uniq: 4.5.0 resolve-from: 5.0.0 - ts-node: 10.9.1(@types/node@20.3.3)(typescript@5.1.6) + ts-node: 10.9.1(@types/node@20.4.0)(typescript@5.1.6) typescript: 5.1.6 transitivePeerDependencies: - '@swc/core' @@ -455,7 +455,7 @@ packages: '@jest/schemas': 29.6.0 '@types/istanbul-lib-coverage': 2.0.4 '@types/istanbul-reports': 3.0.1 - '@types/node': 20.3.3 + '@types/node': 20.4.0 '@types/yargs': 17.0.24 chalk: 4.1.2 dev: true @@ -643,8 +643,8 @@ packages: - encoding dev: true - /@octokit/rest@19.0.11: - resolution: {integrity: sha512-m2a9VhaP5/tUw8FwfnW2ICXlXpLPIqxtg3XcAiGMLj/Xhw3RSBfZ8le/466ktO1Gcjr8oXudGnHhxV1TXJgFxw==} + /@octokit/rest@19.0.13: + resolution: {integrity: sha512-/EzVox5V9gYGdbAI+ovYj3nXQT1TtTHRT+0eZPcuC05UFSWO3mdO9UY1C0i2eLF9Un1ONJkAk+IEtYGAC+TahA==} engines: {node: '>= 14'} dependencies: '@octokit/core': 4.2.4 @@ -704,7 +704,7 @@ packages: config-chain: 1.1.13 dev: true - /@release-it/bumper@4.0.2(release-it@15.11.0): + /@release-it/bumper@4.0.2(release-it@16.0.0): resolution: {integrity: sha512-HjiFIBNvb67cLTbzdm6EqiSRwpB4MJY4TGJFY/ac5qnIxERWK5gx1zBnwbJZffDSoS46hJMugoXrDbITEfX4gA==} engines: {node: '>=14'} peerDependencies: @@ -718,18 +718,18 @@ packages: lodash.castarray: 4.4.0 lodash.get: 4.4.2 lodash.set: 4.3.2 - release-it: 15.11.0 + release-it: 16.0.0 semver: 7.5.3 dev: true - /@release-it/keep-a-changelog@3.1.0(release-it@15.11.0): + /@release-it/keep-a-changelog@3.1.0(release-it@16.0.0): resolution: {integrity: sha512-o8lxZCAFwFkFpo88aIIgS5dR3kDy6yLjJVpe9QAGNY6tpKMMRHxsiJ8MS5NvrEeIC2VTzOoOjN/CDvdd3a4+/A==} engines: {node: '>=14'} peerDependencies: release-it: ^15.0.0-esm.4 dependencies: detect-newline: 4.0.0 - release-it: 15.11.0 + release-it: 16.0.0 string-template: 1.0.0 dev: true @@ -852,7 +852,7 @@ packages: resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==} dependencies: '@types/minimatch': 5.1.2 - '@types/node': 20.3.3 + '@types/node': 20.4.0 dev: true /@types/http-cache-semantics@4.0.1: @@ -891,8 +891,8 @@ packages: resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} dev: true - /@types/node@20.3.3: - resolution: {integrity: sha512-wheIYdr4NYML61AjC8MKj/2jrR/kDQri/CIpVoZwldwhnIrD/j9jIU5bJ8yBKuB2VhpFV7Ab6G2XkBjv9r9Zzw==} + /@types/node@20.4.0: + resolution: {integrity: sha512-jfT7iTf/4kOQ9S7CHV9BIyRaQqHu67mOjsIQBC3BKZvzvUB6zLxEwJ6sBE3ozcvP8kF6Uk5PXN0Q+c0dfhGX0g==} dev: true /@types/normalize-package-data@2.4.1: @@ -1353,7 +1353,7 @@ packages: dependencies: ansi-align: 3.0.1 camelcase: 7.0.1 - chalk: 5.2.0 + chalk: 5.3.0 cli-boxes: 3.0.0 string-width: 5.1.2 type-fest: 2.19.0 @@ -1523,6 +1523,11 @@ packages: engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} dev: true + /chalk@5.3.0: + resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + dev: true + /chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} dev: true @@ -1731,7 +1736,7 @@ packages: resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} dev: true - /cosmiconfig-typescript-loader@4.3.0(@types/node@20.3.3)(cosmiconfig@8.2.0)(ts-node@10.9.1)(typescript@5.1.6): + /cosmiconfig-typescript-loader@4.3.0(@types/node@20.4.0)(cosmiconfig@8.2.0)(ts-node@10.9.1)(typescript@5.1.6): resolution: {integrity: sha512-NTxV1MFfZDLPiBMjxbHRwSh5LaLcPMwNdCutmnHJCKoVnlvldPWlllonKwrsRJ5pYZBIBGRWWU2tfvzxgeSW5Q==} engines: {node: '>=12', npm: '>=6'} peerDependencies: @@ -1740,22 +1745,12 @@ packages: ts-node: '>=10' typescript: '>=3' dependencies: - '@types/node': 20.3.3 + '@types/node': 20.4.0 cosmiconfig: 8.2.0 - ts-node: 10.9.1(@types/node@20.3.3)(typescript@5.1.6) + ts-node: 10.9.1(@types/node@20.4.0)(typescript@5.1.6) typescript: 5.1.6 dev: true - /cosmiconfig@8.1.3: - resolution: {integrity: sha512-/UkO2JKI18b5jVMJUp0lvKFMpa/Gye+ZgZjKD+DGEN9y7NRcf/nK1A0sp67ONmKtnDCNMS44E6jrk0Yc3bDuUw==} - engines: {node: '>=14'} - dependencies: - import-fresh: 3.3.0 - js-yaml: 4.1.0 - parse-json: 5.2.0 - path-type: 4.0.0 - dev: true - /cosmiconfig@8.2.0: resolution: {integrity: sha512-3rTMnFJA1tCOPwRxtgF4wd7Ab2qvDbL8jX+3smjIbS4HlZBagTlpERbdN7iAbWlrfxE3M8c27kTwTawQ7st+OQ==} engines: {node: '>=14'} @@ -2278,7 +2273,7 @@ packages: eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.61.0)(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5)(eslint@8.44.0) eslint-plugin-import: 2.27.5(@typescript-eslint/parser@5.61.0)(eslint-import-resolver-typescript@3.5.5)(eslint@8.44.0) get-tsconfig: 4.6.2 - globby: 13.2.1 + globby: 13.2.2 is-core-module: 2.12.1 is-glob: 4.0.3 synckit: 0.8.5 @@ -2658,7 +2653,7 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: '@jest/expect-utils': 29.6.0 - '@types/node': 20.3.3 + '@types/node': 20.4.0 jest-get-type: 29.4.3 jest-matcher-utils: 29.6.0 jest-message-util: 29.6.0 @@ -3044,8 +3039,8 @@ packages: slash: 3.0.0 dev: true - /globby@13.1.4: - resolution: {integrity: sha512-iui/IiiW+QrJ1X1hKH5qwlMQyv34wJAYwH1vrf8b9kBA4sNiif3gKsMHa+BrdnOpEudWjpotfa7LrTzB1ERS/g==} + /globby@13.2.1: + resolution: {integrity: sha512-DPCBxctI7dN4EeIqjW2KGqgdcUMbrhJ9AzON+PlxCtvppWhubTLD4+a0GFxiym14ZvacUydTPjLPc2DlKz7EIg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dependencies: dir-glob: 3.0.1 @@ -3055,8 +3050,8 @@ packages: slash: 4.0.0 dev: true - /globby@13.2.1: - resolution: {integrity: sha512-DPCBxctI7dN4EeIqjW2KGqgdcUMbrhJ9AzON+PlxCtvppWhubTLD4+a0GFxiym14ZvacUydTPjLPc2DlKz7EIg==} + /globby@13.2.2: + resolution: {integrity: sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dependencies: dir-glob: 3.0.1 @@ -3089,6 +3084,23 @@ packages: responselike: 3.0.0 dev: true + /got@13.0.0: + resolution: {integrity: sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==} + engines: {node: '>=16'} + dependencies: + '@sindresorhus/is': 5.4.1 + '@szmarczak/http-timer': 5.0.1 + cacheable-lookup: 7.0.0 + cacheable-request: 10.2.12 + decompress-response: 6.0.0 + form-data-encoder: 2.1.4 + get-stream: 6.0.1 + http2-wrapper: 2.2.0 + lowercase-keys: 3.0.0 + p-cancelable: 3.0.0 + responselike: 3.0.0 + dev: true + /graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} dev: true @@ -3292,12 +3304,12 @@ packages: engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} dev: true - /inquirer@9.2.6: - resolution: {integrity: sha512-y71l237eJJKS4rl7sQcEUiMhrR0pB/ZnRMMTxLpjJhWL4hdWCT03a6jJnC1w6qIPSRZWEozuieGt3v7XaEJYFw==} + /inquirer@9.2.7: + resolution: {integrity: sha512-Bf52lnfvNxGPJPltiNO2tLBp3zC339KNlGMqOkW+dsvNikBhcVDK5kqU2lVX2FTPzuXUFX5WJDlsw//w3ZwoTw==} engines: {node: '>=14.18.0'} dependencies: ansi-escapes: 4.3.2 - chalk: 5.2.0 + chalk: 5.3.0 cli-cursor: 3.1.0 cli-width: 4.0.0 external-editor: 3.1.0 @@ -3723,7 +3735,7 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: '@jest/types': 29.6.0 - '@types/node': 20.3.3 + '@types/node': 20.4.0 chalk: 4.1.2 ci-info: 3.8.0 graceful-fs: 4.2.11 @@ -4060,7 +4072,7 @@ packages: resolution: {integrity: sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA==} engines: {node: '>=12'} dependencies: - chalk: 5.2.0 + chalk: 5.3.0 is-unicode-supported: 1.3.0 dev: true @@ -4574,7 +4586,7 @@ packages: resolution: {integrity: sha512-ERAyNnZOfqM+Ao3RAvIXkYh5joP220yf59gVe2X/cI6SiCxIdi4c9HZKZD8R6q/RDXEje1THBju6iExiSsgJaQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dependencies: - chalk: 5.2.0 + chalk: 5.3.0 cli-cursor: 4.0.0 cli-spinners: 2.9.0 is-interactive: 2.0.0 @@ -5033,21 +5045,21 @@ packages: rc: 1.2.8 dev: true - /release-it@15.11.0: - resolution: {integrity: sha512-lZwoGEnKYKwGnfxxlA7vtR7vvozPrOSsIgQaHO4bgQ5ARbG3IA6Dmo0IVusv6nR1KmnjH70QIeNAgsWs6Ji/tw==} - engines: {node: '>=14.9'} + /release-it@16.0.0: + resolution: {integrity: sha512-TRZ0gxHyA10666UypV8bMBT06+lj/RXZBqDnBPh4hmnTtZwPVxb7pK3QycV9XfxHKhaCh7XkJSmltG/OZ2+ksQ==} + engines: {node: '>=16'} hasBin: true dependencies: '@iarna/toml': 2.2.5 - '@octokit/rest': 19.0.11 + '@octokit/rest': 19.0.13 async-retry: 1.3.3 - chalk: 5.2.0 - cosmiconfig: 8.1.3 + chalk: 5.3.0 + cosmiconfig: 8.2.0 execa: 7.1.1 git-url-parse: 13.1.0 - globby: 13.1.4 - got: 12.6.1 - inquirer: 9.2.6 + globby: 13.2.1 + got: 13.0.0 + inquirer: 9.2.7 is-ci: 3.0.1 issue-parser: 6.0.0 lodash: 4.17.21 @@ -5059,7 +5071,7 @@ packages: os-name: 5.1.0 promise.allsettled: 1.0.6 proxy-agent: 6.2.1 - semver: 7.5.1 + semver: 7.5.3 shelljs: 0.8.5 update-notifier: 6.0.2 url-join: 5.0.0 @@ -5255,14 +5267,6 @@ packages: hasBin: true dev: true - /semver@7.5.1: - resolution: {integrity: sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==} - engines: {node: '>=10'} - hasBin: true - dependencies: - lru-cache: 6.0.0 - dev: true - /semver@7.5.2: resolution: {integrity: sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==} engines: {node: '>=10'} @@ -5713,7 +5717,7 @@ packages: engines: {node: '>=8'} dev: true - /ts-node@10.9.1(@types/node@20.3.3)(typescript@5.1.6): + /ts-node@10.9.1(@types/node@20.4.0)(typescript@5.1.6): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -5732,7 +5736,7 @@ packages: '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.3.3 + '@types/node': 20.4.0 acorn: 8.10.0 acorn-walk: 8.2.0 arg: 4.1.3 @@ -5930,7 +5934,7 @@ packages: engines: {node: '>=14.16'} dependencies: boxen: 7.1.1 - chalk: 5.2.0 + chalk: 5.3.0 configstore: 6.0.0 has-yarn: 3.0.0 import-lazy: 4.0.0 diff --git a/src/pools/abstract-pool.ts b/src/pools/abstract-pool.ts index 88f2c9e9..730c0dd2 100644 --- a/src/pools/abstract-pool.ts +++ b/src/pools/abstract-pool.ts @@ -886,31 +886,7 @@ export abstract class AbstractPool< this.emitter.emit(PoolEvents.error, error) } if (this.opts.enableTasksQueue === true) { - const workerNodeKey = this.getWorkerNodeKey(worker) - while (this.tasksQueueSize(workerNodeKey) > 0) { - let targetWorkerNodeKey: number = workerNodeKey - let minQueuedTasks = Infinity - for (const [workerNodeId, workerNode] of this.workerNodes.entries()) { - if ( - workerNodeId !== workerNodeKey && - workerNode.usage.tasks.queued === 0 - ) { - targetWorkerNodeKey = workerNodeId - break - } - if ( - workerNodeId !== workerNodeKey && - workerNode.usage.tasks.queued < minQueuedTasks - ) { - minQueuedTasks = workerNode.usage.tasks.queued - targetWorkerNodeKey = workerNodeId - } - } - this.enqueueTask( - targetWorkerNodeKey, - this.dequeueTask(workerNodeKey) as Task - ) - } + this.redistributeQueuedTasks(worker) } if (this.opts.restartWorkerOnError === true) { if (this.getWorkerInfo(this.getWorkerNodeKey(worker)).dynamic) { @@ -935,6 +911,34 @@ export abstract class AbstractPool< return worker } + private redistributeQueuedTasks (worker: Worker): void { + const workerNodeKey = this.getWorkerNodeKey(worker) + while (this.tasksQueueSize(workerNodeKey) > 0) { + let targetWorkerNodeKey: number = workerNodeKey + let minQueuedTasks = Infinity + for (const [workerNodeId, workerNode] of this.workerNodes.entries()) { + if ( + workerNodeId !== workerNodeKey && + workerNode.usage.tasks.queued === 0 + ) { + targetWorkerNodeKey = workerNodeId + break + } + if ( + workerNodeId !== workerNodeKey && + workerNode.usage.tasks.queued < minQueuedTasks + ) { + minQueuedTasks = workerNode.usage.tasks.queued + targetWorkerNodeKey = workerNodeId + } + } + this.enqueueTask( + targetWorkerNodeKey, + this.dequeueTask(workerNodeKey) as Task + ) + } + } + /** * Creates a new dynamic worker and sets it up completely in the pool worker nodes. * -- 2.34.1