From 11856b30474f13160594beac70555f3fdd34038b Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Thu, 10 Aug 2023 18:51:26 +0200 Subject: [PATCH] feat: add node-fetch multithreaded example MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit reference: #790 Signed-off-by: Jérôme Benoit --- .../http-client/node-fetch/httpd-echo.js | 25 ++++++ .../http-client/node-fetch/package.json | 30 +++++++ .../http-client/node-fetch/pnpm-lock.yaml | 78 +++++++++++++++++++ .../http-client/node-fetch/src/main.ts | 25 ++++++ .../http-client/node-fetch/src/pool.ts | 20 +++++ .../http-client/node-fetch/src/types.ts | 11 +++ .../http-client/node-fetch/src/worker.ts | 22 ++++++ .../http-client/node-fetch/tsconfig.json | 14 ++++ 8 files changed, 225 insertions(+) create mode 100644 examples/typescript/http-client/node-fetch/httpd-echo.js create mode 100644 examples/typescript/http-client/node-fetch/package.json create mode 100644 examples/typescript/http-client/node-fetch/pnpm-lock.yaml create mode 100644 examples/typescript/http-client/node-fetch/src/main.ts create mode 100644 examples/typescript/http-client/node-fetch/src/pool.ts create mode 100644 examples/typescript/http-client/node-fetch/src/types.ts create mode 100644 examples/typescript/http-client/node-fetch/src/worker.ts create mode 100644 examples/typescript/http-client/node-fetch/tsconfig.json diff --git a/examples/typescript/http-client/node-fetch/httpd-echo.js b/examples/typescript/http-client/node-fetch/httpd-echo.js new file mode 100644 index 00000000..53b8ad11 --- /dev/null +++ b/examples/typescript/http-client/node-fetch/httpd-echo.js @@ -0,0 +1,25 @@ +import { Server } from 'node:http' + +const port = 8080 +const server = new Server() + +server + .on('request', (request, response) => { + let body = [] + request + .on('data', chunk => { + body.push(chunk) + }) + .on('end', () => { + body = Buffer.concat(body).toString() + + console.info(`==== ${request.method} ${request.url} ====`) + console.info('> Headers') + console.log(request.headers) + + console.info('> Body') + console.info(body) + response.end() + }) + }) + .listen(port) diff --git a/examples/typescript/http-client/node-fetch/package.json b/examples/typescript/http-client/node-fetch/package.json new file mode 100644 index 00000000..3aa26176 --- /dev/null +++ b/examples/typescript/http-client/node-fetch/package.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json.schemastore.org/package", + "name": "http-client-node-fetch", + "version": "1.0.0", + "description": "multithreaded node-fetch", + "main": "dist/main.js", + "type": "module", + "volta": { + "node": "20.5.0", + "pnpm": "8.6.12" + }, + "scripts": { + "build": "pnpm build:clean && npx tsc", + "build:clean": "npx tsc --build --clean", + "start": "node dist/main.js", + "start:httpd-echo": "node dist/httpd-echo.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "node-fetch": "^3.3.2", + "poolifier": "^2.6.22" + }, + "devDependencies": { + "@types/node": "^20.4.9", + "typescript": "^5.1.6" + } +} diff --git a/examples/typescript/http-client/node-fetch/pnpm-lock.yaml b/examples/typescript/http-client/node-fetch/pnpm-lock.yaml new file mode 100644 index 00000000..80df5f9c --- /dev/null +++ b/examples/typescript/http-client/node-fetch/pnpm-lock.yaml @@ -0,0 +1,78 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 + poolifier: + specifier: ^2.6.22 + version: 2.6.22 + +devDependencies: + '@types/node': + specifier: ^20.4.9 + version: 20.4.9 + typescript: + specifier: ^5.1.6 + version: 5.1.6 + +packages: + + /@types/node@20.4.9: + resolution: {integrity: sha512-8e2HYcg7ohnTUbHk8focoklEQYvemQmu9M/f43DZVx43kHn0tE3BY/6gSDxS7k0SprtS0NHvj+L80cGLnoOUcQ==} + dev: true + + /data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + dev: false + + /fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.2.1 + dev: false + + /formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: false + + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + + /node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: false + + /poolifier@2.6.22: + resolution: {integrity: sha512-0pGU1nG8jVEQUb2j1kkiEQ5TdJyHP3a9zFVxF7Q23xHiGM2hpw9BpWvjP0kvE6LmCx0R8ezFDwaXf9nDUOA9gQ==} + engines: {node: '>=16.14.0', pnpm: '>=8.6.0'} + requiresBuild: true + dev: false + + /typescript@5.1.6: + resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} + engines: {node: '>=14.17'} + hasBin: true + dev: true + + /web-streams-polyfill@3.2.1: + resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} + engines: {node: '>= 8'} + dev: false diff --git a/examples/typescript/http-client/node-fetch/src/main.ts b/examples/typescript/http-client/node-fetch/src/main.ts new file mode 100644 index 00000000..86760929 --- /dev/null +++ b/examples/typescript/http-client/node-fetch/src/main.ts @@ -0,0 +1,25 @@ +import { availableParallelism } from 'poolifier' +import { fetchPool } from './pool.js' +import { type WorkerResponse } from './types.js' + +const parallelism = availableParallelism() +const requestUrl = 'http://localhost:8080/' + +const fetchPoolPromises = new Set>() +for (let i = 0; i < availableParallelism(); i++) { + fetchPoolPromises.add(fetchPool.execute({ url: requestUrl })) +} + +try { + const now = performance.now() + const responses = await Promise.all(fetchPoolPromises) + const elapsedTime = performance.now() - now + console.info( + `Received in ${elapsedTime.toFixed(2)}ms an array with ${ + responses.length + } responses from ${parallelism} parallel requests made with node-fetch on ${requestUrl}:\n`, + responses + ) +} catch (error) { + console.error(error) +} diff --git a/examples/typescript/http-client/node-fetch/src/pool.ts b/examples/typescript/http-client/node-fetch/src/pool.ts new file mode 100644 index 00000000..77ceacba --- /dev/null +++ b/examples/typescript/http-client/node-fetch/src/pool.ts @@ -0,0 +1,20 @@ +import { fileURLToPath } from 'node:url' +import { dirname, extname, join } from 'node:path' +import { FixedThreadPool, availableParallelism } from 'poolifier' +import { type WorkerData, type WorkerResponse } from './types.js' + +const workerFile = join( + dirname(fileURLToPath(import.meta.url)), + `worker${extname(fileURLToPath(import.meta.url))}` +) + +export const fetchPool = new FixedThreadPool( + availableParallelism(), + workerFile, + { + enableTasksQueue: true, + errorHandler: (e: Error) => { + console.error(e) + } + } +) diff --git a/examples/typescript/http-client/node-fetch/src/types.ts b/examples/typescript/http-client/node-fetch/src/types.ts new file mode 100644 index 00000000..4ac3a8ff --- /dev/null +++ b/examples/typescript/http-client/node-fetch/src/types.ts @@ -0,0 +1,11 @@ +import { type URL } from 'node:url' +import { type RequestInfo, type RequestInit } from 'node-fetch' + +export interface WorkerData { + url: URL | RequestInfo + init?: RequestInit +} + +export interface WorkerResponse { + text: string +} diff --git a/examples/typescript/http-client/node-fetch/src/worker.ts b/examples/typescript/http-client/node-fetch/src/worker.ts new file mode 100644 index 00000000..6f0d3651 --- /dev/null +++ b/examples/typescript/http-client/node-fetch/src/worker.ts @@ -0,0 +1,22 @@ +import { ThreadWorker } from 'poolifier' +import fetch from 'node-fetch' +import { type WorkerData, type WorkerResponse } from './types.js' + +class FetchWorker extends ThreadWorker { + public constructor () { + super(async (workerData?: WorkerData) => { + const response = await fetch( + (workerData as WorkerData).url, + workerData?.init + ) + // The response is not structured-cloneable, so we return the response text body instead. + return { + text: await response.text() + } + }) + } +} + +const fetchWorker = new FetchWorker() + +export { fetchWorker } diff --git a/examples/typescript/http-client/node-fetch/tsconfig.json b/examples/typescript/http-client/node-fetch/tsconfig.json new file mode 100644 index 00000000..079bc9e1 --- /dev/null +++ b/examples/typescript/http-client/node-fetch/tsconfig.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "Node16", + "rootDir": "./src", + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true + } +} -- 2.34.1