├── .nvmrc ├── example ├── .gitignore ├── package.json ├── components │ ├── processors │ │ ├── punctcount.js │ │ ├── checksum.js │ │ └── wordcount.js │ └── generators │ │ └── helloworld.js ├── index.js └── README.md ├── src ├── components │ └── _ │ │ ├── generators │ │ ├── not-a-component.txt │ │ ├── tick.ts │ │ └── tock.ts │ │ └── processors │ │ └── print.ts ├── lib │ └── bakeryjs │ │ ├── queue │ │ └── PriorityQueueI.ts │ │ ├── builders │ │ ├── VisualBuilder.ts │ │ ├── DefaultVisualBuilder.ts │ │ └── __tests__ │ │ │ └── DefaultVisualBuilder.test.ts │ │ ├── FlowSchemaReaderI.ts │ │ ├── Job.ts │ │ ├── ComponentFactoryI.ts │ │ ├── boxCore │ │ ├── index.ts │ │ ├── validation.ts │ │ └── boxFactory.ts │ │ ├── processingStrategies │ │ ├── index.ts │ │ ├── AggregatorStrategy.ts │ │ ├── ProcessingMode.ts │ │ ├── MapperStrategy.ts │ │ ├── ProcessingStrategy.ts │ │ ├── processingStrategyRegistry.ts │ │ └── GeneratorStrategy.ts │ │ ├── eval │ │ └── every.ts │ │ ├── componentNameParser.ts │ │ ├── Box.ts │ │ ├── FlowSchemaReader.ts │ │ ├── FlowFactory.ts │ │ ├── __tests__ │ │ └── componentNameParser.test.ts │ │ ├── BoxI.ts │ │ ├── scanComponentsPath.ts │ │ ├── stats.ts │ │ ├── FlowCatalog.ts │ │ ├── ServiceProvider.ts │ │ ├── FlowBuilderI.ts │ │ ├── BoxEvents.ts │ │ ├── ComponentFactory.ts │ │ ├── Message.ts │ │ └── errors │ │ └── BoxErrorFactory.ts ├── index.ts ├── types │ └── sb-jsnetworkx.d.ts └── flows │ └── flows.ts ├── tests ├── components │ └── _ │ │ ├── generators │ │ ├── not-a-component.txt │ │ ├── tick.ts │ │ └── tock.ts │ │ └── processors │ │ └── print.ts ├── regressions.test.ts ├── integration │ ├── components │ │ ├── processors │ │ │ ├── identity-processor.ts │ │ │ ├── field-a-provider.ts │ │ │ ├── field-b-provider.ts │ │ │ ├── transform-processor.ts │ │ │ ├── logger-processor.ts │ │ │ ├── merge-fields.ts │ │ │ ├── field-reader.ts │ │ │ ├── error-processor.ts │ │ │ ├── non-error-thrower.ts │ │ │ ├── batch-error-processor.ts │ │ │ ├── slow-processor.ts │ │ │ ├── async-error-processor.ts │ │ │ ├── field-provider.ts │ │ │ ├── parameter-reader.ts │ │ │ ├── custom-service-processor.ts │ │ │ ├── accumulator-processor.ts │ │ │ ├── conditional-error-processor.ts │ │ │ └── batch-processor.ts │ │ └── generators │ │ │ ├── empty-generator.ts │ │ │ ├── error-generator.ts │ │ │ ├── configurable-generator.ts │ │ │ ├── nested-generator.ts │ │ │ └── priority-generator.ts │ └── helpers │ │ ├── test-program.ts │ │ ├── message-collector.ts │ │ └── event-tracker.ts └── scanComponentsPath.int.test.ts ├── benchmarks ├── results │ ├── benchmark-2025-12-14T07-32-45-328Z.json │ ├── benchmark-2025-12-14T07-36-17-132Z.json │ ├── benchmark-2025-12-14T07-40-20-098Z.json │ ├── benchmark-2025-12-14T07-42-28-771Z.json │ ├── benchmark-2025-12-14T10-53-39-475Z.json │ ├── benchmark-2025-12-14T11-57-52-193Z.json │ ├── partial-complex-1000x1000-1765698148770.json │ ├── partial-complex-1000x1000-1765697565328.json │ ├── partial-complex-1000x1000-1765697777132.json │ ├── partial-complex-1000x1000-1765698020098.json │ ├── partial-complex-5000x200-1765709619474.json │ ├── partial-complex-10000x100-1765713472193.json │ ├── benchmark-2025-12-14T07-06-35-386Z.json │ ├── benchmark-2025-12-14T07-12-00-411Z.json │ ├── benchmark-2025-12-14T07-12-02-897Z.json │ ├── benchmark-2025-12-14T07-44-39-182Z.json │ ├── benchmark-2025-12-14T07-55-16-640Z.json │ ├── benchmark-2025-12-14T07-57-28-862Z.json │ ├── benchmark-2025-12-14T07-12-10-193Z.json │ ├── benchmark-2025-12-14T12-02-23-364Z.json │ ├── benchmark-2025-12-14T12-02-46-444Z.json │ ├── benchmark-2025-12-14T07-12-12-960Z.json │ ├── benchmark-2025-12-14T11-58-20-694Z.json │ ├── benchmark-2025-12-14T10-50-02-708Z.json │ ├── benchmark-2025-12-15T06-33-00-893Z.json │ ├── benchmark-2025-12-15T08-45-13-508Z.json │ ├── benchmark-2025-12-15T09-43-48-946Z.json │ ├── benchmark-2025-12-15T09-44-12-497Z.json │ ├── benchmark-2025-12-15T06-01-21-074Z.json │ ├── benchmark-2025-12-15T08-55-39-129Z.json │ ├── benchmark-2025-12-14T09-21-40-812Z.json │ ├── benchmark-2025-12-14T10-48-38-374Z.json │ ├── benchmark-2025-12-15T06-30-50-933Z.json │ └── benchmark-2025-12-15T06-31-06-110Z.json ├── output │ ├── index.ts │ ├── resultFormatter.ts │ └── resultWriter.ts ├── utils │ └── git.ts ├── components │ ├── processors │ │ ├── mapper1.ts │ │ ├── mapper2.ts │ │ ├── mapper3.ts │ │ ├── mapper9.ts │ │ ├── mapper6.ts │ │ ├── mapper7.ts │ │ ├── mapper8.ts │ │ ├── mapper4.ts │ │ ├── mapper5.ts │ │ └── timed-mapper.ts │ └── generators │ │ ├── nested-generator.ts │ │ └── configurable-generator.ts ├── flows │ ├── simpleFlow.ts │ └── complexFlow.ts ├── config │ └── index.ts ├── runner │ ├── memoryTracker.ts │ └── eventHandlers.ts ├── cli │ └── index.ts ├── progress │ └── index.ts ├── run.ts └── types.ts ├── assets └── logo.png ├── .vscode └── settings.json ├── profiling ├── results │ └── .gitkeep ├── profiles │ └── .gitkeep ├── run.ts ├── parser │ └── index.ts ├── reporter │ └── index.ts ├── analyze.ts └── analyzer │ └── thresholds.ts ├── tsconfig.build.json ├── .prettierrc ├── test-data ├── processors │ ├── punctcount.ts │ ├── wordcount.ts │ ├── wordbatchcount.ts │ ├── wordbatchcountslow.ts │ └── checksum.ts └── generators │ ├── bad.ts │ ├── helloworld.ts │ └── hellobatchworld.ts ├── .editorconfig ├── jest.setup.js ├── jest.config.js ├── LICENSE ├── tsconfig.json ├── CONTRIBUTING.md ├── .gitignore ├── .travis.yml ├── package.json └── eslint.config.mjs /.nvmrc: -------------------------------------------------------------------------------- 1 | 24 2 | -------------------------------------------------------------------------------- /example/.gitignore: -------------------------------------------------------------------------------- 1 | package-lock.json 2 | -------------------------------------------------------------------------------- /src/components/_/generators/not-a-component.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/components/_/generators/not-a-component.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-32-45-328Z.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-36-17-132Z.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-40-20-098Z.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-42-28-771Z.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T10-53-39-475Z.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T11-57-52-193Z.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Emplifi/BakeryJS/HEAD/assets/logo.png -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "eslint.validate": [ 3 | "javascript", 4 | "javascriptreact", 5 | "typescript" 6 | ] 7 | } -------------------------------------------------------------------------------- /profiling/results/.gitkeep: -------------------------------------------------------------------------------- 1 | # This directory stores profile analysis results (JSON) 2 | # These files are gitignored as they are generated output 3 | 4 | -------------------------------------------------------------------------------- /profiling/profiles/.gitkeep: -------------------------------------------------------------------------------- 1 | # This directory stores generated CPU profile files (.cpuprofile) 2 | # These files are gitignored as they are large and machine-specific 3 | 4 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/queue/PriorityQueueI.ts: -------------------------------------------------------------------------------- 1 | export interface PriorityQueueI { 2 | push(message: T | T[], priority?: number): Promise | void 3 | length: number 4 | source?: string 5 | target: string 6 | } 7 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/builders/VisualBuilder.ts: -------------------------------------------------------------------------------- 1 | import type { FlowExplicitDescription } from '../FlowBuilderI' 2 | 3 | export interface VisualBuilder { 4 | build(schema: FlowExplicitDescription): Promise | string 5 | } 6 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["src/**/*.ts"], 4 | "exclude": ["node_modules", "**/node_modules/*", "**/__tests__/*"], 5 | "compilerOptions": { 6 | "sourceMap": false 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "tabWidth": 2, 5 | "trailingComma": "none", 6 | "printWidth": 100, 7 | "bracketSpacing": true, 8 | "arrowParens": "avoid", 9 | "endOfLine": "lf" 10 | } 11 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/FlowSchemaReaderI.ts: -------------------------------------------------------------------------------- 1 | import type { FlowExplicitDescription } from './FlowBuilderI' 2 | 3 | export default interface FlowSchemaReaderI { 4 | getFlowSchema(name: string): Promise | FlowExplicitDescription 5 | } 6 | -------------------------------------------------------------------------------- /benchmarks/output/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Output module for benchmark runner 3 | * Re-exports result writing and formatting utilities 4 | */ 5 | 6 | export { saveResults, savePartialResults, PartialResult } from './resultWriter' 7 | export { formatResult, printSummaryComparison } from './resultFormatter' 8 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/Job.ts: -------------------------------------------------------------------------------- 1 | import type { MessageData } from './Message' 2 | 3 | let jobId = 0 4 | 5 | export class Job { 6 | public readonly jobId: string 7 | 8 | public constructor(jobInitialValue?: MessageData) { 9 | this.jobId = `${jobId++}` 10 | if (jobInitialValue) { 11 | Object.assign(this, jobInitialValue) 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bakeryjs-example", 3 | "version": "1.0.0", 4 | "license": "MIT", 5 | "private": true, 6 | "description": "Example project for BakeryJS", 7 | "main": "index.js", 8 | "scripts": { 9 | "start": "node .", 10 | "test": "echo \"Error: no test specified\" && exit 1" 11 | }, 12 | "dependencies": { 13 | "bakeryjs": "file:.." 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test-data/processors/punctcount.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['punct'], 6 | requires: ['msg'], 7 | emits: [], 8 | aggregates: false, 9 | }, 10 | function(serviceProvider: ServiceProvider, value: MessageData) { 11 | return {punct: (value.msg as string).split(/\w+/).length}; 12 | } 13 | ); 14 | -------------------------------------------------------------------------------- /test-data/processors/wordcount.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['words'], 6 | requires: ['msg'], 7 | emits: [], 8 | aggregates: false, 9 | }, 10 | function(serviceProvider: ServiceProvider, value: MessageData) { 11 | return {words: (value.msg as string).split(/\W+/).length}; 12 | } 13 | ); 14 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/ComponentFactoryI.ts: -------------------------------------------------------------------------------- 1 | import type { BatchingBoxInterface, BoxInterface } from './BoxI' 2 | import type { PriorityQueueI } from './queue/PriorityQueueI' 3 | import type { Message } from './Message' 4 | 5 | export default interface ComponentFactoryI { 6 | create( 7 | name: string, 8 | queue?: PriorityQueueI, 9 | parameters?: any 10 | ): Promise 11 | } 12 | -------------------------------------------------------------------------------- /example/components/processors/punctcount.js: -------------------------------------------------------------------------------- 1 | const {boxFactory} = require('bakeryjs'); 2 | 3 | // Processor component which counts punctuations (non-word characters) in a message 4 | module.exports = boxFactory( 5 | { 6 | provides: ['punct'], 7 | requires: ['msg'], 8 | emits: [], 9 | aggregates: false, 10 | }, 11 | function(serviceProvider, value) { 12 | return {punct: value.msg.split(/\w+/).length}; 13 | } 14 | ); 15 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/boxCore/index.ts: -------------------------------------------------------------------------------- 1 | // Re-export all box-related types and functions 2 | export { Box } from './Box' 3 | export { BatchingBox } from './BatchingBox' 4 | export { boxFactory } from './boxFactory' 5 | export { validateAndAddParameters } from './validation' 6 | export { 7 | noopQueue, 8 | BoxExecutiveDefinition, 9 | BoxExecutiveBatchDefinition, 10 | BoxFactorySignature, 11 | BatchingBoxFactorySignature 12 | } from './types' 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | indent_style = space 9 | indent_size = 2 10 | 11 | [*.{ts,js}] 12 | indent_style = tab 13 | indent_size = 4 14 | 15 | [*.md] 16 | indent_size = 4 17 | indent_style = space 18 | trim_trailing_whitespace = false 19 | 20 | [package.json] 21 | insert_final_newline = false 22 | trim_trailing_whitespace = false 23 | -------------------------------------------------------------------------------- /benchmarks/utils/git.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Git utilities for benchmark runner 3 | */ 4 | 5 | import { execSync } from 'child_process' 6 | 7 | /** 8 | * Get the current git commit SHA 9 | * @returns The git commit SHA or undefined if not in a git repository 10 | */ 11 | export function getGitCommitSha(): string | undefined { 12 | try { 13 | return execSync('git rev-parse HEAD', { encoding: 'utf8' }).trim() 14 | } catch { 15 | return undefined 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/processingStrategies/index.ts: -------------------------------------------------------------------------------- 1 | export { ProcessingMode, getProcessingMode } from './ProcessingMode' 2 | export type { ProcessingStrategy, BoxProcessingContext } from './ProcessingStrategy' 3 | export { MapperStrategy } from './MapperStrategy' 4 | export { GeneratorStrategy } from './GeneratorStrategy' 5 | export { AggregatorStrategy } from './AggregatorStrategy' 6 | export { processingStrategyRegistry, getProcessingStrategy } from './processingStrategyRegistry' 7 | -------------------------------------------------------------------------------- /test-data/generators/bad.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['msg'], 6 | requires: [], 7 | emits: ['msg_bad'], 8 | aggregates: false, 9 | }, 10 | async function( 11 | serviceProvider: ServiceProvider, 12 | value: MessageData, 13 | emit: (chunk: MessageData[], priority?: number) => void 14 | ) { 15 | emit([{msg: 'Hello World!'}]); 16 | return; 17 | } 18 | ); 19 | -------------------------------------------------------------------------------- /example/components/processors/checksum.js: -------------------------------------------------------------------------------- 1 | const {boxFactory} = require('bakeryjs'); 2 | 3 | // Processor component which calculates "checksum" for a given message from number of words and punctuations 4 | module.exports = boxFactory( 5 | { 6 | provides: ['checksum'], 7 | requires: ['words', 'punct'], 8 | emits: [], 9 | aggregates: false, 10 | }, 11 | function(serviceProvider, value) { 12 | return {checksum: Math.sqrt(2) * value.words + value.punct}; 13 | } 14 | ); 15 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper1.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 1 - Passthrough mapper for benchmarking 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m1_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m1_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper2.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 2 - Passthrough mapper for benchmarking 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m2_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m2_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper3.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 3 - Passthrough mapper for benchmarking 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m3_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m3_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /test-data/generators/helloworld.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['msg'], 6 | requires: [], 7 | emits: ['msg_helloworld'], 8 | aggregates: false, 9 | }, 10 | async function( 11 | serviceProvider: ServiceProvider, 12 | value: MessageData, 13 | emit: (chunk: MessageData[], priority?: number) => void 14 | ) { 15 | emit([{msg: 'Hello World!'}]); 16 | return; 17 | } 18 | ); 19 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper9.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 9 - Final passthrough mapper for benchmarking 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m9_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m9_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /jest.setup.js: -------------------------------------------------------------------------------- 1 | // Suppress console output during tests to reduce noise 2 | // Uses jest.spyOn for proper mock management 3 | 4 | beforeAll(() => { 5 | // Mock console methods to suppress output during tests 6 | jest.spyOn(console, 'log').mockImplementation(() => {}); 7 | jest.spyOn(console, 'error').mockImplementation(() => {}); 8 | jest.spyOn(console, 'warn').mockImplementation(() => {}); 9 | }); 10 | 11 | afterAll(() => { 12 | // Restore all mocks 13 | jest.restoreAllMocks(); 14 | }); 15 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper6.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 6 - Passthrough mapper for parallel stage benchmarking 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m6_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m6_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper7.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 7 - Passthrough mapper for parallel stage benchmarking 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m7_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m7_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper8.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 8 - Passthrough mapper for parallel stage benchmarking 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m8_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m8_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper4.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 4 - Passthrough mapper for benchmarking (in nested dimension) 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m4_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m4_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /benchmarks/components/processors/mapper5.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Mapper 5 - Passthrough mapper for benchmarking (in nested dimension) 3 | */ 4 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 5 | 6 | module.exports = boxFactory( 7 | { 8 | provides: ['m5_processed'], 9 | requires: [], 10 | emits: [], 11 | aggregates: false 12 | }, 13 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 14 | return { m5_processed: true } 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /tests/regressions.test.ts: -------------------------------------------------------------------------------- 1 | import { Program } from 'bakeryjs' 2 | 3 | test('tracingModel: One dimension completes before the other starts. TypeError: Requested key msg is missing', async () => { 4 | const program = new Program({}, { componentPaths: [`${__dirname}/../test-data/`] }) 5 | 6 | const job = { 7 | process: [ 8 | [ 9 | { 10 | hellobatchworld: [['wordbatchcountslow']], 11 | helloworld: [['wordcount']] 12 | } 13 | ] 14 | ] 15 | } 16 | 17 | await program.run(job) 18 | }) 19 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/eval/every.ts: -------------------------------------------------------------------------------- 1 | export default function every(arr: T[], evalFunction: (val: T) => boolean): boolean { 2 | for (let i = 0; i < arr.length; i++) { 3 | const item = arr[i] 4 | if (item !== undefined && !evalFunction(item)) { 5 | return false 6 | } 7 | } 8 | 9 | return true 10 | } 11 | 12 | export function everyMap(map: Map, evalFunction: (val: T) => boolean): boolean { 13 | for (const i of map.values()) { 14 | if (!evalFunction(i)) { 15 | return false 16 | } 17 | } 18 | 19 | return true 20 | } 21 | -------------------------------------------------------------------------------- /test-data/processors/wordbatchcount.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['words'], 6 | requires: ['msg'], 7 | aggregates: false, 8 | batch: { 9 | maxSize: 3, 10 | timeoutSeconds: 0.3, 11 | }, 12 | }, 13 | async function(serviceProvider: ServiceProvider, values: MessageData[]) { 14 | return values.map((value) => 15 | Object.create(null, { 16 | words: {value: (value.msg as string).split(/\W+/).length}, 17 | }) 18 | ); 19 | } 20 | ); 21 | -------------------------------------------------------------------------------- /example/components/processors/wordcount.js: -------------------------------------------------------------------------------- 1 | const {boxFactory} = require('bakeryjs'); 2 | 3 | // Processor component which calculates number of words in a message 4 | module.exports = boxFactory( 5 | { 6 | provides: ['words'], 7 | requires: ['msg'], 8 | emits: [], 9 | aggregates: false, 10 | }, 11 | // serviceParamsProvider may contain some shared modules, like logger 12 | // second parameter is an object with properties corresponding to 'requires' 13 | function(serviceProvider, {msg}) { 14 | return {words: msg.split(/\W+/).length}; 15 | } 16 | ); 17 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/componentNameParser.ts: -------------------------------------------------------------------------------- 1 | const REMOVABLE_SUBSTRINGS = ['_/', 'boxes/', 'generators/', 'processors/', '.coffee', '.ts', '.js'] 2 | 3 | const MODULE_REGEXP = new RegExp('(?:/|^)[^./][^/]+.(?:coffee|js|ts)$') 4 | 5 | const parseComponentName = (path: string): string | null => { 6 | if (!MODULE_REGEXP.test(path)) { 7 | return null 8 | } 9 | 10 | let name = path 11 | for (const removableSubstring of REMOVABLE_SUBSTRINGS) { 12 | name = name.replace(removableSubstring, '') 13 | } 14 | 15 | return name 16 | } 17 | 18 | export { parseComponentName } 19 | -------------------------------------------------------------------------------- /tests/components/_/processors/print.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData, Logger } from 'bakeryjs' 2 | 3 | const Print = boxFactory( 4 | { 5 | requires: ['jobId', 'raw'], 6 | provides: [], 7 | emits: [], 8 | aggregates: false 9 | }, 10 | function processValue( 11 | services: ServiceProvider, 12 | input: MessageData, 13 | _neverEmit: (chunk: MessageData[], priority?: number) => void 14 | ): MessageData { 15 | services.get('logger').log({ printBox: JSON.stringify(input) }) 16 | return {} 17 | } 18 | ) 19 | export default Print 20 | -------------------------------------------------------------------------------- /tests/integration/components/processors/identity-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A pass-through processor that doesn't transform data. 5 | * Useful for testing flow topology without modifying messages. 6 | */ 7 | const IdentityProcessor = boxFactory( 8 | { 9 | provides: [], 10 | requires: [], 11 | emits: [], 12 | aggregates: false 13 | }, 14 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 15 | return {} 16 | } 17 | ) 18 | 19 | export default IdentityProcessor 20 | -------------------------------------------------------------------------------- /benchmarks/results/partial-complex-1000x1000-1765698148770.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "incomplete", 3 | "reason": "timeout", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "elapsedMs": 120001, 10 | "sentEventCount": 686478, 11 | "messagesProcessed": 75714, 12 | "expectedMessages": 1000000, 13 | "avgTimePerMessageMs": 1.5849248487730143, 14 | "messagesPerSecond": 630.944742127149, 15 | "memoryMB": 1727.5621032714844, 16 | "peakMemoryMB": 2378.3475189208984, 17 | "timestamp": "2025-12-14T07:42:28.769Z", 18 | "environment": {} 19 | } -------------------------------------------------------------------------------- /src/components/_/processors/print.ts: -------------------------------------------------------------------------------- 1 | import type { Logger } from '../../../lib/bakeryjs/ServiceProvider' 2 | import { boxFactory, ServiceProvider, MessageData } from '../../../' 3 | 4 | const Print = boxFactory( 5 | { 6 | requires: ['jobId', 'raw'], 7 | provides: [], 8 | emits: [], 9 | aggregates: false 10 | }, 11 | function processValue( 12 | services: ServiceProvider, 13 | input: MessageData, 14 | _neverEmit: (chunk: MessageData[], priority?: number) => void 15 | ): MessageData { 16 | services.get('logger').log({ printBox: JSON.stringify(input) }) 17 | return {} 18 | } 19 | ) 20 | export default Print 21 | -------------------------------------------------------------------------------- /tests/integration/components/processors/field-a-provider.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A processor that provides field 'fieldA' with value 'valueA'. 5 | * Useful for testing diamond pattern and field accumulation in parallel branches. 6 | */ 7 | const FieldAProvider = boxFactory( 8 | { 9 | provides: ['fieldA'], 10 | requires: [], 11 | emits: [], 12 | aggregates: false 13 | }, 14 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 15 | return { fieldA: 'valueA' } 16 | } 17 | ) 18 | 19 | export default FieldAProvider 20 | -------------------------------------------------------------------------------- /tests/integration/components/processors/field-b-provider.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A processor that provides field 'fieldB' with value 'valueB'. 5 | * Useful for testing diamond pattern and field accumulation in parallel branches. 6 | */ 7 | const FieldBProvider = boxFactory( 8 | { 9 | provides: ['fieldB'], 10 | requires: [], 11 | emits: [], 12 | aggregates: false 13 | }, 14 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 15 | return { fieldB: 'valueB' } 16 | } 17 | ) 18 | 19 | export default FieldBProvider 20 | -------------------------------------------------------------------------------- /test-data/processors/wordbatchcountslow.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['words'], 6 | requires: ['msg'], 7 | aggregates: false, 8 | batch: { 9 | maxSize: 3, 10 | timeoutSeconds: 0.1, 11 | }, 12 | }, 13 | async function(serviceProvider: ServiceProvider, values: MessageData[]) { 14 | const vals = values.map((value) => 15 | Object.create(null, { 16 | words: {value: (value.msg as string).split(/\W+/).length}, 17 | }) 18 | ); 19 | await new Promise((resolve) => setTimeout(resolve, 1200)); 20 | return vals; 21 | } 22 | ); 23 | -------------------------------------------------------------------------------- /tests/integration/components/processors/transform-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A processor that adds a 'transformed' field with the current timestamp. 5 | * Useful for testing that messages flow through processors and are modified. 6 | */ 7 | const TransformProcessor = boxFactory( 8 | { 9 | provides: ['transformed'], 10 | requires: [], 11 | emits: [], 12 | aggregates: false 13 | }, 14 | function processValue(_serviceProvider: ServiceProvider, _value: MessageData): MessageData { 15 | return { transformed: Date.now() } 16 | } 17 | ) 18 | 19 | export default TransformProcessor 20 | -------------------------------------------------------------------------------- /benchmarks/results/partial-complex-1000x1000-1765697565328.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "incomplete", 3 | "reason": "timeout", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "elapsedMs": 119999, 10 | "sentEventCount": 665787, 11 | "messagesProcessed": 73415, 12 | "expectedMessages": 1000000, 13 | "avgTimePerMessageMs": 1.6345297282571682, 14 | "messagesPerSecond": 611.7967649730415, 15 | "memoryMB": 1730.902214050293, 16 | "peakMemoryMB": 2356.5486755371094, 17 | "timestamp": "2025-12-14T07:32:45.326Z", 18 | "environment": { 19 | "BAKERYJS_DISABLE_EXPERIMENTAL_TRACING": "1" 20 | } 21 | } -------------------------------------------------------------------------------- /benchmarks/results/partial-complex-1000x1000-1765697777132.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "incomplete", 3 | "reason": "timeout", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "elapsedMs": 120000, 10 | "sentEventCount": 661236, 11 | "messagesProcessed": 72910, 12 | "expectedMessages": 1000000, 13 | "avgTimePerMessageMs": 1.645864764778494, 14 | "messagesPerSecond": 607.5833333333334, 15 | "memoryMB": 1732.3982467651367, 16 | "peakMemoryMB": 2352.8290252685547, 17 | "timestamp": "2025-12-14T07:36:17.129Z", 18 | "environment": { 19 | "BAKERYJS_DISABLE_EXPERIMENTAL_TRACING": "1" 20 | } 21 | } -------------------------------------------------------------------------------- /benchmarks/results/partial-complex-1000x1000-1765698020098.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "incomplete", 3 | "reason": "timeout", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "elapsedMs": 120001, 10 | "sentEventCount": 689871, 11 | "messagesProcessed": 76091, 12 | "expectedMessages": 1000000, 13 | "avgTimePerMessageMs": 1.5770721898779094, 14 | "messagesPerSecond": 634.0863826134782, 15 | "memoryMB": 1720.537353515625, 16 | "peakMemoryMB": 2373.912811279297, 17 | "timestamp": "2025-12-14T07:40:20.094Z", 18 | "environment": { 19 | "BAKERYJS_DISABLE_EXPERIMENTAL_TRACING": "1" 20 | } 21 | } -------------------------------------------------------------------------------- /tests/integration/helpers/test-program.ts: -------------------------------------------------------------------------------- 1 | import { Program } from 'bakeryjs' 2 | 3 | export interface TestProgramOptions { 4 | componentPaths?: string[] 5 | services?: Record 6 | } 7 | 8 | /** 9 | * Creates a Program instance configured for integration testing. 10 | * By default, includes both the integration test components and the test-data components. 11 | */ 12 | export function createTestProgram(options: TestProgramOptions = {}): Program { 13 | return new Program(options.services ?? {}, { 14 | componentPaths: options.componentPaths ?? [ 15 | `${__dirname}/../components/`, 16 | `${__dirname}/../../../test-data/` 17 | ] 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /test-data/processors/checksum.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['checksum'], 6 | requires: ['words', 'punct'], 7 | emits: [], 8 | aggregates: false, 9 | parameters: { 10 | title: 'Parameter of the box -- positive number', 11 | type: 'number', 12 | minimum: 0, 13 | }, 14 | }, 15 | function(serviceProvider: ServiceProvider, value: MessageData) { 16 | const param = (serviceProvider.parameters as number | undefined) ?? 2; 17 | return { 18 | checksum: 19 | Math.sqrt(param) * (value.words as number) + 20 | (value.punct as number), 21 | }; 22 | } 23 | ); 24 | -------------------------------------------------------------------------------- /benchmarks/results/partial-complex-5000x200-1765709619474.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "incomplete", 3 | "reason": "timeout", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 5000, 7 | "nestedItemCount": 200 8 | }, 9 | "elapsedMs": 125018, 10 | "sentEventCount": 773471, 11 | "messagesProcessed": 83143, 12 | "expectedMessages": 1000000, 13 | "avgTimePerMessageMs": 1.5036503373705543, 14 | "messagesPerSecond": 665.0482330544402, 15 | "memoryMB": 1837.470848083496, 16 | "peakMemoryMB": 2238.449203491211, 17 | "timestamp": "2025-12-14T10:53:39.436Z", 18 | "nodeVersion": "v24.12.0", 19 | "gitCommitSha": "24a8fa4bdfd60d8daf6ecae461f90c89ddf52015", 20 | "environment": {} 21 | } -------------------------------------------------------------------------------- /benchmarks/results/partial-complex-10000x100-1765713472193.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "incomplete", 3 | "reason": "timeout", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 10000, 7 | "nestedItemCount": 100 8 | }, 9 | "elapsedMs": 120002, 10 | "sentEventCount": 798306, 11 | "messagesProcessed": 83109, 12 | "expectedMessages": 1000000, 13 | "avgTimePerMessageMs": 1.4439110084347062, 14 | "messagesPerSecond": 692.5634572757122, 15 | "memoryMB": 1764.871940612793, 16 | "peakMemoryMB": 2388.391990661621, 17 | "timestamp": "2025-12-14T11:57:52.148Z", 18 | "nodeVersion": "v24.12.0", 19 | "gitCommitSha": "613cdc18edfe6bbddf3f9bb77b745467478898f6", 20 | "environment": {} 21 | } -------------------------------------------------------------------------------- /src/lib/bakeryjs/Box.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Box module - re-exports from the box/ directory for backward compatibility. 3 | * 4 | * The Box implementation has been split into smaller, focused modules: 5 | * - box/Box.ts - Core Box abstraction 6 | * - box/BatchingBox.ts - Batching variant 7 | * - box/boxFactory.ts - Factory functions 8 | * - box/types.ts - Type definitions 9 | * - box/validation.ts - Parameter validation 10 | * 11 | * @module Box 12 | */ 13 | 14 | // Re-export everything from the box directory for backward compatibility 15 | export { 16 | boxFactory, 17 | noopQueue, 18 | BoxExecutiveDefinition, 19 | BoxExecutiveBatchDefinition, 20 | BoxFactorySignature, 21 | BatchingBoxFactorySignature 22 | } from './boxCore' 23 | -------------------------------------------------------------------------------- /tests/integration/components/generators/empty-generator.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A generator that emits no messages (empty array). 5 | * Useful for testing edge cases where a generator produces no output. 6 | */ 7 | const EmptyGenerator = boxFactory( 8 | { 9 | provides: ['value'], 10 | requires: [], 11 | emits: ['empty_dim'], 12 | aggregates: false 13 | }, 14 | async function processValue( 15 | _serviceProvider: ServiceProvider, 16 | _value: MessageData, 17 | emit: (chunk: MessageData[], priority?: number) => void 18 | ): Promise { 19 | // Emit an empty array - no messages 20 | emit([]) 21 | } 22 | ) 23 | 24 | export default EmptyGenerator 25 | -------------------------------------------------------------------------------- /test-data/generators/hellobatchworld.ts: -------------------------------------------------------------------------------- 1 | import {boxFactory, ServiceProvider, MessageData} from 'bakeryjs'; 2 | 3 | module.exports = boxFactory( 4 | { 5 | provides: ['msg'], 6 | requires: [], 7 | emits: ['msg_hellobatchworld'], 8 | aggregates: false, 9 | }, 10 | async function( 11 | serviceProvider: ServiceProvider, 12 | value: MessageData, 13 | emit: (chunk: MessageData[], priority?: number) => void 14 | ) { 15 | return new Promise((resolve) => { 16 | emit([{msg: 'Hello World!'}, {msg: 'Yellow World!'}]); 17 | setTimeout( 18 | () => emit([{msg: 'Hallo Welt!'}, {msg: 'Salut Monde!'}]), 19 | 150 20 | ); 21 | setTimeout(() => emit([{msg: 'Ola Mundo!'}]), 200); 22 | setTimeout(resolve, 230); 23 | }); 24 | } 25 | ); 26 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/FlowSchemaReader.ts: -------------------------------------------------------------------------------- 1 | import type { FlowExplicitDescription } from './FlowBuilderI' 2 | import type FlowSchemaReaderI from './FlowSchemaReaderI' 3 | 4 | export default class FlowSchemaReader implements FlowSchemaReaderI { 5 | private readonly flowList: { [key: string]: FlowExplicitDescription } 6 | 7 | public constructor(flowsPath: string) { 8 | // Dynamic require is intentional here - path is determined at runtime 9 | 10 | this.flowList = require(flowsPath).default 11 | } 12 | 13 | // TODO: (idea2) Lazy read from file upon request? 14 | public getFlowSchema(name: string): FlowExplicitDescription { 15 | if (!this.flowList[name]) { 16 | throw new Error(`Flow not found: ${name}`) 17 | } 18 | 19 | return this.flowList[name] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/components/_/generators/tick.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from '../../../' 2 | 3 | const Tick = boxFactory( 4 | { 5 | requires: ['jobId'], 6 | provides: ['raw'], 7 | emits: ['tick'], 8 | aggregates: false 9 | }, 10 | function processValue( 11 | serviceProvider: ServiceProvider, 12 | value: MessageData, 13 | emitCallback: (chunk: MessageData[], priority?: number) => void 14 | ): Promise { 15 | let i = 0 16 | return new Promise((resolve: (result?: any) => void): void => { 17 | const id = setInterval((): void => { 18 | if (i >= 3) { 19 | clearInterval(id) 20 | resolve() 21 | } 22 | i += 1 23 | emitCallback([{ raw: i }], 1) 24 | }, 1000) 25 | }) 26 | } 27 | ) 28 | 29 | export default Tick 30 | -------------------------------------------------------------------------------- /src/components/_/generators/tock.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from '../../..' 2 | 3 | const Tock = boxFactory( 4 | { 5 | requires: ['jobId'], 6 | provides: ['raw'], 7 | emits: ['tock'], 8 | aggregates: false 9 | }, 10 | function processValue( 11 | serviceProvider: ServiceProvider, 12 | value: MessageData, 13 | emitCallback: (chunk: MessageData[], priority?: number) => void 14 | ): Promise { 15 | let i = 0 16 | return new Promise((resolve: (result?: any) => void): void => { 17 | const id = setInterval((): void => { 18 | if (i >= 3) { 19 | clearInterval(id) 20 | resolve() 21 | } 22 | i += 1 23 | emitCallback([{ raw: i }], 1) 24 | }, 1000) 25 | }) 26 | } 27 | ) 28 | 29 | export default Tock 30 | -------------------------------------------------------------------------------- /tests/components/_/generators/tick.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | const Tick = boxFactory( 4 | { 5 | requires: ['jobId'], 6 | provides: ['raw'], 7 | emits: ['tick'], 8 | aggregates: false 9 | }, 10 | function processValue( 11 | serviceProvider: ServiceProvider, 12 | value: MessageData, 13 | emitCallback: (chunk: MessageData[], priority?: number) => void 14 | ): Promise { 15 | let i = 0 16 | return new Promise((resolve: (result?: any) => void): void => { 17 | const id = setInterval((): void => { 18 | if (i >= 3) { 19 | clearInterval(id) 20 | resolve() 21 | } 22 | i += 1 23 | emitCallback([{ raw: i }], 1) 24 | }, 1000) 25 | }) 26 | } 27 | ) 28 | 29 | export default Tick 30 | -------------------------------------------------------------------------------- /tests/components/_/generators/tock.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | const Tock = boxFactory( 4 | { 5 | requires: ['jobId'], 6 | provides: ['raw'], 7 | emits: ['tock'], 8 | aggregates: false 9 | }, 10 | function processValue( 11 | serviceProvider: ServiceProvider, 12 | value: MessageData, 13 | emitCallback: (chunk: MessageData[], priority?: number) => void 14 | ): Promise { 15 | let i = 0 16 | return new Promise((resolve: (result?: any) => void): void => { 17 | const id = setInterval((): void => { 18 | if (i >= 3) { 19 | clearInterval(id) 20 | resolve() 21 | } 22 | i += 1 23 | emitCallback([{ raw: i }], 1) 24 | }, 1000) 25 | }) 26 | } 27 | ) 28 | 29 | export default Tock 30 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/processingStrategies/AggregatorStrategy.ts: -------------------------------------------------------------------------------- 1 | import type { Message } from '../Message' 2 | import type { ProcessingStrategy, BoxProcessingContext } from './ProcessingStrategy' 3 | import VError from 'verror' 4 | 5 | /** 6 | * Strategy for processing messages in aggregator mode. 7 | * Aggregates multiple messages into a single output (not yet implemented). 8 | */ 9 | export class AggregatorStrategy implements ProcessingStrategy { 10 | public async execute(_msg: Message, context: BoxProcessingContext): Promise { 11 | throw new VError( 12 | { 13 | name: 'NotImplementedError', 14 | message: "Box '%s': Aggregator has not been implemented yet.", 15 | info: { 16 | name: context.name, 17 | meta: context.meta 18 | } 19 | }, 20 | context.name 21 | ) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-06-35-386Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "simple-10", 4 | "flowType": "simple", 5 | "config": { 6 | "itemCount": 10 7 | }, 8 | "metrics": { 9 | "totalTimeMs": 18, 10 | "messagesProcessed": 10, 11 | "avgTimePerMessageMs": 1.8, 12 | "memoryUsedMB": 4.009605407714844, 13 | "peakMemoryMB": 52.833641052246094, 14 | "heapStats": { 15 | "heapUsed": 56160496, 16 | "heapTotal": 80609280, 17 | "external": 2680299 18 | } 19 | }, 20 | "eventTimings": { 21 | "firstSentMs": null, 22 | "lastSentMs": null, 23 | "drainCompleteMs": 18, 24 | "sentEventCount": 12 25 | }, 26 | "timestamp": "2025-12-14T07:06:35.385Z", 27 | "nodeVersion": "v22.18.0", 28 | "environment": {} 29 | } 30 | ] -------------------------------------------------------------------------------- /tests/integration/components/processors/logger-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData, Logger } from 'bakeryjs' 2 | 3 | /** 4 | * A processor that uses the logger service. 5 | * Useful for testing custom logger injection. 6 | * 7 | * This processor logs the incoming message using the logger service 8 | * and provides a 'logged' field indicating logging was performed. 9 | */ 10 | const LoggerProcessor = boxFactory( 11 | { 12 | provides: ['logged'], 13 | requires: [], 14 | emits: [], 15 | aggregates: false 16 | }, 17 | function processValue(serviceProvider: ServiceProvider, _value: MessageData): MessageData { 18 | const logger = serviceProvider.get('logger') 19 | logger.log({ loggerProcessor: _value }) 20 | return { logged: true } 21 | } 22 | ) 23 | 24 | export default LoggerProcessor 25 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-12-00-411Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "simple-10000", 4 | "flowType": "simple", 5 | "config": { 6 | "itemCount": 10000 7 | }, 8 | "metrics": { 9 | "totalTimeMs": 625, 10 | "messagesProcessed": 10000, 11 | "avgTimePerMessageMs": 0.0625, 12 | "memoryUsedMB": 33.82587432861328, 13 | "peakMemoryMB": 93.84246063232422, 14 | "heapStats": { 15 | "heapUsed": 90053504, 16 | "heapTotal": 125075456, 17 | "external": 2656971 18 | } 19 | }, 20 | "eventTimings": { 21 | "firstSentMs": 28, 22 | "lastSentMs": 624, 23 | "drainCompleteMs": 625, 24 | "sentEventCount": 10101 25 | }, 26 | "timestamp": "2025-12-14T07:12:00.410Z", 27 | "nodeVersion": "v22.18.0", 28 | "environment": {} 29 | } 30 | ] -------------------------------------------------------------------------------- /tests/integration/components/processors/merge-fields.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A processor that requires both 'fieldA' and 'fieldB' and merges them. 5 | * Useful for testing diamond pattern where multiple parallel branches converge. 6 | * 7 | * Requires: 8 | * - fieldA: Value from branch A 9 | * - fieldB: Value from branch B 10 | * 11 | * Provides: 12 | * - mergedField: Combined value of fieldA and fieldB 13 | */ 14 | const MergeFields = boxFactory( 15 | { 16 | provides: ['mergedField'], 17 | requires: ['fieldA', 'fieldB'], 18 | emits: [], 19 | aggregates: false 20 | }, 21 | function processValue(_serviceProvider: ServiceProvider, value: MessageData): MessageData { 22 | return { mergedField: `${value.fieldA}+${value.fieldB}` } 23 | } 24 | ) 25 | 26 | export default MergeFields 27 | -------------------------------------------------------------------------------- /tests/scanComponentsPath.int.test.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'path' 2 | import { scanComponentsPath } from 'bakeryjs/scanComponentsPath' 3 | 4 | const componentsPath = join(__dirname, 'components') 5 | 6 | describe('.scanComponentsPath', () => { 7 | it('finds components in a directory', () => { 8 | const result = scanComponentsPath(componentsPath) 9 | expect(result).toEqual( 10 | expect.objectContaining({ 11 | tick: join(componentsPath, '_/generators/tick.ts'), 12 | tock: join(componentsPath, '_/generators/tock.ts'), 13 | print: join(componentsPath, '_/processors/print.ts') 14 | }) 15 | ) 16 | }) 17 | it('ignores invalid files', () => { 18 | const result = scanComponentsPath(componentsPath) 19 | expect(Object.keys(result)).toEqual( 20 | expect.not.arrayContaining([expect.stringContaining('not-a-component')]) 21 | ) 22 | }) 23 | }) 24 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | module.exports = { 3 | preset: 'ts-jest', 4 | testEnvironment: 'node', 5 | collectCoverageFrom: ['src/**/*.{js,ts}', '!src/**/*.d.ts'], 6 | coverageDirectory: 'coverage', 7 | coverageReporters: ['text', 'text-summary', 'lcov', 'html', 'json'], 8 | coverageThreshold: { 9 | global: { 10 | branches: 50, 11 | functions: 50, 12 | lines: 50, 13 | statements: 50, 14 | }, 15 | }, 16 | moduleNameMapper: { 17 | '^bakeryjs$': '/src/index.ts', 18 | '^bakeryjs/(.*)$': '/src/lib/bakeryjs/$1', 19 | }, 20 | testPathIgnorePatterns: ['/(build|docs|node_modules)/'], 21 | setupFilesAfterEnv: ['/jest.setup.js'], 22 | transform: { 23 | '^.+\\.tsx?$': [ 24 | 'ts-jest', 25 | { 26 | tsconfig: 'tsconfig.json', 27 | }, 28 | ], 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-12-02-897Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-100x100", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 100, 7 | "nestedItemCount": 100 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 905, 11 | "messagesProcessed": 10000, 12 | "avgTimePerMessageMs": 0.0905, 13 | "memoryUsedMB": 68.21662139892578, 14 | "peakMemoryMB": 126.98080444335938, 15 | "heapStats": { 16 | "heapUsed": 123802480, 17 | "heapTotal": 158334976, 18 | "external": 2658763 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 22, 23 | "lastSentMs": 903, 24 | "drainCompleteMs": 905, 25 | "sentEventCount": 90504 26 | }, 27 | "timestamp": "2025-12-14T07:12:02.896Z", 28 | "nodeVersion": "v22.18.0", 29 | "environment": {} 30 | } 31 | ] -------------------------------------------------------------------------------- /src/lib/bakeryjs/FlowFactory.ts: -------------------------------------------------------------------------------- 1 | import type ComponentFactoryI from './ComponentFactoryI' 2 | import type FlowBuilderI from './FlowBuilderI' 3 | import type { FlowExplicitDescription } from './FlowBuilderI' 4 | import type { Flow } from './Flow' 5 | import type { PriorityQueueI } from './queue/PriorityQueueI' 6 | import type { Message } from './Message' 7 | 8 | export default class FlowFactory { 9 | private readonly componentFactory: ComponentFactoryI 10 | private readonly builder: FlowBuilderI 11 | 12 | public constructor(componentFactory: ComponentFactoryI, builder: FlowBuilderI) { 13 | this.componentFactory = componentFactory 14 | this.builder = builder 15 | } 16 | 17 | public async create( 18 | schema: FlowExplicitDescription, 19 | drain?: PriorityQueueI 20 | ): Promise { 21 | return this.builder.build(schema, this.componentFactory, drain) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-44-39-182Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "simple-10", 4 | "flowType": "simple", 5 | "config": { 6 | "itemCount": 10 7 | }, 8 | "metrics": { 9 | "totalTimeMs": 20, 10 | "messagesProcessed": 10, 11 | "avgTimePerMessageMs": 2, 12 | "memoryUsedMB": -4.232444763183594, 13 | "peakMemoryMB": 49.825584411621094, 14 | "heapStats": { 15 | "heapUsed": 52985416, 16 | "heapTotal": 83853312, 17 | "external": 2655723 18 | } 19 | }, 20 | "eventTimings": { 21 | "firstSentMs": 14, 22 | "lastSentMs": 18, 23 | "drainCompleteMs": 20, 24 | "sentEventCount": 12 25 | }, 26 | "timestamp": "2025-12-14T07:44:39.149Z", 27 | "nodeVersion": "v22.18.0", 28 | "gitCommitSha": "6bf863f750e0f3e341a09ec3e831f01f672a00c4", 29 | "environment": {} 30 | } 31 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-55-16-640Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "simple-10", 4 | "flowType": "simple", 5 | "config": { 6 | "itemCount": 10 7 | }, 8 | "metrics": { 9 | "totalTimeMs": 18, 10 | "messagesProcessed": 10, 11 | "avgTimePerMessageMs": 1.8, 12 | "memoryUsedMB": 3.9801025390625, 13 | "peakMemoryMB": 52.13922882080078, 14 | "heapStats": { 15 | "heapUsed": 55434032, 16 | "heapTotal": 83329024, 17 | "external": 2672107 18 | } 19 | }, 20 | "eventTimings": { 21 | "firstSentMs": 11, 22 | "lastSentMs": 17, 23 | "drainCompleteMs": 18, 24 | "sentEventCount": 12 25 | }, 26 | "timestamp": "2025-12-14T07:55:16.600Z", 27 | "nodeVersion": "v22.18.0", 28 | "gitCommitSha": "6bf863f750e0f3e341a09ec3e831f01f672a00c4", 29 | "environment": {} 30 | } 31 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-57-28-862Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "simple-5", 4 | "flowType": "simple", 5 | "config": { 6 | "itemCount": 5 7 | }, 8 | "metrics": { 9 | "totalTimeMs": 19, 10 | "messagesProcessed": 5, 11 | "avgTimePerMessageMs": 3.8, 12 | "memoryUsedMB": 3.7282638549804688, 13 | "peakMemoryMB": 52.043724060058594, 14 | "heapStats": { 15 | "heapUsed": 55072288, 16 | "heapTotal": 83591168, 17 | "external": 2671787 18 | } 19 | }, 20 | "eventTimings": { 21 | "firstSentMs": 12, 22 | "lastSentMs": 17, 23 | "drainCompleteMs": 19, 24 | "sentEventCount": 7 25 | }, 26 | "timestamp": "2025-12-14T07:57:28.827Z", 27 | "nodeVersion": "v22.18.0", 28 | "gitCommitSha": "6bf863f750e0f3e341a09ec3e831f01f672a00c4", 29 | "environment": {} 30 | } 31 | ] -------------------------------------------------------------------------------- /tests/integration/components/processors/field-reader.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A processor that requires 'providedField' and outputs its value. 5 | * Useful for testing field requirement and data flow. 6 | * 7 | * Requires: 8 | * - providedField: The field to read 9 | * 10 | * Provides: 11 | * - readValue: The value of providedField that was read 12 | * - readTimestamp: When the field was read 13 | */ 14 | const FieldReader = boxFactory( 15 | { 16 | provides: ['readValue', 'readTimestamp'], 17 | requires: ['providedField'], 18 | emits: [], 19 | aggregates: false 20 | }, 21 | function processValue(_serviceProvider: ServiceProvider, value: MessageData): MessageData { 22 | return { 23 | readValue: value.providedField, 24 | readTimestamp: Date.now() 25 | } 26 | } 27 | ) 28 | 29 | export default FieldReader 30 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-12-10-193Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "simple-10000", 4 | "flowType": "simple", 5 | "config": { 6 | "itemCount": 10000 7 | }, 8 | "metrics": { 9 | "totalTimeMs": 640, 10 | "messagesProcessed": 10000, 11 | "avgTimePerMessageMs": 0.064, 12 | "memoryUsedMB": 40.39404296875, 13 | "peakMemoryMB": 93.52953338623047, 14 | "heapStats": { 15 | "heapUsed": 94204848, 16 | "heapTotal": 122732544, 17 | "external": 2665803 18 | } 19 | }, 20 | "eventTimings": { 21 | "firstSentMs": 10, 22 | "lastSentMs": 639, 23 | "drainCompleteMs": 640, 24 | "sentEventCount": 10101 25 | }, 26 | "timestamp": "2025-12-14T07:12:10.191Z", 27 | "nodeVersion": "v22.18.0", 28 | "environment": { 29 | "BAKERYJS_DISABLE_EXPERIMENTAL_TRACING": "1" 30 | } 31 | } 32 | ] -------------------------------------------------------------------------------- /src/lib/bakeryjs/processingStrategies/ProcessingMode.ts: -------------------------------------------------------------------------------- 1 | import type { BoxMeta, BatchingBoxMeta } from '../BoxI' 2 | 3 | /** 4 | * Processing modes that a Box can operate in. 5 | * Each mode determines how messages are processed. 6 | */ 7 | export enum ProcessingMode { 8 | Mapper = 'mapper', 9 | Generator = 'generator', 10 | Aggregator = 'aggregator' 11 | } 12 | 13 | /** 14 | * Determines the processing mode from box metadata. 15 | * Single source of truth for mode determination logic. 16 | * 17 | * @param meta - The box metadata 18 | * @returns The processing mode 19 | */ 20 | export function getProcessingMode(meta: BoxMeta | BatchingBoxMeta): ProcessingMode { 21 | if (meta.aggregates) { 22 | return ProcessingMode.Aggregator 23 | } 24 | // Only BoxMeta has emits property 25 | if ('emits' in meta && (meta as BoxMeta).emits.length > 0) { 26 | return ProcessingMode.Generator 27 | } 28 | return ProcessingMode.Mapper 29 | } 30 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T12-02-23-364Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-1000x100", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 100 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 1537, 11 | "messagesProcessed": 100000, 12 | "avgTimePerMessageMs": 0.01537, 13 | "memoryUsedMB": 161.6480941772461, 14 | "peakMemoryMB": 0, 15 | "heapStats": { 16 | "heapUsed": 336178984, 17 | "heapTotal": 441401344, 18 | "external": 7911794 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 136, 23 | "lastSentMs": 1536, 24 | "drainCompleteMs": 1537, 25 | "sentEventCount": 905031 26 | }, 27 | "timestamp": "2025-12-14T12:02:23.326Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "613cdc18edfe6bbddf3f9bb77b745467478898f6", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T12-02-46-444Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-5000x100", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 5000, 7 | "nestedItemCount": 100 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 7363, 11 | "messagesProcessed": 500000, 12 | "avgTimePerMessageMs": 0.014726, 13 | "memoryUsedMB": 550.3235931396484, 14 | "peakMemoryMB": 0, 15 | "heapStats": { 16 | "heapUsed": 744122336, 17 | "heapTotal": 873349120, 18 | "external": 7880494 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 135, 23 | "lastSentMs": 7363, 24 | "drainCompleteMs": 7363, 25 | "sentEventCount": 4525151 26 | }, 27 | "timestamp": "2025-12-14T12:02:46.411Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "613cdc18edfe6bbddf3f9bb77b745467478898f6", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T07-12-12-960Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-100x100", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 100, 7 | "nestedItemCount": 100 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 918, 11 | "messagesProcessed": 10000, 12 | "avgTimePerMessageMs": 0.0918, 13 | "memoryUsedMB": 66.24581909179688, 14 | "peakMemoryMB": 121.02208709716797, 15 | "heapStats": { 16 | "heapUsed": 121365264, 17 | "heapTotal": 158072832, 18 | "external": 2655339 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 24, 23 | "lastSentMs": 916, 24 | "drainCompleteMs": 918, 25 | "sentEventCount": 90504 26 | }, 27 | "timestamp": "2025-12-14T07:12:12.959Z", 28 | "nodeVersion": "v22.18.0", 29 | "environment": { 30 | "BAKERYJS_DISABLE_EXPERIMENTAL_TRACING": "1" 31 | } 32 | } 33 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T11-58-20-694Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-10000x100", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 10000, 7 | "nestedItemCount": 100 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 15809, 11 | "messagesProcessed": 1000000, 12 | "avgTimePerMessageMs": 0.015809, 13 | "memoryUsedMB": 1358.4032669067383, 14 | "peakMemoryMB": 0, 15 | "heapStats": { 16 | "heapUsed": 1491117912, 17 | "heapTotal": 1606877184, 18 | "external": 7914011 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 26, 23 | "lastSentMs": 15808, 24 | "drainCompleteMs": 15809, 25 | "sentEventCount": 9050301 26 | }, 27 | "timestamp": "2025-12-14T11:58:20.653Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "613cdc18edfe6bbddf3f9bb77b745467478898f6", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T10-50-02-708Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-1000x100", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 100 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 12923, 11 | "messagesProcessed": 100000, 12 | "avgTimePerMessageMs": 0.12923, 13 | "memoryUsedMB": 208.18551635742188, 14 | "peakMemoryMB": 564.0338592529297, 15 | "heapStats": { 16 | "heapUsed": 377336624, 17 | "heapTotal": 539213824, 18 | "external": 7882590 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 134, 23 | "lastSentMs": 12922, 24 | "drainCompleteMs": 12923, 25 | "sentEventCount": 905031 26 | }, 27 | "timestamp": "2025-12-14T10:50:02.666Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "24a8fa4bdfd60d8daf6ecae461f90c89ddf52015", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T06-33-00-893Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-1000x1000", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 14508, 11 | "messagesProcessed": 1000000, 12 | "avgTimePerMessageMs": 0.014508, 13 | "memoryUsedMB": 1380.376724243164, 14 | "peakMemoryMB": 1452.545295715332, 15 | "heapStats": { 16 | "heapUsed": 1523099536, 17 | "heapTotal": 1598799872, 18 | "external": 7913795 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 25, 23 | "lastSentMs": 14508, 24 | "drainCompleteMs": 14508, 25 | "sentEventCount": 9005031 26 | }, 27 | "timestamp": "2025-12-15T06:33:00.855Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T08-45-13-508Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-1000x1000", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 13494, 11 | "messagesProcessed": 1000000, 12 | "avgTimePerMessageMs": 0.013494, 13 | "memoryUsedMB": 1039.0700073242188, 14 | "peakMemoryMB": 1112.7973022460938, 15 | "heapStats": { 16 | "heapUsed": 1166847928, 17 | "heapTotal": 1296105472, 18 | "external": 7913739 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 23, 23 | "lastSentMs": 13493, 24 | "drainCompleteMs": 13494, 25 | "sentEventCount": 9005031 26 | }, 27 | "timestamp": "2025-12-15T08:45:13.461Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T09-43-48-946Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-1000x1000", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 15608, 11 | "messagesProcessed": 1000000, 12 | "avgTimePerMessageMs": 0.015608, 13 | "memoryUsedMB": 1310.08544921875, 14 | "peakMemoryMB": 1384.0175704956055, 15 | "heapStats": { 16 | "heapUsed": 1451246648, 17 | "heapTotal": 1577664512, 18 | "external": 7913683 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 24, 23 | "lastSentMs": 14969, 24 | "drainCompleteMs": 15608, 25 | "sentEventCount": 9005031 26 | }, 27 | "timestamp": "2025-12-15T09:43:48.907Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "625e95b165dc2ef2cfdd213f254d01027f0b87b5", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T09-44-12-497Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-1000x1000", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 14313, 11 | "messagesProcessed": 1000000, 12 | "avgTimePerMessageMs": 0.014313, 13 | "memoryUsedMB": 1358.763526916504, 14 | "peakMemoryMB": 1432.1696319580078, 15 | "heapStats": { 16 | "heapUsed": 1501734088, 17 | "heapTotal": 1579483136, 18 | "external": 7913643 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 23, 23 | "lastSentMs": 14313, 24 | "drainCompleteMs": 14313, 25 | "sentEventCount": 9005031 26 | }, 27 | "timestamp": "2025-12-15T09:44:12.454Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "625e95b165dc2ef2cfdd213f254d01027f0b87b5", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T06-01-21-074Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-400000x10", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 400000, 7 | "nestedItemCount": 10 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 98288, 11 | "messagesProcessed": 4000000, 12 | "avgTimePerMessageMs": 0.024572, 13 | "memoryUsedMB": 1521.173713684082, 14 | "peakMemoryMB": 1677.9374465942383, 15 | "heapStats": { 16 | "heapUsed": 1759440168, 17 | "heapTotal": 3152560128, 18 | "external": 7874347 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 135, 23 | "lastSentMs": 98287, 24 | "drainCompleteMs": 98288, 25 | "sentEventCount": 38012001 26 | }, 27 | "timestamp": "2025-12-15T06:01:21.027Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T08-55-39-129Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-1000x1000", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 1000, 7 | "nestedItemCount": 1000 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 14833, 11 | "messagesProcessed": 1000000, 12 | "avgTimePerMessageMs": 0.014833, 13 | "memoryUsedMB": 1021.7399520874023, 14 | "peakMemoryMB": 1181.3593139648438, 15 | "heapStats": { 16 | "heapUsed": 1238740216, 17 | "heapTotal": 1381711872, 18 | "external": 7880134 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 142, 23 | "lastSentMs": 14833, 24 | "drainCompleteMs": 14833, 25 | "sentEventCount": 9005031 26 | }, 27 | "timestamp": "2025-12-15T08:55:39.081Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 30 | "environment": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /src/lib/bakeryjs/__tests__/componentNameParser.test.ts: -------------------------------------------------------------------------------- 1 | import { parseComponentName } from '../componentNameParser' 2 | 3 | type PathTestDefinition = { 4 | path: string 5 | expected: string | null 6 | } 7 | 8 | describe('component name parser', () => { 9 | const paths: PathTestDefinition[] = [ 10 | { 11 | path: '_/boxes/generators/processors/test-001.coffee.js.ts', 12 | expected: 'test-001' 13 | }, 14 | { 15 | path: 'test-002.ts', 16 | expected: 'test-002' 17 | }, 18 | { 19 | path: '_/.secret/test-003.ts', 20 | expected: '.secret/test-003' 21 | }, 22 | { 23 | path: '.gitignore', 24 | expected: null 25 | }, 26 | { 27 | path: '_/', 28 | expected: null 29 | } 30 | ] 31 | 32 | paths.forEach(({ path, expected }: PathTestDefinition, index: number) => { 33 | it(`parses name from component file path #${index}`, () => { 34 | const name = parseComponentName(path) 35 | 36 | expect(name).toBe(expected) 37 | }) 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /tests/integration/components/processors/error-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface ErrorProcessorParams { 4 | errorMessage?: string 5 | } 6 | 7 | /** 8 | * A processor that throws an error when processing. 9 | * Useful for testing error handling and propagation in flows. 10 | * 11 | * Parameters: 12 | * - errorMessage: Custom error message (optional, defaults to "Intentional test error") 13 | */ 14 | const ErrorProcessor = boxFactory( 15 | { 16 | provides: [], 17 | requires: [], 18 | emits: [], 19 | aggregates: false, 20 | parameters: { 21 | type: 'object', 22 | properties: { 23 | errorMessage: { type: 'string' } 24 | } 25 | } 26 | }, 27 | function processValue(serviceProvider: ServiceProvider, _value: MessageData): MessageData { 28 | const params = (serviceProvider.parameters as ErrorProcessorParams) ?? {} 29 | throw new Error(params.errorMessage ?? 'Intentional test error') 30 | } 31 | ) 32 | 33 | export default ErrorProcessor 34 | -------------------------------------------------------------------------------- /tests/integration/components/processors/non-error-thrower.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface NonErrorThrowerParams { 4 | throwValue?: string 5 | } 6 | 7 | /** 8 | * A processor that throws a non-Error value (string). 9 | * Useful for testing that non-Error throws are converted to Error. 10 | * 11 | * Parameters: 12 | * - throwValue: The string value to throw (optional, defaults to "string error thrown") 13 | */ 14 | const NonErrorThrower = boxFactory( 15 | { 16 | provides: [], 17 | requires: [], 18 | emits: [], 19 | aggregates: false, 20 | parameters: { 21 | type: 'object', 22 | properties: { 23 | throwValue: { type: 'string' } 24 | } 25 | } 26 | }, 27 | function processValue(serviceProvider: ServiceProvider, _value: MessageData): MessageData { 28 | const params = (serviceProvider.parameters as NonErrorThrowerParams) ?? {} 29 | 30 | throw params.throwValue ?? 'string error thrown' 31 | } 32 | ) 33 | 34 | export default NonErrorThrower 35 | -------------------------------------------------------------------------------- /benchmarks/components/processors/timed-mapper.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Timed Mapper Box for Benchmarking 3 | * 4 | * A simple mapper that tracks processing time and optionally adds 5 | * an artificial delay to simulate real-world processing. 6 | */ 7 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 8 | 9 | module.exports = boxFactory( 10 | { 11 | provides: ['processed', 'processorId', 'processedAt'], 12 | requires: [], 13 | emits: [], 14 | aggregates: false, 15 | parameters: { 16 | title: 'Artificial delay in milliseconds (0 = no delay)', 17 | type: 'number', 18 | minimum: 0, 19 | maximum: 1000, 20 | default: 0 21 | } 22 | }, 23 | async function processValue( 24 | serviceProvider: ServiceProvider, 25 | value: MessageData 26 | ): Promise { 27 | const delayMs = (serviceProvider.parameters as number) || 0 28 | 29 | if (delayMs > 0) { 30 | await new Promise(resolve => setTimeout(resolve, delayMs)) 31 | } 32 | 33 | return { 34 | processed: true, 35 | processorId: 'mapper', 36 | processedAt: Date.now() 37 | } 38 | } 39 | ) 40 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/boxCore/validation.ts: -------------------------------------------------------------------------------- 1 | import Ajv from 'ajv' 2 | import type { ServiceProvider } from '../ServiceProvider' 3 | import { BoxErrorFactory } from '../errors/BoxErrorFactory' 4 | 5 | /** 6 | * Validates box parameters against a JSON schema and returns the updated service provider. 7 | * 8 | * @param schema - The JSON schema to validate against 9 | * @param parameters - The parameters to validate 10 | * @param serviceProvider - The current service provider 11 | * @returns The updated service provider with parameters added 12 | * @throws BoxParametersValidationError if validation fails 13 | */ 14 | export function validateAndAddParameters( 15 | schema: object | string | undefined, 16 | parameters: unknown, 17 | serviceProvider: ServiceProvider 18 | ): ServiceProvider { 19 | if (!schema || !parameters) { 20 | return serviceProvider 21 | } 22 | 23 | const ajvValidator = new Ajv() 24 | if (ajvValidator.validate(schema, parameters)) { 25 | return serviceProvider.addParameters(parameters) 26 | } 27 | 28 | throw BoxErrorFactory.validationError(schema, parameters, ajvValidator.errors) 29 | } 30 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { Program } from './lib/bakeryjs/Program' 2 | import { boxFactory } from './lib/bakeryjs/Box' 3 | import type { BoxExecutiveDefinition, BoxExecutiveBatchDefinition } from './lib/bakeryjs/Box' 4 | import type { BoxMeta, BatchingBoxMeta } from './lib/bakeryjs/BoxI' 5 | import { ServiceProvider } from './lib/bakeryjs/ServiceProvider' 6 | import type { Logger, Service, ServiceContainer } from './lib/bakeryjs/ServiceProvider' 7 | import type { MessageData } from './lib/bakeryjs/Message' 8 | 9 | export { 10 | Program, 11 | boxFactory, 12 | BoxMeta, 13 | BatchingBoxMeta, 14 | BoxExecutiveDefinition, 15 | BoxExecutiveBatchDefinition, 16 | ServiceProvider, 17 | Logger, 18 | Service, 19 | ServiceContainer, 20 | MessageData 21 | } 22 | 23 | if (require.main === module) { 24 | const drainCbk = (msg: any): void => { 25 | console.log(`drain: ${JSON.stringify(msg, undefined, 4)}`) 26 | } 27 | const flowArg = process.argv[2] 28 | if (flowArg) { 29 | new Program({}, {}).run({ flow: flowArg }, drainCbk) 30 | } else { 31 | console.error('Usage: bakeryjs ') 32 | process.exit(1) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 Socialbakers a.s. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "CommonJS", 5 | "lib": ["ES2020"], 6 | "declaration": true, 7 | "declarationMap": true, 8 | "sourceMap": true, 9 | "outDir": "build", 10 | "strict": true, 11 | "noImplicitAny": true, 12 | "strictNullChecks": true, 13 | "strictFunctionTypes": true, 14 | "strictBindCallApply": true, 15 | "strictPropertyInitialization": true, 16 | "noImplicitThis": true, 17 | "alwaysStrict": true, 18 | "noUnusedLocals": true, 19 | "noImplicitReturns": true, 20 | "noFallthroughCasesInSwitch": true, 21 | "noUncheckedIndexedAccess": true, 22 | "exactOptionalPropertyTypes": false, 23 | "forceConsistentCasingInFileNames": true, 24 | "skipLibCheck": true, 25 | "moduleResolution": "node", 26 | "baseUrl": "./", 27 | "paths": { 28 | "bakeryjs": ["src/index.ts"], 29 | "bakeryjs/*": ["src/lib/bakeryjs/*"] 30 | }, 31 | "typeRoots": ["./node_modules/@types", "./src/types"], 32 | "esModuleInterop": true, 33 | "resolveJsonModule": true, 34 | "experimentalDecorators": true, 35 | "emitDecoratorMetadata": true 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We would love to accept your patches and contributions to this project. There are a few small guidelines you need to follow. 4 | 5 | # Contributor License Agreement 6 | 7 | Contributions to this project must be accompanied by a [Contributor License Agreement](https://gist.github.com/vorcigernix/e87d5ecd75e3277640eb8e1c85cf065c). You retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. Signing your CLA is automated as a part of code submission. 8 | 9 | # Code reviews 10 | 11 | All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more information on using pull requests. 12 | 13 | # Community Guidelines 14 | 15 | This project follows [Socialbakers Open Community Code of Conduct](CODE_OF_CONDUCT.md). 16 | 17 | Except as otherwise noted, the content of this page is licensed under CC-BY-4.0 license. Third-party product names and logos may be the trademarks of their respective owners. 18 | -------------------------------------------------------------------------------- /benchmarks/flows/simpleFlow.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Simple Flow Definition for Benchmarking 3 | * 4 | * A minimal flow to establish baseline performance: 5 | * [job] → [configurable-generator (N items)] → [mapper1] → [mapper2] → [drain] 6 | * 7 | * This flow has: 8 | * - 1 generator: Produces N messages 9 | * - 2 mappers: Simple transformations 10 | * - 1 dimension: Root dimension only 11 | */ 12 | 13 | import { FlowExplicitDescription } from '../../src/lib/bakeryjs/FlowBuilderI' 14 | 15 | /** 16 | * Creates a simple flow configuration with the specified item count 17 | * @param itemCount Number of items for the generator to produce 18 | * @returns Flow description for the simple benchmark flow 19 | */ 20 | export function createSimpleFlow(itemCount: number): FlowExplicitDescription { 21 | return { 22 | process: [ 23 | [ 24 | { 25 | 'configurable-generator': [['mapper1'], ['mapper2']] 26 | } 27 | ] 28 | ], 29 | parameters: { 30 | 'configurable-generator': itemCount 31 | } 32 | } 33 | } 34 | 35 | /** 36 | * Default simple flow configuration 37 | */ 38 | export const simpleFlow: FlowExplicitDescription = createSimpleFlow(100) 39 | 40 | export default simpleFlow 41 | -------------------------------------------------------------------------------- /example/components/generators/helloworld.js: -------------------------------------------------------------------------------- 1 | const {boxFactory} = require('bakeryjs'); 2 | const {promisify} = require('util'); 3 | 4 | const timeout = promisify(setTimeout); 5 | 6 | // boxFactory creates a component, 7 | // it must be the default export of the module 8 | module.exports = boxFactory( 9 | // Component's metadata 10 | { 11 | provides: ['msg'], // What kind of data this component provides? 12 | requires: [], // What kind of data this component requires to work? 13 | emits: ['msg'], // This component can emit multiple messages at once, which must be reflected in this property 14 | aggregates: false, // Whether the component can aggregate multiple messages 15 | }, 16 | // Logic of the component, can be async function which emits data over time 17 | async function(serviceProvider, value, emit) { 18 | // Each message is an object with key corresponding to 'emits' property in metadata 19 | emit([{msg: 'Hello world!'}]); 20 | await timeout(230); 21 | // Component can emit multiple messages at once 22 | emit([{msg: '¡Hola mundo!'}, {msg: 'Ahoj světe!'}]); 23 | await timeout(150); 24 | emit([{msg: 'Hallo Welt!'}, {msg: 'Bonjour monde!'}]); 25 | await timeout(200); 26 | } 27 | ); 28 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/processingStrategies/MapperStrategy.ts: -------------------------------------------------------------------------------- 1 | import type { Message, MessageData } from '../Message' 2 | import type { ProcessingStrategy, BoxProcessingContext } from './ProcessingStrategy' 3 | import { BoxErrorFactory } from '../errors/BoxErrorFactory' 4 | 5 | /** 6 | * Strategy for processing messages in mapper mode. 7 | * Transforms a single message into a single output message. 8 | */ 9 | export class MapperStrategy implements ProcessingStrategy { 10 | public async execute(msg: Message, context: BoxProcessingContext): Promise { 11 | try { 12 | const result = await context.processValue( 13 | msg.getInput(context.meta.requires), 14 | (_chunk: MessageData[], _priority?: number) => this.neverEmitCallback(context.name) 15 | ) 16 | msg.setOutput(context.meta.provides, result) 17 | context.queue.push(msg) 18 | } catch (error) { 19 | throw BoxErrorFactory.invocationError( 20 | { name: context.name, meta: context.meta }, 21 | 'mapper', 22 | BoxErrorFactory.toError(error), 23 | msg.getInput(context.meta.requires) 24 | ) 25 | } 26 | } 27 | 28 | private neverEmitCallback(boxName: string): never { 29 | throw BoxErrorFactory.inconsistentBoxError(boxName) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tests/integration/components/processors/batch-error-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | import { BatchingBoxMeta } from '../../../../src/lib/bakeryjs/BoxI' 3 | 4 | interface BatchErrorProcessorParams { 5 | errorMessage?: string 6 | } 7 | 8 | /** 9 | * A batching processor that throws an error when processing. 10 | * Useful for testing error handling in BatchingBox flows. 11 | * 12 | * Parameters: 13 | * - errorMessage: Custom error message (optional, defaults to "Intentional batch error") 14 | */ 15 | const BatchErrorProcessor = boxFactory( 16 | { 17 | provides: [], 18 | requires: [], 19 | aggregates: false, 20 | batch: { 21 | maxSize: 3, 22 | timeoutSeconds: 0.1 23 | }, 24 | parameters: { 25 | type: 'object', 26 | properties: { 27 | errorMessage: { type: 'string' } 28 | } 29 | } 30 | } as BatchingBoxMeta, 31 | async function processValue( 32 | serviceProvider: ServiceProvider, 33 | _batch: MessageData[] 34 | ): Promise { 35 | const params = (serviceProvider.parameters as BatchErrorProcessorParams) ?? {} 36 | throw new Error(params.errorMessage ?? 'Intentional batch error') 37 | } 38 | ) 39 | 40 | export default BatchErrorProcessor 41 | -------------------------------------------------------------------------------- /tests/integration/components/processors/slow-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface SlowProcessorParams { 4 | /** Delay in milliseconds before processing completes */ 5 | delay?: number 6 | } 7 | 8 | /** 9 | * A processor that introduces a configurable delay before returning. 10 | * Useful for testing timing-sensitive flows and async processing. 11 | * 12 | * Parameters: 13 | * - delay: Milliseconds to wait before returning (default: 50) 14 | * 15 | * Adds a 'processedAfterDelay' timestamp to the message. 16 | */ 17 | const SlowProcessor = boxFactory( 18 | { 19 | provides: ['processedAfterDelay'], 20 | requires: [], 21 | emits: [], 22 | aggregates: false, 23 | parameters: { 24 | type: 'object', 25 | properties: { 26 | delay: { type: 'number', minimum: 0 } 27 | } 28 | } 29 | }, 30 | async function processValue( 31 | serviceProvider: ServiceProvider, 32 | _value: MessageData 33 | ): Promise { 34 | const params = (serviceProvider.parameters as SlowProcessorParams) ?? {} 35 | const delay = params.delay ?? 50 36 | 37 | await new Promise(resolve => setTimeout(resolve, delay)) 38 | 39 | return { processedAfterDelay: Date.now() } 40 | } 41 | ) 42 | 43 | export default SlowProcessor 44 | -------------------------------------------------------------------------------- /src/types/sb-jsnetworkx.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'sb-jsnetworkx' { 2 | export interface AttributeDict { 3 | [index: string]: any 4 | } 5 | 6 | export type Node = number | string | object 7 | 8 | export interface Edge { 9 | 0: Node 10 | 1: Node 11 | } 12 | 13 | export interface EdgeWithAttribs extends Edge { 14 | 2: AttributeDict 15 | } 16 | 17 | export interface NodeWithAttribs { 18 | 0: Node 19 | 1: AttributeDict 20 | } 21 | 22 | export class DiGraph { 23 | public node: Map 24 | public constructor() 25 | public addNode(n: Node, attribs?: AttributeDict): void 26 | public addEdge(u: Node, v: Node, attribs?: AttributeDict): void 27 | public addEdgesFrom(ebunch: Edge[], attribs?: AttributeDict): void 28 | public outEdges(u?: Node | Node[], withData?: false): Edge[] 29 | public outEdges(u: Node | Node[], withData: true): EdgeWithAttribs[] 30 | public inEdges(u?: Node | Node[], withData?: false): Edge[] 31 | public inEdges(u: Node | Node[], withData: true): EdgeWithAttribs[] 32 | public nodes(optData?: false): Node[] 33 | public nodes(optData: true): NodeWithAttribs[] 34 | public hasNode(n: Node): boolean 35 | public nodesIter(optData?: false): Iterable 36 | public nodesIter(optData: true): Iterable 37 | } 38 | 39 | export function topologicalSort(g: DiGraph, optNBunch?: Node[]): Node[] 40 | } 41 | -------------------------------------------------------------------------------- /tests/integration/components/processors/async-error-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface AsyncErrorProcessorParams { 4 | errorMessage?: string 5 | delayBeforeError?: number 6 | } 7 | 8 | /** 9 | * A processor that throws an error asynchronously after a delay. 10 | * Useful for testing async error handling in flows. 11 | * 12 | * Parameters: 13 | * - errorMessage: Custom error message (optional, defaults to "Intentional async error") 14 | * - delayBeforeError: Milliseconds to wait before throwing (optional, default: 10) 15 | */ 16 | const AsyncErrorProcessor = boxFactory( 17 | { 18 | provides: [], 19 | requires: [], 20 | emits: [], 21 | aggregates: false, 22 | parameters: { 23 | type: 'object', 24 | properties: { 25 | errorMessage: { type: 'string' }, 26 | delayBeforeError: { type: 'number', minimum: 0 } 27 | } 28 | } 29 | }, 30 | async function processValue( 31 | serviceProvider: ServiceProvider, 32 | _value: MessageData 33 | ): Promise { 34 | const params = (serviceProvider.parameters as AsyncErrorProcessorParams) ?? {} 35 | const delay = params.delayBeforeError ?? 10 36 | 37 | await new Promise(resolve => setTimeout(resolve, delay)) 38 | 39 | throw new Error(params.errorMessage ?? 'Intentional async error') 40 | } 41 | ) 42 | 43 | export default AsyncErrorProcessor 44 | -------------------------------------------------------------------------------- /tests/integration/helpers/message-collector.ts: -------------------------------------------------------------------------------- 1 | import { MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * A utility class for collecting messages from a drain callback. 5 | * Useful for asserting on the messages that exit a flow. 6 | */ 7 | export class MessageCollector { 8 | public messages: MessageData[] = [] 9 | 10 | /** 11 | * Drain callback function to pass to program.run() 12 | */ 13 | public drain = (msg: MessageData): void => { 14 | this.messages.push(msg) 15 | } 16 | 17 | /** 18 | * Clears all collected messages 19 | */ 20 | public clear(): void { 21 | this.messages = [] 22 | } 23 | 24 | /** 25 | * Returns the number of collected messages 26 | */ 27 | public get count(): number { 28 | return this.messages.length 29 | } 30 | 31 | /** 32 | * Returns the first collected message, or undefined if none 33 | */ 34 | public get first(): MessageData | undefined { 35 | return this.messages[0] 36 | } 37 | 38 | /** 39 | * Returns the last collected message, or undefined if none 40 | */ 41 | public get last(): MessageData | undefined { 42 | return this.messages[this.messages.length - 1] 43 | } 44 | 45 | /** 46 | * Checks if any collected message has the specified property with the given value 47 | */ 48 | public hasMessageWith(property: string, value: unknown): boolean { 49 | return this.messages.some(msg => msg[property] === value) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /tests/integration/components/processors/field-provider.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface FieldProviderParams { 4 | /** Field name to provide */ 5 | fieldName?: string 6 | /** Field value to set */ 7 | fieldValue?: unknown 8 | } 9 | 10 | /** 11 | * A processor that provides a configurable field with a configurable value. 12 | * Useful for testing field provision and accumulation. 13 | * 14 | * Parameters: 15 | * - fieldName: Name of the field to provide (default: "providedField") 16 | * - fieldValue: Value to set for the field (default: "provided-value") 17 | * 18 | * Note: Due to BoxMeta limitations, this always provides 'providedField'. 19 | * The fieldName parameter is for documentation/testing reference only. 20 | */ 21 | const FieldProvider = boxFactory( 22 | { 23 | provides: ['providedField'], 24 | requires: [], 25 | emits: [], 26 | aggregates: false, 27 | parameters: { 28 | type: 'object', 29 | properties: { 30 | fieldName: { type: 'string' }, 31 | fieldValue: {} 32 | } 33 | } 34 | }, 35 | function processValue(serviceProvider: ServiceProvider, _value: MessageData): MessageData { 36 | const params = (serviceProvider.parameters as FieldProviderParams) ?? {} 37 | const fieldValue = params.fieldValue ?? 'provided-value' 38 | 39 | return { providedField: fieldValue } 40 | } 41 | ) 42 | 43 | export default FieldProvider 44 | -------------------------------------------------------------------------------- /tests/integration/components/generators/error-generator.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface ErrorGeneratorParams { 4 | errorMessage?: string 5 | delayBeforeError?: number 6 | } 7 | 8 | /** 9 | * A generator that throws an error during processing. 10 | * Useful for testing error handling in generator flows. 11 | * 12 | * Parameters: 13 | * - errorMessage: Custom error message (optional, defaults to "Intentional generator error") 14 | * - delayBeforeError: Milliseconds to wait before throwing (optional) 15 | */ 16 | const ErrorGenerator = boxFactory( 17 | { 18 | provides: ['value'], 19 | requires: [], 20 | emits: ['error_dim'], 21 | aggregates: false, 22 | parameters: { 23 | type: 'object', 24 | properties: { 25 | errorMessage: { type: 'string' }, 26 | delayBeforeError: { type: 'number', minimum: 0 } 27 | } 28 | } 29 | }, 30 | async function processValue( 31 | serviceProvider: ServiceProvider, 32 | _value: MessageData, 33 | _emit: (chunk: MessageData[], priority?: number) => void 34 | ): Promise { 35 | const params = (serviceProvider.parameters as ErrorGeneratorParams) ?? {} 36 | 37 | if (params.delayBeforeError) { 38 | await new Promise(resolve => setTimeout(resolve, params.delayBeforeError)) 39 | } 40 | 41 | throw new Error(params.errorMessage ?? 'Intentional generator error') 42 | } 43 | ) 44 | 45 | export default ErrorGenerator 46 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/BoxI.ts: -------------------------------------------------------------------------------- 1 | import type { Message } from './Message' 2 | import type { EventEmitter } from 'events' 3 | 4 | export type BoxMeta = { 5 | provides: string[] 6 | requires: string[] 7 | emits: string[] 8 | aggregates: boolean 9 | concurrency?: number 10 | parameters?: object | string 11 | } 12 | 13 | export type BatchingBoxMeta = { 14 | provides: string[] 15 | requires: string[] 16 | aggregates: boolean 17 | concurrency?: number 18 | batch: { 19 | maxSize: number 20 | timeoutSeconds?: number 21 | } 22 | parameters?: object | string 23 | } 24 | 25 | export type OnCleanCallback = () => Promise | void 26 | 27 | export interface BoxInterface extends EventEmitter { 28 | // metadata: what I provide, what I require 29 | // needed to barely check the dependencies of the pipeline 30 | readonly meta: BoxMeta 31 | // cleaning actions, e.g. disconnecting the DBs, cleaning internal cache, etc. 32 | readonly onClean: OnCleanCallback[] 33 | // the processing function itself 34 | process(message: Message): Promise 35 | } 36 | 37 | export interface BatchingBoxInterface extends EventEmitter { 38 | // metadata: what I provide, what I require 39 | // needed to barely check the dependencies of the pipeline 40 | readonly meta: BatchingBoxMeta 41 | // cleaning actions, e.g. disconnecting the DBs, cleaning internal cache, etc. 42 | readonly onClean: OnCleanCallback[] 43 | // the processing function itself 44 | process(batch: Message[]): Promise 45 | } 46 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/processingStrategies/ProcessingStrategy.ts: -------------------------------------------------------------------------------- 1 | import type { Message, MessageData } from '../Message' 2 | 3 | /** 4 | * Context provided to processing strategies containing all dependencies 5 | * needed to process messages. 6 | */ 7 | export interface BoxProcessingContext { 8 | /** Box name for error reporting */ 9 | readonly name: string 10 | /** Metadata for the box */ 11 | readonly meta: { 12 | requires: string[] 13 | provides: string[] 14 | emits?: string[] 15 | } 16 | /** Process the value using the box's business logic */ 17 | readonly processValue: ( 18 | msg: MessageData, 19 | emit: (batch: MessageData[], priority?: number) => void 20 | ) => Promise | MessageData | Promise 21 | /** The output queue to push processed messages */ 22 | readonly queue: { 23 | push: (msgs: Message | Message[], priority?: number) => void 24 | } 25 | /** Emit events for tracing */ 26 | readonly emit: (event: string, data: any) => void 27 | } 28 | 29 | /** 30 | * Strategy interface for processing messages. 31 | * Implementations handle the specific logic for each processing mode. 32 | */ 33 | export interface ProcessingStrategy { 34 | /** 35 | * Execute the processing strategy on a message. 36 | * @param msg - The message to process 37 | * @param context - The box context providing dependencies 38 | * @returns Promise resolving when processing is complete 39 | */ 40 | execute(msg: Message, context: BoxProcessingContext): Promise 41 | } 42 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/processingStrategies/processingStrategyRegistry.ts: -------------------------------------------------------------------------------- 1 | import type { ProcessingStrategy } from './ProcessingStrategy' 2 | import { ProcessingMode } from './ProcessingMode' 3 | import { MapperStrategy } from './MapperStrategy' 4 | import { GeneratorStrategy } from './GeneratorStrategy' 5 | import { AggregatorStrategy } from './AggregatorStrategy' 6 | 7 | /** 8 | * Registry of processing strategies keyed by ProcessingMode. 9 | * 10 | * To add a new processing mode: 11 | * 1. Add the new enum value to ProcessingMode 12 | * 2. Create the new strategy class implementing ProcessingStrategy 13 | * 3. Add the new entry to this registry 14 | * 15 | * This follows the Open/Closed Principle - Box.ts remains closed for modification. 16 | */ 17 | export const processingStrategyRegistry: Record = { 18 | [ProcessingMode.Mapper]: new MapperStrategy(), 19 | [ProcessingMode.Generator]: new GeneratorStrategy(), 20 | [ProcessingMode.Aggregator]: new AggregatorStrategy() 21 | } 22 | 23 | /** 24 | * Gets the processing strategy for a given mode. 25 | * @param mode - The processing mode 26 | * @returns The corresponding strategy 27 | * @throws Error if no strategy is registered for the mode 28 | */ 29 | export function getProcessingStrategy(mode: ProcessingMode): ProcessingStrategy { 30 | const strategy = processingStrategyRegistry[mode] 31 | if (!strategy) { 32 | throw new Error(`No processing strategy registered for mode: ${mode}`) 33 | } 34 | return strategy 35 | } 36 | -------------------------------------------------------------------------------- /benchmarks/components/generators/nested-generator.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Nested Generator Box for Complex Flow Benchmarking 3 | * 4 | * This generator creates child messages for each incoming message, 5 | * simulating a second level of generation in a complex flow. 6 | * The number of items is controlled by the 'nestedItemCount' parameter. 7 | */ 8 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 9 | 10 | const DEFAULT_NESTED_COUNT = 10 11 | 12 | module.exports = boxFactory( 13 | { 14 | provides: ['nestedItem', 'nestedGeneratorId', 'nestedTimestamp', 'parentItem'], 15 | requires: ['item'], 16 | emits: ['benchmark_nested_items'], 17 | aggregates: false, 18 | parameters: { 19 | title: 'Number of nested items to generate per parent', 20 | type: 'number', 21 | minimum: 1, 22 | maximum: 100000, 23 | default: DEFAULT_NESTED_COUNT 24 | } 25 | }, 26 | async function processValue( 27 | serviceProvider: ServiceProvider, 28 | value: MessageData, 29 | emit: (chunk: MessageData[], priority?: number) => void 30 | ): Promise { 31 | const nestedCount = (serviceProvider.parameters as number) || DEFAULT_NESTED_COUNT 32 | const nestedGeneratorId = 'gen2' 33 | const parentItem = value.item as number 34 | 35 | const items: MessageData[] = [] 36 | 37 | for (let i = 0; i < nestedCount; i++) { 38 | items.push({ 39 | nestedItem: i, 40 | parentItem, 41 | nestedGeneratorId, 42 | nestedTimestamp: Date.now() 43 | }) 44 | } 45 | 46 | emit(items) 47 | } 48 | ) 49 | -------------------------------------------------------------------------------- /tests/integration/components/processors/parameter-reader.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface ParameterReaderParams { 4 | customParam?: string 5 | numericParam?: number 6 | } 7 | 8 | /** 9 | * A processor that reads parameters from its schema and provides them in the output. 10 | * Useful for testing parameter injection via the box parameters schema. 11 | * 12 | * Parameters: 13 | * - customParam: A custom string parameter (optional) 14 | * - numericParam: A numeric parameter (optional) 15 | * 16 | * Provides: 17 | * - paramCustom: The value of customParam 18 | * - paramNumeric: The value of numericParam 19 | * - hasParameters: Boolean indicating if parameters were provided 20 | */ 21 | const ParameterReader = boxFactory( 22 | { 23 | provides: ['paramCustom', 'paramNumeric', 'hasParameters'], 24 | requires: [], 25 | emits: [], 26 | aggregates: false, 27 | parameters: { 28 | type: 'object', 29 | properties: { 30 | customParam: { type: 'string' }, 31 | numericParam: { type: 'number' } 32 | } 33 | } 34 | }, 35 | function processValue(serviceProvider: ServiceProvider, _value: MessageData): MessageData { 36 | const params = (serviceProvider.parameters as ParameterReaderParams) ?? {} 37 | return { 38 | paramCustom: params.customParam, 39 | paramNumeric: params.numericParam, 40 | hasParameters: params.customParam !== undefined || params.numericParam !== undefined 41 | } 42 | } 43 | ) 44 | 45 | export default ParameterReader 46 | -------------------------------------------------------------------------------- /tests/integration/components/processors/custom-service-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * Shared state for capturing what the custom service received. 5 | * Allows test assertions on custom service usage. 6 | */ 7 | export const customServiceCalls: Array<{ method: string; args: unknown[] }> = [] 8 | 9 | /** 10 | * Clears the custom service calls. Call this in beforeEach() hooks. 11 | */ 12 | export function clearCustomServiceCalls(): void { 13 | customServiceCalls.length = 0 14 | } 15 | 16 | /** 17 | * Interface for the custom service used in tests. 18 | */ 19 | interface CustomService { 20 | process(value: MessageData): unknown 21 | } 22 | 23 | /** 24 | * A processor that uses a custom service named 'customService'. 25 | * Useful for testing arbitrary custom service injection. 26 | * 27 | * This processor calls `customService.process(value)` and captures 28 | * the result in the message. 29 | */ 30 | const CustomServiceProcessor = boxFactory( 31 | { 32 | provides: ['customServiceResult'], 33 | requires: [], 34 | emits: [], 35 | aggregates: false 36 | }, 37 | function processValue(serviceProvider: ServiceProvider, value: MessageData): MessageData { 38 | const customService = serviceProvider.get('customService') 39 | const result = customService.process(value) 40 | customServiceCalls.push({ method: 'process', args: [value] }) 41 | return { customServiceResult: result } 42 | } 43 | ) 44 | 45 | export default CustomServiceProcessor 46 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/scanComponentsPath.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'path' 2 | import * as fs from 'fs' 3 | import { parseComponentName } from './componentNameParser' 4 | 5 | type ComponentsMap = { [componentName: string]: string } 6 | 7 | function isValidDirectory(file: string): boolean { 8 | return file !== '.' && file !== '..' 9 | } 10 | 11 | function processDirectory( 12 | filePath: string, 13 | parentDir: string, 14 | file: string, 15 | availableComponents: ComponentsMap 16 | ): void { 17 | if (isValidDirectory(file)) { 18 | scanComponentsPath(filePath, join(parentDir, file), availableComponents) 19 | } 20 | } 21 | 22 | function processFile( 23 | filePath: string, 24 | parentDir: string, 25 | file: string, 26 | availableComponents: ComponentsMap 27 | ): void { 28 | const name = parseComponentName(join(parentDir, file)) 29 | if (name) { 30 | availableComponents[name] = filePath 31 | } 32 | } 33 | 34 | // TODO: Make async 35 | function scanComponentsPath( 36 | componentsPath: string, 37 | parentDir: string = '', 38 | availableComponents: ComponentsMap = {} 39 | ): ComponentsMap { 40 | const files = fs.readdirSync(componentsPath) 41 | for (const file of files) { 42 | const filePath = join(componentsPath, file) 43 | const stat = fs.statSync(filePath) 44 | if (stat.isDirectory()) { 45 | processDirectory(filePath, parentDir, file, availableComponents) 46 | } else { 47 | processFile(filePath, parentDir, file, availableComponents) 48 | } 49 | } 50 | return availableComponents 51 | } 52 | 53 | export { scanComponentsPath } 54 | -------------------------------------------------------------------------------- /benchmarks/components/generators/configurable-generator.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Configurable Generator Box for Benchmarking 3 | * 4 | * Generates a configurable number of messages for benchmarking purposes. 5 | * The number of items is controlled by the 'itemCount' parameter. 6 | */ 7 | import { boxFactory, ServiceProvider, MessageData } from '../../../src' 8 | 9 | const DEFAULT_ITEM_COUNT = 100 10 | 11 | module.exports = boxFactory( 12 | { 13 | provides: ['item', 'generatorId', 'timestamp'], 14 | requires: [], 15 | emits: ['benchmark_items'], 16 | aggregates: false, 17 | parameters: { 18 | title: 'Number of items to generate', 19 | type: 'number', 20 | minimum: 1, 21 | maximum: 10000000, 22 | default: DEFAULT_ITEM_COUNT 23 | } 24 | }, 25 | async function processValue( 26 | serviceProvider: ServiceProvider, 27 | value: MessageData, 28 | emit: (chunk: MessageData[], priority?: number) => void 29 | ): Promise { 30 | const itemCount = (serviceProvider.parameters as number) || DEFAULT_ITEM_COUNT 31 | const generatorId = 'gen1' 32 | const batchSize = 100 // Emit in batches for efficiency 33 | 34 | const items: MessageData[] = [] 35 | 36 | for (let i = 0; i < itemCount; i++) { 37 | items.push({ 38 | item: i, 39 | generatorId, 40 | timestamp: Date.now() 41 | }) 42 | 43 | // Emit in batches 44 | if (items.length >= batchSize) { 45 | emit(items.slice()) 46 | items.length = 0 47 | } 48 | } 49 | 50 | // Emit remaining items 51 | if (items.length > 0) { 52 | emit(items) 53 | } 54 | } 55 | ) 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | build/ 3 | .env 4 | docs/ 5 | junit.xml 6 | 7 | ### Node ### 8 | # Logs 9 | logs 10 | *.log 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | 15 | # Runtime data 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | 21 | # Directory for instrumented libs generated by jscoverage/JSCover 22 | lib-cov 23 | 24 | # Coverage directory used by tools like istanbul 25 | coverage 26 | 27 | # nyc test coverage 28 | .nyc_output 29 | 30 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 31 | .grunt 32 | 33 | # Bower dependency directory (https://bower.io/) 34 | bower_components 35 | 36 | # node-waf configuration 37 | .lock-wscript 38 | 39 | # Compiled binary addons (https://nodejs.org/api/addons.html) 40 | build/Release 41 | 42 | # Dependency directories 43 | node_modules/ 44 | jspm_packages/ 45 | 46 | # TypeScript v1 declaration files 47 | typings/ 48 | 49 | # Optional npm cache directory 50 | .npm 51 | 52 | # Optional eslint cache 53 | .eslintcache 54 | 55 | # Optional REPL history 56 | .node_repl_history 57 | 58 | # Output of 'npm pack' 59 | *.tgz 60 | 61 | # Yarn Integrity file 62 | .yarn-integrity 63 | 64 | # dotenv environment variables file 65 | .env 66 | 67 | # parcel-bundler cache (https://parceljs.org/) 68 | .cache 69 | 70 | # next.js build output 71 | .next 72 | 73 | # nuxt.js build output 74 | .nuxt 75 | 76 | # vuepress build output 77 | .vuepress/dist 78 | 79 | # Serverless directories 80 | .serverless 81 | 82 | # Profiling 83 | profiling/profiles/*.cpuprofile 84 | profiling/profiles/*.metadata.json 85 | profiling/results/*.json 86 | !profiling/profiles/.gitkeep 87 | !profiling/results/.gitkeep 88 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/stats.ts: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import { EventEmitter } from 'events' 3 | 4 | // TODO: Ugly pattern -- EventEmitter should emits events about changes of internal state. 5 | // If every push for each Message and each queue emits a message, will it overwhelm the nodejs system? 6 | /** 7 | * Events emitted: 8 | * TODO: flow-related: 9 | * - 'sent', timestamp, source, target, batchSize 10 | * - 'queue_in', {boxName: string, batchSize: number} 11 | * - 'queue_stats', {boxName: string, size: number} 12 | * (end of flow-related) 13 | * - 'box_timing', {boxName: string, duration: number} 14 | */ 15 | const eventEmitter = new EventEmitter() 16 | 17 | function qTrace(statsd: boolean = false): MethodDecorator { 18 | return function (target: any, property: string | symbol, descriptor: PropertyDescriptor): void { 19 | assert(property === 'push', 'Queue push decorator not on push!') 20 | 21 | const originValue = descriptor.value 22 | descriptor.value = function (...argsList: any[]) { 23 | const src = (this as any).source 24 | const tgt = (this as any).target 25 | const batchSize: number = 26 | // TODO: fragile way of detecting T[] vs T 27 | // If Message is subclass of T? 28 | Array.isArray(argsList[0]) ? argsList[0].length : 1 29 | 30 | if (src && tgt) { 31 | eventEmitter.emit('sent', Date.now(), (this as any).source, (this as any).target, batchSize) 32 | } 33 | 34 | if (statsd) { 35 | eventEmitter.emit('queue_in', { 36 | boxName: (this as any).target, 37 | batchSize: batchSize 38 | }) 39 | } 40 | return originValue.apply(this as any, argsList) 41 | } 42 | } 43 | } 44 | 45 | export { eventEmitter, qTrace } 46 | -------------------------------------------------------------------------------- /tests/integration/components/processors/accumulator-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | /** 4 | * Shared state for accumulating processed messages across multiple calls. 5 | * This allows test assertions on the order and content of processed messages. 6 | */ 7 | export const accumulatedMessages: MessageData[] = [] 8 | 9 | /** 10 | * Clears the accumulated messages. Call this in beforeEach() hooks. 11 | */ 12 | export function clearAccumulator(): void { 13 | accumulatedMessages.length = 0 14 | } 15 | 16 | /** 17 | * A processor that accumulates all processed messages in a shared array. 18 | * Useful for testing the order and content of messages through a flow. 19 | * 20 | * The processor adds a 'processedAt' field with the processing timestamp 21 | * and an 'accumulatorIndex' field with the order in which it was processed. 22 | * 23 | * Requires 'value' field to capture the message value for priority testing. 24 | * 25 | * Use clearAccumulator() in beforeEach() to reset between tests. 26 | */ 27 | const AccumulatorProcessor = boxFactory( 28 | { 29 | provides: ['processedAt', 'accumulatorIndex'], 30 | requires: ['value'], 31 | emits: [], 32 | aggregates: false 33 | }, 34 | function processValue(_serviceProvider: ServiceProvider, value: MessageData): MessageData { 35 | const index = accumulatedMessages.length 36 | const processedAt = Date.now() 37 | 38 | // Store a copy of the incoming message with processing metadata 39 | accumulatedMessages.push({ 40 | ...value, 41 | processedAt, 42 | accumulatorIndex: index 43 | }) 44 | 45 | return { processedAt, accumulatorIndex: index } 46 | } 47 | ) 48 | 49 | export default AccumulatorProcessor 50 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/FlowCatalog.ts: -------------------------------------------------------------------------------- 1 | import type { Flow } from './Flow' 2 | import type FlowSchemaReaderI from './FlowSchemaReaderI' 3 | import FlowFactory from './FlowFactory' 4 | import type ComponentFactoryI from './ComponentFactoryI' 5 | import type FlowBuilderI from './FlowBuilderI' 6 | import type { FlowExplicitDescription } from './FlowBuilderI' 7 | import type { VisualBuilder } from './builders/VisualBuilder' 8 | import type { PriorityQueueI } from './queue/PriorityQueueI' 9 | import type { Message } from './Message' 10 | import Debug from 'debug' 11 | 12 | const debug = Debug('bakeryjs:flowCatalog') 13 | 14 | export class FlowCatalog { 15 | private readonly flowSchemaReader: FlowSchemaReaderI 16 | private readonly flowFactory: FlowFactory 17 | private readonly visualBuilder: VisualBuilder 18 | 19 | public constructor( 20 | flowSchemaReader: FlowSchemaReaderI, 21 | componentFactory: ComponentFactoryI, 22 | builder: FlowBuilderI, 23 | visualBuilder: VisualBuilder 24 | ) { 25 | this.flowSchemaReader = flowSchemaReader 26 | this.visualBuilder = visualBuilder 27 | this.flowFactory = new FlowFactory(componentFactory, builder) 28 | } 29 | 30 | public async getFlow(flowName: string, drain?: PriorityQueueI): Promise { 31 | const schema = await this.flowSchemaReader.getFlowSchema(flowName) 32 | 33 | debug('getFlow: %s', flowName) 34 | return this.buildFlow(schema, drain) 35 | } 36 | 37 | public async buildFlow( 38 | schema: FlowExplicitDescription, 39 | drain?: PriorityQueueI 40 | ): Promise { 41 | if (debug.enabled) { 42 | const visualSchema = await this.visualBuilder.build(schema) 43 | console.log(visualSchema) 44 | } 45 | return await this.flowFactory.create(schema, drain) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/integration/components/generators/configurable-generator.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface ConfigurableGeneratorParams { 4 | count: number 5 | delay?: number 6 | priority?: number 7 | } 8 | 9 | /** 10 | * A configurable generator that emits a specified number of messages. 11 | * 12 | * Parameters: 13 | * - count: Number of messages to emit (required) 14 | * - delay: Milliseconds to wait before emitting (optional) 15 | * - priority: Priority level for emitted messages (optional) 16 | * 17 | * Each emitted message contains: 18 | * - value: A string like "item-0", "item-1", etc. 19 | * - index: The numeric index of the message 20 | */ 21 | const ConfigurableGenerator = boxFactory( 22 | { 23 | provides: ['value', 'index'], 24 | requires: [], 25 | emits: ['configurable_dim'], 26 | aggregates: false, 27 | parameters: { 28 | type: 'object', 29 | properties: { 30 | count: { type: 'number', minimum: 0 }, 31 | delay: { type: 'number', minimum: 0 }, 32 | priority: { type: 'number' } 33 | }, 34 | required: ['count'] 35 | } 36 | }, 37 | async function processValue( 38 | serviceProvider: ServiceProvider, 39 | _value: MessageData, 40 | emit: (chunk: MessageData[], priority?: number) => void 41 | ): Promise { 42 | const params = serviceProvider.parameters as ConfigurableGeneratorParams 43 | const messages: MessageData[] = [] 44 | 45 | for (let i = 0; i < params.count; i++) { 46 | messages.push({ value: `item-${i}`, index: i }) 47 | } 48 | 49 | if (params.delay) { 50 | await new Promise(resolve => setTimeout(resolve, params.delay)) 51 | } 52 | 53 | emit(messages, params.priority) 54 | } 55 | ) 56 | 57 | export default ConfigurableGenerator 58 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/ServiceProvider.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Base interface for services registered in ServiceProvider. 3 | * Services may optionally implement lifecycle methods. 4 | * 5 | * @publicapi 6 | */ 7 | export interface Service { 8 | initialize?(): Promise 9 | destroy?(): Promise 10 | } 11 | 12 | /** 13 | * Logger interface for the built-in logging service. 14 | * 15 | * @publicapi 16 | */ 17 | export interface Logger extends Service { 18 | log(message: unknown): void 19 | error(message: unknown): void 20 | } 21 | 22 | /** 23 | * Container type for services. 24 | * Services can be any object - the Service interface is a recommended base. 25 | * 26 | * @publicapi 27 | */ 28 | export type ServiceContainer = { 29 | [key: string]: unknown 30 | } 31 | 32 | /** 33 | * Container for both built-in and user-defined services. 34 | * 35 | * # The built-in services 36 | * 1. logger, with methods `log(message)` and `error(message)`. 37 | * 38 | * @publicapi 39 | */ 40 | export class ServiceProvider { 41 | /** @internalapi */ 42 | private readonly services: ServiceContainer 43 | public readonly parameters: unknown 44 | 45 | /** @internalapi */ 46 | public constructor(services: ServiceContainer) { 47 | this.services = services 48 | } 49 | 50 | /** @publicapi */ 51 | public get(name: string): T { 52 | const service = this.services[name] 53 | if (service == null) { 54 | throw new Error(`Service "${name}" was not found.`) 55 | } 56 | 57 | return service as T 58 | } 59 | 60 | /** @internalapi */ 61 | public setAllIn(theContainer: ServiceContainer): void { 62 | Object.assign(this.services, theContainer) 63 | } 64 | 65 | /** @internalapi */ 66 | public addParameters(params: unknown): ServiceProvider { 67 | return Object.create(this, { parameters: { value: params } }) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /tests/integration/components/generators/nested-generator.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface NestedGeneratorParams { 4 | /** Number of messages to emit */ 5 | count?: number 6 | /** Prefix to add to values */ 7 | prefix?: string 8 | } 9 | 10 | /** 11 | * A generator intended for use inside another generator's sub-flow. 12 | * Useful for testing nested generator scenarios (2-level dimension nesting). 13 | * 14 | * Parameters: 15 | * - count: Number of messages to emit (default: 2) 16 | * - prefix: Prefix for the value field (default: "nested") 17 | * 18 | * Each emitted message contains: 19 | * - nestedValue: A string like "nested-0", "nested-1", etc. 20 | * - nestedIndex: The numeric index of the message 21 | * - parentValue: The value field from the parent message (if present) 22 | */ 23 | const NestedGenerator = boxFactory( 24 | { 25 | provides: ['nestedValue', 'nestedIndex', 'parentValue'], 26 | requires: ['value'], 27 | emits: ['nested_dim'], 28 | aggregates: false, 29 | parameters: { 30 | type: 'object', 31 | properties: { 32 | count: { type: 'number', minimum: 0 }, 33 | prefix: { type: 'string' } 34 | } 35 | } 36 | }, 37 | async function processValue( 38 | serviceProvider: ServiceProvider, 39 | value: MessageData, 40 | emit: (chunk: MessageData[], priority?: number) => void 41 | ): Promise { 42 | const params = (serviceProvider.parameters as NestedGeneratorParams) ?? {} 43 | const count = params.count ?? 2 44 | const prefix = params.prefix ?? 'nested' 45 | 46 | const messages: MessageData[] = [] 47 | for (let i = 0; i < count; i++) { 48 | messages.push({ 49 | nestedValue: `${prefix}-${i}`, 50 | nestedIndex: i, 51 | parentValue: value.value ?? null 52 | }) 53 | } 54 | 55 | emit(messages) 56 | } 57 | ) 58 | 59 | export default NestedGenerator 60 | -------------------------------------------------------------------------------- /src/flows/flows.ts: -------------------------------------------------------------------------------- 1 | export default { 2 | all: { 3 | process: [ 4 | [ 5 | { 6 | tick: [['print']] 7 | } 8 | ] 9 | ] 10 | }, 11 | process2: { 12 | process: [ 13 | [ 14 | { 15 | s1_p1: [ 16 | ['s1_p1_s1_p1', 's1_p1_s1_p2', 's1_p1_s1_p3'], 17 | ['s1_p1_s2'], 18 | [ 19 | { 20 | s1_p1_s3_p1: [['s1_p1_s3_p1_s1'], ['s1_p1_s3_p1_s2']] 21 | }, 22 | 's1_p1_s3_p2', 23 | 's1_p1_s3_p3' 24 | ], 25 | [ 26 | 's1_p1_s4_p1', 27 | 's1_p1_s4_p2', 28 | 's1_p1_s4_p3', 29 | 's1_p1_s4_p4', 30 | 's1_p1_s4_p5', 31 | 's1_p1_s4_p6', 32 | 's1_p1_s4_p7' 33 | ], 34 | ['s1_p1_s5_p1', 's1_p1_s5_p2'], 35 | ['s1_p1_s6_p1'], 36 | ['s1_p1_s7_p1', 's1_p1_s7_p2'] 37 | ], 38 | s1_p2: [['s1_p2_s1']] 39 | } 40 | ], 41 | ['s2'], 42 | ['s3_p1', 's3_p2'], 43 | ['s4_p1', 's4_p2'] 44 | ] 45 | }, 46 | process3: { 47 | process: [ 48 | [ 49 | { 50 | fb_grab_feed: [ 51 | ['fb_post_to_mbf', 'fans', 'read_post_metaindex'], 52 | ['compare_etag'], 53 | [ 54 | { 55 | fb_grab_commments: [['fb_comment_to_mbf'], ['fb_comment_save']] 56 | }, 57 | 'fb_grab_video', 58 | 'fb_grab_album' 59 | ], 60 | [ 61 | 'fb_save_video', 62 | 'fb_save_album', 63 | 'calculate_response_time', 64 | 'ppd', 65 | 'acat', 66 | 'bttp', 67 | 'interaction_per_1k_fans' 68 | ], 69 | ['compare_builder_etag', 'compare_content_api_etag'], 70 | ['write_post_metaindex'], 71 | ['put_to_content_api', 'push_to_community'] 72 | ], 73 | fb_grab_profile_insights: [['save_profile_insights']] 74 | } 75 | ], 76 | ['fb_profile_to_mbf'], 77 | ['fb_save_profile', 'fb_save_profile_to_s3'], 78 | ['call_remote_api', 'schedule_next_run'] 79 | ] 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /profiling/run.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env npx ts-node 2 | /** 3 | * BakeryJS CPU Profile Collection Entry Point 4 | * 5 | * Collects CPU profiles by running benchmark flows with Node.js --cpu-prof flag. 6 | * 7 | * Usage: 8 | * npm run profile # Run profiling with defaults 9 | * npm run profile -- --duration=60 # Run for 60 seconds 10 | * npm run profile -- --flow=simple # Run simple flow 11 | * npm run profile -- --items=1000 # Run with 1000 items 12 | */ 13 | 14 | import { parseProfilingArgs, printProfilingHeader } from './cli' 15 | import { collectProfile, saveCollectionMetadata } from './collector' 16 | 17 | /** 18 | * Main entry point for profile collection 19 | */ 20 | async function main(): Promise { 21 | const args = parseProfilingArgs() 22 | 23 | printProfilingHeader(args) 24 | 25 | console.log('Collecting CPU profile...') 26 | console.log('(This will run the benchmark with Node.js --cpu-prof flag)') 27 | console.log() 28 | 29 | const result = await collectProfile(args) 30 | 31 | if (result.success && result.profilePath) { 32 | saveCollectionMetadata(result) 33 | console.log() 34 | console.log('=' + '='.repeat(59)) 35 | console.log('Profile Collection Complete') 36 | console.log('=' + '='.repeat(59)) 37 | console.log(`Profile: ${result.profilePath}`) 38 | console.log(`Duration: ${(result.durationMs / 1000).toFixed(1)}s`) 39 | console.log() 40 | console.log('To analyze this profile, run:') 41 | console.log(` npm run profile:analyze -- --file=${result.profilePath}`) 42 | console.log() 43 | console.log('Or analyze the most recent profile:') 44 | console.log(' npm run profile:analyze') 45 | } else { 46 | console.error() 47 | console.error('Profile collection failed!') 48 | if (result.error) { 49 | console.error(`Error: ${result.error}`) 50 | } 51 | process.exit(1) 52 | } 53 | } 54 | 55 | main().catch(err => { 56 | console.error('Profile collection failed:', err) 57 | process.exit(1) 58 | }) 59 | -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | // Program is an entry point for BakeryJS 2 | const {Program} = require('bakeryjs'); 3 | 4 | // In its minimal form it is initialized with at least one root directory of components 5 | const program = new Program( 6 | // This object is a ServiceProvider, it is passed to each component 7 | // and can be used for dependency injection into components 8 | // (think of e.g. logger or database handler) 9 | {}, 10 | // These are options to initialize the Program 11 | { 12 | componentPaths: [`${__dirname}/components/`], 13 | } 14 | ); 15 | 16 | // Program is an event emitter 17 | // 'sent' event is emitted when a message is sent between components. 18 | // It can be used for simple tracing, as in here, or advanced instrumentations. 19 | program.on('sent', (timestamp, source, target, batchSize) => { 20 | console.log( 21 | `${new Date( 22 | timestamp 23 | ).toLocaleTimeString()} Sent: ${source} --> ${target} (${batchSize})` 24 | ); 25 | }); 26 | 27 | // Job describes what the Program should do. 28 | // The program can either handle multiple incoming jobs, each with unique data flow, 29 | // or it can be kicked-off with a single job and run indifenitely 30 | // prettier-ignore 31 | const job = { 32 | // at least process property is required; it contains a description of data flow 33 | // data flow description contains a sequence of arrays of components, 34 | // each line is run serially and components inside the array run in parallel 35 | process: [ 36 | ['helloworld'], // helloworld is a generator 37 | ['wordcount', 'punctcount'], // these are processors which will run in parallel for each emitted message 38 | ['checksum'], // this processor will run after the previous processors 39 | ] 40 | }; 41 | 42 | // This will start the program with your job 43 | // The second argument is an optional 'drain' function which receives 44 | // all resulting messages which passed through the data flow 45 | program.run(job, (msg) => { 46 | console.log('Drain received a message', msg); 47 | }); 48 | -------------------------------------------------------------------------------- /tests/integration/components/processors/conditional-error-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface ConditionalErrorParams { 4 | /** Fail on odd indices if true */ 5 | failOnOddIndex?: boolean 6 | /** Fail on even indices if true */ 7 | failOnEvenIndex?: boolean 8 | /** Custom error message */ 9 | errorMessage?: string 10 | } 11 | 12 | /** 13 | * A processor that throws an error conditionally based on message content. 14 | * Useful for testing partial failure scenarios where some messages fail 15 | * and others succeed. 16 | * 17 | * Parameters: 18 | * - failOnOddIndex: Throw error if index is odd (default: true) 19 | * - failOnEvenIndex: Throw error if index is even (default: false) 20 | * - errorMessage: Custom error message prefix 21 | * 22 | * Requires 'index' field from incoming message. 23 | * Provides 'processed' field for successfully processed messages. 24 | */ 25 | const ConditionalErrorProcessor = boxFactory( 26 | { 27 | provides: ['processed'], 28 | requires: ['index'], 29 | emits: [], 30 | aggregates: false, 31 | parameters: { 32 | type: 'object', 33 | properties: { 34 | failOnOddIndex: { type: 'boolean' }, 35 | failOnEvenIndex: { type: 'boolean' }, 36 | errorMessage: { type: 'string' } 37 | } 38 | } 39 | }, 40 | function processValue(serviceProvider: ServiceProvider, value: MessageData): MessageData { 41 | const params = (serviceProvider.parameters as ConditionalErrorParams) ?? {} 42 | const failOnOdd = params.failOnOddIndex ?? true 43 | const failOnEven = params.failOnEvenIndex ?? false 44 | const index = value.index as number 45 | 46 | const shouldFail = (failOnOdd && index % 2 === 1) || (failOnEven && index % 2 === 0) 47 | 48 | if (shouldFail) { 49 | const message = params.errorMessage ?? 'Conditional error' 50 | throw new Error(`${message} at index ${index}`) 51 | } 52 | 53 | return { processed: true } 54 | } 55 | ) 56 | 57 | export default ConditionalErrorProcessor 58 | -------------------------------------------------------------------------------- /profiling/parser/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Profile Parsing Entry Point 3 | * 4 | * Re-exports parser functionality and provides high-level parsing API 5 | */ 6 | 7 | // Re-export types and utilities 8 | export { 9 | ParsedProfile, 10 | parseProfileFile, 11 | parseProfileJson, 12 | getNode, 13 | getRootNode, 14 | calculateSelfTimeMs, 15 | isNativeOrInternal, 16 | extractFilePath, 17 | isBakeryJsCode 18 | } from './cpuProfileParser' 19 | 20 | export { 21 | CallTreeOptions, 22 | buildCallTree, 23 | aggregateFunctions, 24 | getHotspots, 25 | getBottlenecks, 26 | getMaxDepth 27 | } from './callTree' 28 | 29 | import { CallTreeNode, FunctionSummary, ProfileMetadata } from '../types' 30 | import { parseProfileFile, ParsedProfile } from './cpuProfileParser' 31 | import { buildCallTree, aggregateFunctions, CallTreeOptions } from './callTree' 32 | 33 | /** Result of parsing a profile */ 34 | export interface ParseResult { 35 | /** Profile metadata */ 36 | metadata: ProfileMetadata 37 | /** Root of the call tree */ 38 | callTree: CallTreeNode 39 | /** Aggregated function summaries */ 40 | functionSummaries: Map 41 | /** The raw parsed profile */ 42 | parsedProfile: ParsedProfile 43 | } 44 | 45 | /** 46 | * Parse a CPU profile file and build the call tree 47 | * 48 | * This is the main entry point for profile parsing, combining 49 | * parsing and tree building into a single convenient function. 50 | * 51 | * @param filePath Path to the .cpuprofile file 52 | * @param options Options for parsing and tree building 53 | * @returns Parsed profile with call tree and function summaries 54 | */ 55 | export function parseProfile(filePath: string, options: CallTreeOptions = {}): ParseResult { 56 | // Parse the raw profile 57 | const parsedProfile = parseProfileFile(filePath) 58 | 59 | // Build the call tree 60 | const callTree = buildCallTree(parsedProfile, options) 61 | 62 | // Aggregate function data 63 | const functionSummaries = aggregateFunctions(callTree) 64 | 65 | return { 66 | metadata: parsedProfile.metadata, 67 | callTree, 68 | functionSummaries, 69 | parsedProfile 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/FlowBuilderI.ts: -------------------------------------------------------------------------------- 1 | import type ComponentFactoryI from './ComponentFactoryI' 2 | import type { PriorityQueueI } from './queue/PriorityQueueI' 3 | import type { Message } from './Message' 4 | import type { Flow } from './Flow' 5 | 6 | export type SchemaComponent = string | SchemaObject 7 | export type ConcurrentSchemaComponent = SchemaComponent[] 8 | export type SerialSchemaComponent = ConcurrentSchemaComponent[] 9 | export type SchemaObject = { [key: string]: SerialSchemaComponent } 10 | export type FlowExplicitDescription = { 11 | [key: string]: SerialSchemaComponent | { [key: string]: any } | undefined 12 | process: SerialSchemaComponent 13 | parameters?: { [key: string]: any } 14 | } 15 | 16 | // TODO: export this automatically from type SchemaObject 17 | export const SchemaObjectValidation = { 18 | type: 'object', 19 | $id: 'bakeryjs/flowbuilder', 20 | title: 'Flow defined by processing steps', 21 | required: ['process'], 22 | properties: { 23 | parameters: { 24 | type: 'object', 25 | title: 26 | 'Set of parameters passed to particular boxes. Key is a box identifier, value is arbitrary.', 27 | patternProperties: { 28 | '^.*$': { title: 'Arbitrary parameter value' } 29 | } 30 | }, 31 | process: { 32 | $id: 'process', 33 | type: 'array', 34 | title: 'Series of sets of boxes. Each set is executed in parallel.', 35 | minItems: 1, 36 | items: { 37 | type: 'array', 38 | title: 'Set of boxes to be run in parallel consuming outputs of boxes of previous stage', 39 | minItems: 1, 40 | items: { 41 | oneOf: [ 42 | { type: 'string', title: 'name of the box' }, 43 | { 44 | type: 'object', 45 | title: 'generator and boxes processing its output', 46 | patternProperties: { 47 | '^.*$': { 48 | $ref: 'process' 49 | } 50 | } 51 | } 52 | ] 53 | } 54 | } 55 | } 56 | } 57 | } 58 | 59 | // I am building Flow and receive the entry to it (the Queue) 60 | export default interface FlowBuilderI { 61 | build( 62 | schema: FlowExplicitDescription, 63 | componentFactory: ComponentFactoryI, 64 | drain?: PriorityQueueI 65 | ): Promise | Flow 66 | } 67 | -------------------------------------------------------------------------------- /tests/integration/components/generators/priority-generator.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | 3 | interface PriorityGeneratorParams { 4 | /** Array of items with their priority and optional delay */ 5 | items?: Array<{ value: string; priority: number; delay?: number }> 6 | } 7 | 8 | /** 9 | * A generator that emits messages with different priorities. 10 | * Each emitted message has a specific priority set at emission time. 11 | * 12 | * Parameters: 13 | * - items: Array of {value, priority, delay?} objects to emit 14 | * 15 | * Each emitted message contains: 16 | * - value: The string value from the item 17 | * - emitOrder: The order in which the message was emitted (0-based) 18 | * - emitPriority: The priority with which the message was emitted 19 | */ 20 | const PriorityGenerator = boxFactory( 21 | { 22 | provides: ['value', 'emitOrder', 'emitPriority'], 23 | requires: [], 24 | emits: ['priority_dim'], 25 | aggregates: false, 26 | parameters: { 27 | type: 'object', 28 | properties: { 29 | items: { 30 | type: 'array', 31 | items: { 32 | type: 'object', 33 | properties: { 34 | value: { type: 'string' }, 35 | priority: { type: 'number' }, 36 | delay: { type: 'number', minimum: 0 } 37 | }, 38 | required: ['value', 'priority'] 39 | } 40 | } 41 | } 42 | } 43 | }, 44 | async function processValue( 45 | serviceProvider: ServiceProvider, 46 | _value: MessageData, 47 | emit: (chunk: MessageData[], priority?: number) => void 48 | ): Promise { 49 | const params = (serviceProvider.parameters as PriorityGeneratorParams) ?? {} 50 | const items = params.items ?? [ 51 | { value: 'low', priority: 1 }, 52 | { value: 'high', priority: 5 }, 53 | { value: 'medium', priority: 3 } 54 | ] 55 | 56 | for (let i = 0; i < items.length; i++) { 57 | const item = items[i] 58 | if (!item) continue 59 | if (item.delay) { 60 | await new Promise(resolve => setTimeout(resolve, item.delay)) 61 | } 62 | emit([{ value: item.value, emitOrder: i, emitPriority: item.priority }], item.priority) 63 | } 64 | } 65 | ) 66 | 67 | export default PriorityGenerator 68 | -------------------------------------------------------------------------------- /benchmarks/output/resultFormatter.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Result formatting utilities for benchmark runner 3 | */ 4 | 5 | import { BenchmarkResult } from '../types' 6 | 7 | /** 8 | * Format a benchmark result for console output 9 | */ 10 | export function formatResult(result: BenchmarkResult): string { 11 | const lines = [ 12 | `\n${'='.repeat(60)}`, 13 | `Benchmark: ${result.name}`, 14 | `${'='.repeat(60)}`, 15 | `Flow Type: ${result.flowType}`, 16 | `Config: items=${result.config.itemCount}${ 17 | result.config.nestedItemCount ? `, nested=${result.config.nestedItemCount}` : '' 18 | }`, 19 | ``, 20 | `Performance Metrics:`, 21 | ` Total Time: ${result.metrics.totalTimeMs.toFixed(2)} ms`, 22 | ` Messages Processed: ${result.metrics.messagesProcessed}`, 23 | ` Avg Time/Message: ${result.metrics.avgTimePerMessageMs.toFixed(4)} ms`, 24 | ` Memory Used: ${result.metrics.memoryUsedMB.toFixed(2)} MB`, 25 | ` Peak Memory: ${result.metrics.peakMemoryMB.toFixed(2)} MB`, 26 | ``, 27 | `Event Timings:`, 28 | ` First Sent: ${result.eventTimings.firstSentMs.toFixed(2)} ms`, 29 | ` Last Sent: ${result.eventTimings.lastSentMs.toFixed(2)} ms`, 30 | ` Drain Complete: ${result.eventTimings.drainCompleteMs.toFixed(2)} ms`, 31 | ` Total Sent Events: ${result.eventTimings.sentEventCount}` 32 | ] 33 | return lines.join('\n') 34 | } 35 | 36 | /** 37 | * Print summary comparison table for multiple benchmark results 38 | */ 39 | export function printSummaryComparison(results: BenchmarkResult[]): void { 40 | if (results.length <= 1) { 41 | return 42 | } 43 | 44 | console.log('\n' + '='.repeat(60)) 45 | console.log('SUMMARY COMPARISON') 46 | console.log('='.repeat(60)) 47 | console.log('\n| Benchmark | Messages | Time (ms) | Avg/Msg (ms) | Memory (MB) |') 48 | console.log('|-----------|----------|-----------|--------------|-------------|') 49 | 50 | for (const r of results) { 51 | console.log( 52 | `| ${r.name.padEnd(9)} | ${String(r.metrics.messagesProcessed).padStart( 53 | 8 54 | )} | ${r.metrics.totalTimeMs.toFixed(1).padStart(9)} | ${r.metrics.avgTimePerMessageMs 55 | .toFixed(4) 56 | .padStart(12)} | ${r.metrics.memoryUsedMB.toFixed(2).padStart(11)} |` 57 | ) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/builders/DefaultVisualBuilder.ts: -------------------------------------------------------------------------------- 1 | import type { VisualBuilder } from './VisualBuilder' 2 | import type { 3 | ConcurrentSchemaComponent, 4 | FlowExplicitDescription, 5 | SchemaObject, 6 | SerialSchemaComponent 7 | } from '../FlowBuilderI' 8 | 9 | const TERMINAL_WIDTH = 80 10 | 11 | export class DefaultVisualBuilder implements VisualBuilder { 12 | public build(schema: FlowExplicitDescription): string { 13 | return this.printSchema({ process: schema.process }) 14 | } 15 | 16 | private printSchema(schema: SchemaObject, indent: string = ''): string { 17 | let output = '' 18 | for (const key of Object.keys(schema)) { 19 | output += `${indent}* ${key} ${'*'.repeat(TERMINAL_WIDTH - 3 - indent.length - key.length)}\n` 20 | indent += '* ' 21 | const subSchema = schema[key] 22 | if (subSchema) { 23 | output += this.printSerial(subSchema, indent) 24 | } 25 | indent = indent.substr(0, indent.length - 2) 26 | output += `${indent}${'*'.repeat(TERMINAL_WIDTH - indent.length)}\n` 27 | } 28 | return output 29 | } 30 | 31 | private printSerial(series: SerialSchemaComponent, indent: string = ''): string { 32 | let output = '' 33 | output += `${indent}SERIAL ${'-'.repeat(TERMINAL_WIDTH - 7 - indent.length)}\n` 34 | for (const serial of series) { 35 | indent += '| ' 36 | output += this.printConcurrent(serial, indent) 37 | indent = indent.substr(0, indent.length - 2) 38 | } 39 | output += `${indent}${'-'.repeat(TERMINAL_WIDTH - indent.length)}\n` 40 | return output 41 | } 42 | 43 | private printConcurrent(concurrencies: ConcurrentSchemaComponent, indent: string = ''): string { 44 | let output = '' 45 | output += `${indent}CONCURRENT ${'-'.repeat(TERMINAL_WIDTH - 11 - indent.length)}\n` 46 | for (const concurrent of concurrencies) { 47 | indent += ' ' 48 | if (typeof concurrent !== 'string') { 49 | output += this.printSchema(concurrent, indent) 50 | output += `${indent}${' '.repeat(TERMINAL_WIDTH - indent.length)}\n` 51 | } else { 52 | output += `${indent}- ${concurrent}\n` 53 | } 54 | indent = indent.substr(0, indent.length - 2) 55 | } 56 | output += `${indent}${'-'.repeat(TERMINAL_WIDTH - indent.length)}\n` 57 | return output 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /benchmarks/config/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Benchmark configuration building 3 | */ 4 | 5 | import { BenchmarkConfig } from '../types' 6 | import { ParsedArgs } from '../cli' 7 | 8 | /** A benchmark configuration with its flow type */ 9 | export interface BenchmarkConfigEntry { 10 | flowType: 'simple' | 'complex' 11 | config: BenchmarkConfig 12 | } 13 | 14 | /** 15 | * Build benchmark configurations based on CLI arguments 16 | */ 17 | export function buildBenchmarkConfigs(args: ParsedArgs): BenchmarkConfigEntry[] { 18 | const configs: BenchmarkConfigEntry[] = [] 19 | 20 | // Check if user provided custom items/nested values 21 | const hasCustomConfig = args.items !== 100 || args.nestedItems !== 10 22 | 23 | if (hasCustomConfig) { 24 | // Use only the custom configuration 25 | if (args.flowType === 'simple' || args.flowType === 'all') { 26 | configs.push({ 27 | flowType: 'simple', 28 | config: { itemCount: args.items } 29 | }) 30 | } 31 | if (args.flowType === 'complex' || args.flowType === 'all') { 32 | configs.push({ 33 | flowType: 'complex', 34 | config: { itemCount: args.items, nestedItemCount: args.nestedItems } 35 | }) 36 | } 37 | } else { 38 | // Use default benchmark suite 39 | if (args.flowType === 'all' || args.flowType === 'simple') { 40 | configs.push( 41 | { flowType: 'simple', config: { itemCount: 10 } }, 42 | { flowType: 'simple', config: { itemCount: 100 } }, 43 | { flowType: 'simple', config: { itemCount: 1000 } } 44 | ) 45 | } 46 | 47 | if (args.flowType === 'all' || args.flowType === 'complex') { 48 | configs.push( 49 | { flowType: 'complex', config: { itemCount: 10, nestedItemCount: 5 } }, 50 | { flowType: 'complex', config: { itemCount: 10, nestedItemCount: 10 } }, 51 | { flowType: 'complex', config: { itemCount: 50, nestedItemCount: 10 } }, 52 | { flowType: 'complex', config: { itemCount: 100, nestedItemCount: 10 } } 53 | ) 54 | } 55 | } 56 | 57 | return configs 58 | } 59 | 60 | /** 61 | * Calculate expected message count for a benchmark configuration 62 | */ 63 | export function calculateExpectedMessages( 64 | flowType: 'simple' | 'complex', 65 | config: BenchmarkConfig 66 | ): number { 67 | return flowType === 'simple' 68 | ? config.itemCount 69 | : config.itemCount * (config.nestedItemCount || 10) 70 | } 71 | -------------------------------------------------------------------------------- /benchmarks/runner/memoryTracker.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Memory tracking utilities for benchmark runner 3 | * 4 | * IMPORTANT: Accurate peak memory tracking during heavy async processing is 5 | * challenging in Node.js. When the event loop is saturated with microtasks 6 | * (Promises), timer-based sampling (setInterval/setImmediate) gets starved 7 | * and never executes. Worker threads can't access the parent's heap memory. 8 | * 9 | * Current approach: 10 | * - Track start memory before processing begins 11 | * - Track end memory after processing completes 12 | * - Peak memory is approximated as the end memory (highest point before GC) 13 | * 14 | * For true peak tracking, BakeryJS would need to expose memory sampling hooks. 15 | */ 16 | 17 | /** Memory tracking state */ 18 | export interface MemoryTrackerState { 19 | peakMemory: number 20 | startMemory: number 21 | } 22 | 23 | /** 24 | * Create a memory tracker 25 | * Captures the initial heap state for baseline comparison 26 | * @returns Memory tracker state 27 | */ 28 | export function createMemoryTracker(): MemoryTrackerState { 29 | const mem = process.memoryUsage() 30 | return { 31 | peakMemory: mem.heapUsed, 32 | startMemory: mem.heapUsed 33 | } 34 | } 35 | 36 | /** 37 | * Update peak memory with current snapshot 38 | * Call this when you have an opportunity to sample (e.g., between processing phases) 39 | */ 40 | export function updatePeakMemory(state: MemoryTrackerState): void { 41 | const mem = process.memoryUsage() 42 | if (mem.heapUsed > state.peakMemory) { 43 | state.peakMemory = mem.heapUsed 44 | } 45 | } 46 | 47 | /** 48 | * Stop memory tracking and take final peak sample 49 | * The end of processing is typically when memory is highest 50 | */ 51 | export function stopMemoryTracker(state: MemoryTrackerState): void { 52 | updatePeakMemory(state) 53 | } 54 | 55 | /** 56 | * Get current memory snapshot 57 | */ 58 | export function getMemorySnapshot(): { 59 | heapUsed: number 60 | heapTotal: number 61 | external: number 62 | rss: number 63 | } { 64 | const mem = process.memoryUsage() 65 | return { 66 | heapUsed: mem.heapUsed, 67 | heapTotal: mem.heapTotal, 68 | external: mem.external, 69 | rss: mem.rss 70 | } 71 | } 72 | 73 | /** 74 | * Force garbage collection if available 75 | */ 76 | export function forceGC(): void { 77 | if (global.gc) { 78 | global.gc() 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /tests/integration/components/processors/batch-processor.ts: -------------------------------------------------------------------------------- 1 | import { boxFactory, ServiceProvider, MessageData } from 'bakeryjs' 2 | import { BatchingBoxMeta } from '../../../../src/lib/bakeryjs/BoxI' 3 | 4 | /** 5 | * Shared state for tracking batch processing order and sizes. 6 | * This allows test assertions on how batches are formed and processed. 7 | */ 8 | export const batchProcessingLog: Array<{ 9 | batchSize: number 10 | items: MessageData[] 11 | processedAt: number 12 | }> = [] 13 | 14 | /** 15 | * Clears the batch processing log. Call this in beforeEach() hooks. 16 | */ 17 | export function clearBatchLog(): void { 18 | batchProcessingLog.length = 0 19 | } 20 | 21 | /** 22 | * A batching processor that collects messages into batches before processing. 23 | * Useful for testing BatchingBox behavior. 24 | * 25 | * Parameters: 26 | * - maxSize: Maximum batch size (default: 3) 27 | * - timeoutSeconds: Timeout before processing partial batch (default: 0.1) 28 | * 29 | * Provides: 30 | * - batchId: A unique identifier for the batch this message was processed in 31 | * - batchIndex: The index of this message within its batch 32 | * - batchSize: The total size of the batch 33 | */ 34 | const BatchProcessor = boxFactory( 35 | { 36 | provides: ['batchId', 'batchIndex', 'batchSize'], 37 | requires: ['index'], 38 | aggregates: false, 39 | batch: { 40 | maxSize: 3, 41 | timeoutSeconds: 0.1 42 | }, 43 | parameters: { 44 | type: 'object', 45 | properties: { 46 | maxSize: { type: 'number', minimum: 1 }, 47 | timeoutSeconds: { type: 'number', minimum: 0 } 48 | } 49 | } 50 | } as BatchingBoxMeta, 51 | async function processValue( 52 | _serviceProvider: ServiceProvider, 53 | batch: MessageData[] 54 | ): Promise { 55 | const batchId = Date.now() + Math.random() 56 | const processedAt = Date.now() 57 | 58 | // Log the batch for test assertions 59 | batchProcessingLog.push({ 60 | batchSize: batch.length, 61 | items: batch.map(item => ({ ...item })), 62 | processedAt 63 | }) 64 | 65 | // Return batch results with batch metadata 66 | // Note: The batch processor only provides new fields; original fields 67 | // are preserved by the framework when setOutput is called 68 | return batch.map((_item, index) => ({ 69 | batchId, 70 | batchIndex: index, 71 | batchSize: batch.length 72 | })) 73 | } 74 | ) 75 | 76 | export default BatchProcessor 77 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - node 4 | - lts/* 5 | - lts/carbon 6 | cache: 7 | directories: 8 | - "$HOME/.npm" 9 | install: npm ci 10 | jobs: 11 | include: 12 | - stage: release 13 | node_js: lts/* 14 | if: tag IS present AND repo = 'Socialbakers/BakeryJS' 15 | script: skip 16 | deploy: 17 | provider: npm 18 | skip_cleanup: true 19 | email: 20 | secure: rwB8GZihhZtAj2dfT4euRM8VdZk/noiqiEGEmHDy3+gYwc8uIZagQ1Gx8I3AiUDJAneoQvF61wPEAS7TEqHg62sqVHSyHn0ksIuT/geFsQQtU+JtH15RSSj9H2qq4v2lvZ/Tw1DWw/SWQV+FwNsIdMaljE22nXCdqQJeojI0WdO61AGIbvd6GOHigcb4WNI/g0mWK34VB4+BR6y+IyLcoPKaGo8/ETL2lcQgWx6HGupnt8Xct01sBXDfbM+SLG40jWN2DsxghwypwwCzL4yJzmuRdybw/Em3oz08O6Ofhm6vHbp5cUBnKCPzhPRj4k11/Ln2GVK4GE5AcVVMnvEErECxHFM8bKB84qLM0I9vnFwVGjXgTdu2/tVsSwpPFgRovNv3+4MA3EECXyy9LPC43I5GMFvjqEsYojmhQ7m9OB3b9qpLaUN6c+TCG7z8Tm1+yn+xqQwxJ4IOnenlC4CwTWLZVgrx0pbas+caK8vqzohQIz7ATKHl44BNhhoVHxzBxKub1kkyU7JYUPrsZDahDxxEvj3cB2xQXw9DSCiP4aQfx/9FrFK4ClmxHhL/+3DZf7LTIhcWKTjZ1d1RidJhU0GWzFvxwUkk2it8LJGYTC8B4thHBpGreLypBpN7E5rqBhIstqTMKG8EBXIRvidnSTDtwTiE3kC+8cyZmRv2DE0= 21 | api_key: 22 | secure: JcvfRvMQmgV8vlJlF9gRWkXz0ChOzmUqa1h2cX8LmVz/O+6BEVYjkDvZ4sOGfwCgdV2UX5+FB1tViWL4uCTYcS2qS220neS1Q6ttWay68f4dMhGOm13ZM8+ZJV5uQYgzGQhsRazNgmfzyqLiKVdOD7vBBfrB0SPs6CEWx8IiYn9K+pPlfIAr5g2kLKUnntOqdqTWUwhXehC23HrjsUrFXl3OMg90QHdb3d9u7f/LYGa7eYZj//VlcRtLD/9DOuOaqfFZ6qi61PWBrfuv+fWPSJf6irDE0CWtnXQmRf92PF70odGELXW1uS/VQM+Q4mc8cryEwTH8XGwXxVh0qKV+zphYmdd2AWXwcDvMKgvXy9iAdkYhg551UI4omRLGE2RHjFDcoSa9LKpxkxi7Ew4InxNZp1gvcWU4HBtSZu7ZY41pL2QwUrmQCEYBLy4pWhgXBVvXlQbJL0vTArhCC1gHnTfcy6JJf7qD2gG23b+EAeHTHcqV2HHEu4hZPtFDHDB2iMFqabRD6A463NdfoUHhSBcKD2MyY/skh2t7SVpTUL2IzDWGVpo4qI/2t9rvDCDQyFU4jryjOr1hBr4S8Cnh4LOaRXAGrroI43CfPzfgkNucHgF0Yo27eW8DJ/wMeCQvrGTiiMirr/Xb0Bb3abCBeK5u1Ifmas91PT8g12Swmlk= 23 | notifications: 24 | slack: 25 | secure: bsPosC5EHo21GN5GISUZgPsy1yMRNbzTPlFKwTGtDAouT5NShJzwEsr8LeHwM8HcXnw4NTmZH+IId7HL1sTMJiv11/HcDt9FpQjxILIjOq6J6XsmaGbHIiA1u744+3OvChWuFxWUky3m/L8rVk55fM7ZNmKh2lr+6iufbtGJeCw137S0U9WzlOY3Ckp4Nct4EELCiYJBYNL4e2M0zq+vvpM67RLgbu3No4v1ym2Tx4Jm4oCKIpAqCYCWAnmfygxAaP0rxQDfGS62pgIELmqnovyUDikiL8xqucALSYxds5OedUJt9JChafe/++aygdv/NpFP6/TlOy07Wuw7O/tqdw0oLPFcC1MAydeKjvpg0HT4rL1kKCaxOz3iz7lkPYpv0njf5GpoaAGgwVHPPhKWbrurIhnlWdldI24nf+v9RdXBa1Vw/TzHukVudrUa2HP/AW4628bsUSnj8PPC0iPP193FtbY2+e6SHJ1VfgnuQzCiE0T+SAIXlnUXHlR9+EX6nAsRsE9sUtWMckpCrp28aR7bBD6gGVWyMGsCcPSzOVbYTxWkY6Us2xAKcU3qB0p3GQvemyh9TS2qYWW21ecOSKbhxcy1Gw/kZzb0+cGM5VaXO3uX17Xb9iw8N15JSU5P2SxP2pAoUJZernP3kFlFvWDHX8A0LHzceWRmA3KYCg8= 26 | -------------------------------------------------------------------------------- /benchmarks/runner/eventHandlers.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Event handling utilities for benchmark runner 3 | */ 4 | 5 | import { Program } from '../../src' 6 | import { EventTimelineEntry } from '../types' 7 | 8 | /** State for tracking sent events */ 9 | export interface SentEventState { 10 | firstSentTimestamp: number 11 | lastSentTimestamp: number 12 | sentCount: number 13 | } 14 | 15 | /** 16 | * Create initial sent event state 17 | */ 18 | export function createSentEventState(): SentEventState { 19 | return { 20 | firstSentTimestamp: -1, 21 | lastSentTimestamp: 0, 22 | sentCount: 0 23 | } 24 | } 25 | 26 | /** 27 | * Subscribe to program events for benchmarking 28 | * @param program - The BakeryJS Program instance 29 | * @param sentState - State object to track sent events 30 | * @param timeline - Optional timeline array for verbose mode 31 | * @param verbose - Whether to record detailed timeline 32 | */ 33 | export function subscribeToEvents( 34 | program: Program, 35 | sentState: SentEventState, 36 | timeline: EventTimelineEntry[], 37 | verbose: boolean 38 | ): void { 39 | program.on('sent', (timestamp: number, source: string, target: string, batchSize: number) => { 40 | if (sentState.firstSentTimestamp < 0) { 41 | sentState.firstSentTimestamp = timestamp 42 | } 43 | sentState.lastSentTimestamp = timestamp 44 | sentState.sentCount++ 45 | 46 | if (verbose) { 47 | timeline.push({ 48 | timestampMs: timestamp, 49 | event: 'sent', 50 | source, 51 | target, 52 | batchSize 53 | }) 54 | } 55 | }) 56 | 57 | program.on('run', () => { 58 | if (verbose) { 59 | timeline.push({ timestampMs: Date.now(), event: 'run' }) 60 | } 61 | }) 62 | } 63 | 64 | /** 65 | * Calculate event timings from state 66 | */ 67 | export function calculateEventTimings( 68 | sentState: SentEventState, 69 | startTime: number, 70 | totalTimeMs: number, 71 | timeline: EventTimelineEntry[], 72 | verbose: boolean 73 | ): { 74 | firstSentMs: number 75 | lastSentMs: number 76 | drainCompleteMs: number 77 | sentEventCount: number 78 | timeline?: EventTimelineEntry[] 79 | } { 80 | const firstSentMs = 81 | sentState.firstSentTimestamp >= 0 ? sentState.firstSentTimestamp - startTime : 0 82 | const lastSentMs = 83 | sentState.lastSentTimestamp > 0 ? sentState.lastSentTimestamp - startTime : totalTimeMs 84 | 85 | return { 86 | firstSentMs, 87 | lastSentMs, 88 | drainCompleteMs: totalTimeMs, 89 | sentEventCount: sentState.sentCount, 90 | ...(verbose ? { timeline } : {}) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /profiling/reporter/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Reporter Entry Point 3 | * 4 | * Orchestrates report generation for profile analysis results. 5 | */ 6 | 7 | // Re-export reporters 8 | export { printConsoleReport } from './consoleReporter' 9 | 10 | export { 11 | generateJsonReport, 12 | printJsonReport, 13 | saveJsonReport, 14 | loadAnalysisResult, 15 | getResultsDir 16 | } from './jsonReporter' 17 | 18 | import { AnalysisResult } from '../types' 19 | import { printConsoleReport } from './consoleReporter' 20 | import { printJsonReport, saveJsonReport } from './jsonReporter' 21 | 22 | /** Options for generating reports */ 23 | export interface ReportOptions { 24 | /** Output JSON format instead of console */ 25 | json?: boolean 26 | /** Output file path for JSON (only if json is true) */ 27 | outputPath?: string 28 | /** Suppress console output */ 29 | silent?: boolean 30 | } 31 | 32 | /** 33 | * Generate a report from analysis results 34 | * 35 | * @param result The analysis result to report 36 | * @param options Report generation options 37 | * @returns Path to saved JSON file if json output was requested, undefined otherwise 38 | */ 39 | export function generateReport( 40 | result: AnalysisResult, 41 | options: ReportOptions = {} 42 | ): string | undefined { 43 | if (options.json) { 44 | if (options.silent) { 45 | // Just save to file 46 | return saveJsonReport(result, options.outputPath) 47 | } else if (options.outputPath) { 48 | // Print and save 49 | printJsonReport(result) 50 | return saveJsonReport(result, options.outputPath) 51 | } else { 52 | // Just print 53 | printJsonReport(result) 54 | return undefined 55 | } 56 | } else { 57 | if (!options.silent) { 58 | printConsoleReport(result) 59 | } 60 | return undefined 61 | } 62 | } 63 | 64 | /** 65 | * Print a summary of the analysis for CI output 66 | */ 67 | export function printCiSummary(result: AnalysisResult): void { 68 | if (result.passed) { 69 | console.log('✅ Profile analysis PASSED') 70 | } else { 71 | console.log('❌ Profile analysis FAILED') 72 | } 73 | 74 | console.log(` Hotspots found: ${result.hotspots.length}`) 75 | console.log(` Bottlenecks found: ${result.bottlenecks.length}`) 76 | console.log(` Regressions found: ${result.regressions.length}`) 77 | } 78 | 79 | /** 80 | * Determine the exit code based on analysis result 81 | * 82 | * @param result The analysis result 83 | * @returns 0 if passed, 1 if failed 84 | */ 85 | export function getExitCode(result: AnalysisResult): number { 86 | return result.passed ? 0 : 1 87 | } 88 | -------------------------------------------------------------------------------- /benchmarks/cli/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * CLI argument parsing for benchmark runner 3 | */ 4 | 5 | /** Parsed command line arguments */ 6 | export interface ParsedArgs { 7 | flowType: 'simple' | 'complex' | 'all' 8 | items: number 9 | nestedItems: number 10 | verbose: boolean 11 | runs: number 12 | timeout: number 13 | } 14 | 15 | /** Default values for CLI arguments */ 16 | const DEFAULTS = { 17 | flowType: 'all' as const, 18 | items: 100, 19 | nestedItems: 10, 20 | verbose: false, 21 | runs: 1, 22 | timeout: 0 // 0 = no timeout 23 | } 24 | 25 | /** 26 | * Parse command line arguments 27 | * 28 | * Supported arguments: 29 | * --flow=simple|complex|all 30 | * --items= 31 | * --nested= 32 | * --verbose 33 | * --runs= 34 | * --timeout= 35 | */ 36 | export function parseArgs(): ParsedArgs { 37 | const args = process.argv.slice(2) 38 | let flowType: 'simple' | 'complex' | 'all' = DEFAULTS.flowType 39 | let items = DEFAULTS.items 40 | let nestedItems = DEFAULTS.nestedItems 41 | let verbose = DEFAULTS.verbose 42 | let runs = DEFAULTS.runs 43 | let timeout = DEFAULTS.timeout 44 | 45 | for (const arg of args) { 46 | if (arg.startsWith('--flow=')) { 47 | const value = arg.split('=')[1] 48 | if (value === 'simple' || value === 'complex' || value === 'all') { 49 | flowType = value 50 | } 51 | } else if (arg.startsWith('--items=')) { 52 | items = parseInt(arg.split('=')[1] as string, 10) 53 | } else if (arg.startsWith('--nested=')) { 54 | nestedItems = parseInt(arg.split('=')[1] as string, 10) 55 | } else if (arg === '--verbose') { 56 | verbose = true 57 | } else if (arg.startsWith('--runs=')) { 58 | runs = parseInt(arg.split('=')[1] as string, 10) 59 | } else if (arg.startsWith('--timeout=')) { 60 | timeout = parseInt(arg.split('=')[1] as string, 10) * 1000 // Convert to ms 61 | } 62 | } 63 | 64 | return { flowType, items, nestedItems, verbose, runs, timeout } 65 | } 66 | 67 | /** 68 | * Print benchmark header information to console 69 | */ 70 | export function printBenchmarkHeader(args: ParsedArgs): void { 71 | console.log('BakeryJS Flow Benchmark') 72 | console.log('=======================') 73 | console.log(`Flow Type: ${args.flowType}`) 74 | console.log(`Items: ${args.items}`) 75 | console.log(`Nested Items: ${args.nestedItems}`) 76 | console.log(`Runs: ${args.runs}`) 77 | console.log(`Verbose: ${args.verbose}`) 78 | console.log(`Timeout: ${args.timeout > 0 ? `${args.timeout / 1000}s` : 'none'}`) 79 | console.log(`Tracing Disabled: ${process.env.BAKERYJS_DISABLE_EXPERIMENTAL_TRACING || 'no'}`) 80 | } 81 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/BoxEvents.ts: -------------------------------------------------------------------------------- 1 | import type { Message } from './Message' 2 | import type { PriorityQueueI } from './queue/PriorityQueueI' 3 | import type { EventEmitter } from 'events' 4 | 5 | interface RevocableQueue extends PriorityQueueI { 6 | revoke(): void 7 | } 8 | 9 | export type MsgEvent = { 10 | boxName: string 11 | messageId: string 12 | parentMsgId: string | undefined 13 | generated?: number 14 | } 15 | 16 | type ModuleOutput = { 17 | generatorTrace: (priorityQ: PriorityQueueI, boxName: string) => PriorityQueueI 18 | guardQueue: (priorityQ: PriorityQueueI) => RevocableQueue 19 | } 20 | 21 | export function boxEvents(flowEmitter: EventEmitter): ModuleOutput { 22 | /** 23 | * Wraps a `push` method of the queue to emit tracing information about pushed 24 | * messages. Intended to track emitted messages in generators. 25 | * 26 | * To ensure it captures only messages generated before the generator 27 | * resolves and not those emitted later, it uses revocable proxy. 28 | * 29 | * @param priorityQ - The priority queue to wrap 30 | * @returns priorityQ with method `push` shadowed and a method for revocation 31 | */ 32 | function generatorTrace( 33 | priorityQ: PriorityQueueI, 34 | boxName: string 35 | ): PriorityQueueI { 36 | function tracedPush(msgs: Message[] | Message, priority?: number): void { 37 | let messages: Message[] 38 | if (!Array.isArray(msgs)) { 39 | messages = [msgs] 40 | } else { 41 | messages = msgs 42 | } 43 | 44 | priorityQ.push.apply(priorityQ, [msgs, priority]) 45 | const messagesTrace: MsgEvent[] = messages.map(m => { 46 | return { 47 | boxName: boxName, 48 | messageId: m.id, 49 | parentMsgId: m.parent && m.parent.id 50 | } 51 | }) 52 | flowEmitter.emit('msg_finished', messagesTrace) 53 | } 54 | 55 | return Object.create(priorityQ, { 56 | push: { value: tracedPush } 57 | }) 58 | } 59 | 60 | function guardQueue(priorityQ: PriorityQueueI): RevocableQueue { 61 | // Prevent errors when generator (wrongly) resolves before emits have finished 62 | // Make pushing into queue through proxy and revoke it once generator resolves 63 | // Any push after will result in TypeError 64 | const { proxy, revoke } = Proxy.revocable(priorityQ.push, { 65 | apply: (tgt, thisArg, argsList) => { 66 | Reflect.apply(tgt, priorityQ, argsList) 67 | } 68 | }) 69 | return Object.create(priorityQ, { 70 | push: { value: proxy }, 71 | revoke: { value: revoke } 72 | }) 73 | } 74 | 75 | return { generatorTrace, guardQueue } 76 | } 77 | -------------------------------------------------------------------------------- /benchmarks/flows/complexFlow.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Complex Flow Definition for Benchmarking 3 | * 4 | * A deeply nested flow to stress-test the TracingModel: 5 | * 6 | * [job] → [gen1 (N items)] → [mapper1] → [mapper2] → [mapper3] 7 | * ↓ 8 | * [gen2 (M items)] → [mapper4] → [mapper5] 9 | * ↓ 10 | * [parallel: mapper6, mapper7, mapper8] 11 | * ↓ 12 | * [mapper9] 13 | * 14 | * This flow has: 15 | * - 2 generators: Creates nested dimensions 16 | * - 9 mappers: Including parallel processing stages 17 | * - 3 dimensions: Root, gen1's dimension (benchmark_items), gen2's dimension (benchmark_nested_items) 18 | * 19 | * Total messages processed = N * M (for the nested dimension) 20 | * TracingModel must track: N messages at level 1, N*M messages at level 2 21 | */ 22 | 23 | import { FlowExplicitDescription } from '../../src/lib/bakeryjs/FlowBuilderI' 24 | 25 | /** 26 | * Creates a complex flow configuration with the specified item counts 27 | * @param itemCount Number of items for the first generator (N) 28 | * @param nestedItemCount Number of items per parent for the nested generator (M) 29 | * @returns Flow description for the complex benchmark flow 30 | */ 31 | export function createComplexFlow( 32 | itemCount: number, 33 | nestedItemCount: number 34 | ): FlowExplicitDescription { 35 | return { 36 | process: [ 37 | [ 38 | { 39 | // First generator creates N items 40 | 'configurable-generator': [ 41 | // Stage 1: Initial mappers in the first dimension 42 | ['mapper1', 'mapper2', 'mapper3'], 43 | // Stage 2: Nested generator with its sub-flow 44 | [ 45 | { 46 | // Second generator creates M items per parent = N*M total 47 | 'nested-generator': [ 48 | // Stage 1 in nested dimension 49 | ['mapper4', 'mapper5'], 50 | // Stage 2: Parallel mappers 51 | ['mapper6', 'mapper7', 'mapper8'], 52 | // Stage 3: Final mapper 53 | ['mapper9'] 54 | ] 55 | } 56 | ] 57 | ] 58 | } 59 | ] 60 | ], 61 | parameters: { 62 | 'configurable-generator': itemCount, 63 | 'nested-generator': nestedItemCount 64 | } 65 | } 66 | } 67 | 68 | /** 69 | * Default complex flow configuration 70 | * N=100 first level items, M=10 nested items = 1000 total leaf messages 71 | */ 72 | export const complexFlow: FlowExplicitDescription = createComplexFlow(100, 10) 73 | 74 | export default complexFlow 75 | -------------------------------------------------------------------------------- /benchmarks/progress/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Progress tracking and logging for benchmark runner 3 | */ 4 | 5 | /** Progress logging interval in milliseconds */ 6 | export const PROGRESS_LOG_INTERVAL = 5000 7 | 8 | /** State for tracking benchmark progress */ 9 | export interface ProgressState { 10 | startTime: number 11 | sentCount: number 12 | drainedCount: number 13 | lastLogTime: number 14 | lastDrainedCount: number 15 | lastSentCount: number 16 | peakMemory: number 17 | expectedMessages: number 18 | } 19 | 20 | /** 21 | * Create initial progress state 22 | */ 23 | export function createProgressState(expectedMessages: number): ProgressState { 24 | const now = Date.now() 25 | return { 26 | startTime: now, 27 | sentCount: 0, 28 | drainedCount: 0, 29 | lastLogTime: now, 30 | lastDrainedCount: 0, 31 | lastSentCount: 0, 32 | peakMemory: 0, 33 | expectedMessages 34 | } 35 | } 36 | 37 | /** 38 | * Log progress during long-running benchmarks 39 | */ 40 | export function logProgress(state: ProgressState): void { 41 | const now = Date.now() 42 | const elapsed = now - state.startTime 43 | const intervalMs = now - state.lastLogTime 44 | const elapsedSec = (elapsed / 1000).toFixed(1) 45 | const mem = process.memoryUsage() 46 | const memMB = (mem.heapUsed / 1024 / 1024).toFixed(1) 47 | 48 | // Calculate instantaneous rate (messages in last interval) 49 | const msgsDelta = state.drainedCount - state.lastDrainedCount 50 | const sentDelta = state.sentCount - state.lastSentCount 51 | const instantRate = intervalMs > 0 ? ((msgsDelta / intervalMs) * 1000).toFixed(1) : '0' 52 | 53 | const pct = 54 | state.expectedMessages > 0 55 | ? ((state.drainedCount / state.expectedMessages) * 100).toFixed(1) 56 | : '?' 57 | 58 | console.log( 59 | ` [${elapsedSec}s] Progress: ${state.drainedCount}/${state.expectedMessages} msgs (${pct}%), ` + 60 | `+${msgsDelta} msgs, +${sentDelta} sent, ${instantRate} msgs/sec, ${memMB} MB heap` 61 | ) 62 | 63 | // Update last values for next interval 64 | state.lastLogTime = now 65 | state.lastDrainedCount = state.drainedCount 66 | state.lastSentCount = state.sentCount 67 | } 68 | 69 | /** 70 | * Create a progress logging interval 71 | * @returns The interval ID for cleanup 72 | */ 73 | export function startProgressLogging( 74 | state: ProgressState, 75 | getSentCount: () => number, 76 | getDrainedCount: () => number, 77 | getPeakMemory: () => number 78 | ): NodeJS.Timeout { 79 | return setInterval(() => { 80 | state.sentCount = getSentCount() 81 | state.drainedCount = getDrainedCount() 82 | state.peakMemory = getPeakMemory() 83 | logProgress(state) 84 | }, PROGRESS_LOG_INTERVAL) 85 | } 86 | -------------------------------------------------------------------------------- /benchmarks/run.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env npx ts-node 2 | /** 3 | * BakeryJS Flow Benchmark Runner 4 | * 5 | * Executes benchmark flows and collects performance statistics. 6 | * 7 | * Usage: 8 | * npm run benchmark # Run all benchmarks 9 | * npm run benchmark:simple # Run simple flow benchmarks only 10 | * npm run benchmark:complex # Run complex flow benchmarks only 11 | * npm run benchmark:verbose # Run with detailed event timeline 12 | * 13 | * # With custom options: 14 | * npm run benchmark -- --items=1000 15 | * npm run benchmark -- --flow=simple --items=500 16 | * npm run benchmark -- --timeout=60 17 | * 18 | * # Compare with tracing disabled: 19 | * BAKERYJS_DISABLE_EXPERIMENTAL_TRACING=1 npm run benchmark 20 | */ 21 | 22 | import { BenchmarkResult } from './types' 23 | import { parseArgs, printBenchmarkHeader, ParsedArgs } from './cli' 24 | import { buildBenchmarkConfigs, BenchmarkConfigEntry } from './config' 25 | import { runBenchmark } from './runner' 26 | import { formatResult, printSummaryComparison, saveResults } from './output' 27 | 28 | /** 29 | * Run all benchmarks for the given configurations 30 | */ 31 | async function runAllBenchmarks( 32 | configs: BenchmarkConfigEntry[], 33 | args: ParsedArgs 34 | ): Promise { 35 | const results: BenchmarkResult[] = [] 36 | 37 | for (const { flowType, config } of configs) { 38 | console.log(`\nRunning ${flowType} benchmark with config:`, config) 39 | 40 | for (let run = 0; run < args.runs; run++) { 41 | if (args.runs > 1) console.log(` Run ${run + 1}/${args.runs}...`) 42 | 43 | try { 44 | const result = await runBenchmark({ 45 | flowType, 46 | config, 47 | verbose: args.verbose, 48 | timeout: args.timeout 49 | }) 50 | results.push(result) 51 | console.log(formatResult(result)) 52 | } catch (err) { 53 | console.error(`\nBenchmark error: ${(err as Error).message}`) 54 | // Continue with other benchmarks 55 | } 56 | } 57 | } 58 | 59 | return results 60 | } 61 | 62 | /** 63 | * Main entry point for the benchmark runner 64 | */ 65 | async function main(): Promise { 66 | // Parse command line arguments 67 | const args = parseArgs() 68 | 69 | // Print header information 70 | printBenchmarkHeader(args) 71 | 72 | // Build benchmark configurations 73 | const configs = buildBenchmarkConfigs(args) 74 | 75 | // Run all benchmarks 76 | const results = await runAllBenchmarks(configs, args) 77 | 78 | // Print summary comparison 79 | printSummaryComparison(results) 80 | 81 | // Save results to file 82 | saveResults(results) 83 | 84 | // Force exit since Program may have lingering event listeners 85 | process.exit(0) 86 | } 87 | 88 | main().catch(err => { 89 | console.error('Benchmark failed:', err) 90 | process.exit(1) 91 | }) 92 | -------------------------------------------------------------------------------- /tests/integration/helpers/event-tracker.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Represents a 'sent' event emitted when messages transition between boxes 3 | */ 4 | export interface SentEvent { 5 | timestamp: number 6 | source: string 7 | target: string 8 | batchSize: number 9 | } 10 | 11 | /** 12 | * Represents a 'run' event emitted when a flow starts 13 | */ 14 | export interface RunEvent { 15 | timestamp: number 16 | flow: unknown 17 | job: unknown 18 | } 19 | 20 | /** 21 | * A utility class for tracking events emitted by a Program during flow execution. 22 | * Useful for asserting on message transitions and flow execution. 23 | */ 24 | export class EventTracker { 25 | public sentEvents: SentEvent[] = [] 26 | public runEvents: RunEvent[] = [] 27 | 28 | /** 29 | * Callback for tracking 'sent' events. 30 | * Use with program.on('sent', eventTracker.trackSent) 31 | */ 32 | public trackSent = ( 33 | timestamp: number, 34 | source: string, 35 | target: string, 36 | batchSize: number 37 | ): void => { 38 | this.sentEvents.push({ timestamp, source, target, batchSize }) 39 | } 40 | 41 | /** 42 | * Callback for tracking 'run' events. 43 | * The 'run' event emits (flow, job) - we capture the timestamp ourselves 44 | * Use with program.on('run', eventTracker.trackRun) 45 | */ 46 | public trackRun = (flow: unknown, job: unknown): void => { 47 | this.runEvents.push({ timestamp: Date.now(), flow, job }) 48 | } 49 | 50 | /** 51 | * Returns all source→target transitions as simple objects 52 | */ 53 | public getTransitions(): Array<{ from: string; to: string }> { 54 | return this.sentEvents.map(e => ({ from: e.source, to: e.target })) 55 | } 56 | 57 | /** 58 | * Returns transitions to a specific target box 59 | */ 60 | public getTransitionsTo(target: string): SentEvent[] { 61 | return this.sentEvents.filter(e => e.target === target) 62 | } 63 | 64 | /** 65 | * Returns transitions from a specific source box 66 | */ 67 | public getTransitionsFrom(source: string): SentEvent[] { 68 | return this.sentEvents.filter(e => e.source === source) 69 | } 70 | 71 | /** 72 | * Checks if a specific transition occurred 73 | */ 74 | public hasTransition(from: string, to: string): boolean { 75 | return this.sentEvents.some(e => e.source === from && e.target === to) 76 | } 77 | 78 | /** 79 | * Clears all tracked events 80 | */ 81 | public clear(): void { 82 | this.sentEvents = [] 83 | this.runEvents = [] 84 | } 85 | 86 | /** 87 | * Returns the total number of sent events 88 | */ 89 | public get sentCount(): number { 90 | return this.sentEvents.length 91 | } 92 | 93 | /** 94 | * Returns the total batch size across all sent events 95 | */ 96 | public get totalBatchSize(): number { 97 | return this.sentEvents.reduce((sum, e) => sum + e.batchSize, 0) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/processingStrategies/GeneratorStrategy.ts: -------------------------------------------------------------------------------- 1 | import type { Message, MessageData } from '../Message' 2 | import type { ProcessingStrategy, BoxProcessingContext } from './ProcessingStrategy' 3 | import VError from 'verror' 4 | import { BoxErrorFactory } from '../errors/BoxErrorFactory' 5 | 6 | const MISBEHAVE_DESCRIPTION = 7 | 'Generator should return a promise that would be resolved once all the messages had been emitted.' + 8 | ' This error occurs when generator attempts to emit messages after its promise has been resolved.' + 9 | ' The code of the generator should be repaired to keep the contract.' 10 | 11 | /** 12 | * Creates a revocable queue proxy that prevents emissions after resolution. 13 | */ 14 | function createGuardedQueue(queue: { 15 | push: (msgs: Message | Message[], priority?: number) => void 16 | }): { 17 | push: (msgs: Message | Message[], priority?: number) => void 18 | revoke: () => void 19 | } { 20 | const { proxy, revoke } = Proxy.revocable(queue.push, { 21 | apply: (tgt, _thisArg, argsList) => { 22 | Reflect.apply(tgt, queue, argsList) 23 | } 24 | }) 25 | return { push: proxy, revoke } 26 | } 27 | 28 | /** 29 | * Strategy for processing messages in generator mode. 30 | * Transforms a single message into multiple output messages. 31 | */ 32 | export class GeneratorStrategy implements ProcessingStrategy { 33 | public async execute(msg: Message, context: BoxProcessingContext): Promise { 34 | let siblingsCount = 0 35 | const guardedQ = createGuardedQueue(context.queue) 36 | 37 | try { 38 | const retValue = await context.processValue( 39 | msg.getInput(context.meta.requires), 40 | (chunk: MessageData[], priority?: number) => { 41 | siblingsCount += chunk.length 42 | guardedQ.push( 43 | chunk.map(msgData => { 44 | const parent: Message = msg.create() 45 | parent.setOutput(context.meta.provides, msgData) 46 | return parent 47 | }), 48 | priority 49 | ) 50 | } 51 | ) 52 | 53 | guardedQ.revoke() 54 | context.emit('generation_finished', [ 55 | { 56 | boxName: context.name, 57 | messageId: msg.id, 58 | parentMsgId: msg.parent && msg.parent.id, 59 | generated: siblingsCount 60 | } 61 | ]) 62 | 63 | return retValue 64 | } catch (error) { 65 | throw this.handleError(error, msg, context) 66 | } 67 | } 68 | 69 | private handleError(error: unknown, msg: Message, context: BoxProcessingContext): VError { 70 | const boxInfo = { name: context.name, meta: context.meta } 71 | const inputValue = msg.getInput(context.meta.requires) 72 | 73 | if (error instanceof TypeError && (error as Error).message.includes('revoked')) { 74 | return BoxErrorFactory.misbehaveError(boxInfo, MISBEHAVE_DESCRIPTION, inputValue) 75 | } 76 | 77 | return BoxErrorFactory.invocationError( 78 | boxInfo, 79 | 'generator', 80 | BoxErrorFactory.toError(error), 81 | inputValue 82 | ) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bakeryjs", 3 | "version": "1.0.0-alpha.1", 4 | "description": "FBP-inspired data processing library for Node.js", 5 | "main": "build/index.js", 6 | "types": "build/index.d.ts", 7 | "author": "Socialbakers ", 8 | "contributors": [ 9 | "Milan Lepík ", 10 | "Jakub Slovan ", 11 | "Martin Štekl ", 12 | "Jan Vlnas " 13 | ], 14 | "license": "MIT", 15 | "files": [ 16 | "build/" 17 | ], 18 | "engines": { 19 | "node": ">=24.0.0", 20 | "npm": ">=11.0.0" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/Socialbakers/BakeryJS.git" 25 | }, 26 | "dependencies": { 27 | "ajv": "^8.17.1", 28 | "debug": "^4.4.3", 29 | "sb-jsnetworkx": "^0.3.6", 30 | "verror": "^1.10.1" 31 | }, 32 | "devDependencies": { 33 | "@types/debug": "^4.1.12", 34 | "@types/jest": "^29.5.14", 35 | "@types/node": "^20.19.27", 36 | "@types/verror": "^1.10.11", 37 | "@typescript-eslint/eslint-plugin": "^8.49.0", 38 | "@typescript-eslint/parser": "^8.49.0", 39 | "eslint": "^9.39.2", 40 | "eslint-config-prettier": "^10.1.8", 41 | "eslint-plugin-jest": "^29.4.0", 42 | "eslint-plugin-prettier": "^5.5.4", 43 | "globals": "^16.5.0", 44 | "jest": "^29.7.0", 45 | "json5": "^2.2.3", 46 | "nodemon": "^3.1.11", 47 | "prettier": "^3.7.4", 48 | "ts-jest": "^29.4.6", 49 | "ts-node": "^10.9.2", 50 | "typedoc": "^0.28.15", 51 | "typescript": "^5.9.3", 52 | "typescript-eslint": "^8.49.0" 53 | }, 54 | "scripts": { 55 | "start": "npm run build:live", 56 | "build:live": "nodemon --exec ./node_modules/.bin/ts-node -- ./src/index.ts all", 57 | "build": "tsc -b tsconfig.build.json", 58 | "build:watch": "tsc -b tsconfig.build.json --watch", 59 | "prepare": "npm run build", 60 | "test": "jest", 61 | "test:watch": "jest --watch", 62 | "coverage": "jest --coverage", 63 | "lint": "eslint src/ tests/ benchmarks/ profiling/", 64 | "lint:fix": "eslint src/ tests/ benchmarks/ profiling/ --fix", 65 | "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" \"benchmarks/**/*.ts\" \"profiling/**/*.ts\"", 66 | "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\" \"benchmarks/**/*.ts\" \"profiling/**/*.ts\"", 67 | "typecheck": "tsc --noEmit", 68 | "code-quality": "npm run format && npm run lint:fix && npm run typecheck", 69 | "doc": "typedoc --out ./docs/ src/", 70 | "benchmark": "ts-node --transpile-only benchmarks/run.ts", 71 | "benchmark:simple": "ts-node --transpile-only benchmarks/run.ts --flow=simple", 72 | "benchmark:complex": "ts-node --transpile-only benchmarks/run.ts --flow=complex", 73 | "benchmark:verbose": "ts-node --transpile-only benchmarks/run.ts --verbose", 74 | "profile": "ts-node --transpile-only profiling/run.ts", 75 | "profile:analyze": "ts-node --transpile-only profiling/analyze.ts" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /profiling/analyze.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env npx ts-node 2 | /** 3 | * BakeryJS CPU Profile Analysis Entry Point 4 | * 5 | * Analyzes CPU profiles and generates hotspot reports. 6 | * 7 | * Usage: 8 | * npm run profile:analyze # Analyze most recent profile 9 | * npm run profile:analyze -- --file= # Analyze specific profile 10 | * npm run profile:analyze -- --json # Output JSON format 11 | * npm run profile:analyze -- --baseline= # Compare against baseline 12 | */ 13 | 14 | import { parseProfilingArgs, printAnalysisHeader } from './cli' 15 | import { findMostRecentProfile } from './collector' 16 | import { parseProfile } from './parser' 17 | import { analyzeProfile } from './analyzer' 18 | import { generateReport, printCiSummary, getExitCode, loadAnalysisResult } from './reporter' 19 | 20 | /** 21 | * Main entry point for profile analysis 22 | */ 23 | async function main(): Promise { 24 | const args = parseProfilingArgs() 25 | 26 | // Find the profile to analyze 27 | let profilePath = args.file 28 | if (!profilePath) { 29 | profilePath = findMostRecentProfile() ?? undefined 30 | if (!profilePath) { 31 | console.error('No profile files found. Run "npm run profile" first to collect a profile.') 32 | process.exit(1) 33 | } 34 | } 35 | 36 | if (!args.json) { 37 | printAnalysisHeader(args) 38 | console.log(`Analyzing: ${profilePath}`) 39 | console.log() 40 | } 41 | 42 | // Parse the profile 43 | let parseResult 44 | try { 45 | parseResult = parseProfile(profilePath, { 46 | includeInternals: args.includeInternals 47 | }) 48 | } catch (err) { 49 | console.error(`Failed to parse profile: ${(err as Error).message}`) 50 | process.exit(1) 51 | } 52 | 53 | // Load baseline if provided 54 | let baselineResult = undefined 55 | if (args.baseline) { 56 | baselineResult = loadAnalysisResult(args.baseline) ?? undefined 57 | if (!baselineResult) { 58 | console.error(`Failed to load baseline: ${args.baseline}`) 59 | process.exit(1) 60 | } 61 | } 62 | 63 | // Analyze the profile 64 | const analysisResult = analyzeProfile(parseResult, { 65 | thresholds: { 66 | hotspotSelfTimePercent: args.hotspotThreshold, 67 | regressionChangePercent: args.regressionThreshold, 68 | topN: args.topN 69 | }, 70 | includeInternals: args.includeInternals, 71 | baseline: baselineResult 72 | }) 73 | 74 | // Generate report 75 | const savedPath = generateReport(analysisResult, { 76 | json: args.json, 77 | outputPath: args.output 78 | }) 79 | 80 | if (savedPath && !args.json) { 81 | console.log(`\nResults saved to: ${savedPath}`) 82 | } 83 | 84 | // Print CI summary 85 | if (!args.json) { 86 | console.log() 87 | printCiSummary(analysisResult) 88 | } 89 | 90 | // Exit with appropriate code 91 | process.exit(getExitCode(analysisResult)) 92 | } 93 | 94 | main().catch(err => { 95 | console.error('Profile analysis failed:', err) 96 | process.exit(1) 97 | }) 98 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import tseslint from 'typescript-eslint' 2 | import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended' 3 | import jest from 'eslint-plugin-jest' 4 | import globals from 'globals' 5 | 6 | export default tseslint.config( 7 | // Global ignores 8 | { 9 | ignores: ['build/**', 'node_modules/**', 'docs/**'] 10 | }, 11 | 12 | // Base recommended configs 13 | ...tseslint.configs.recommended, 14 | 15 | // Base config for all files 16 | { 17 | languageOptions: { 18 | ecmaVersion: 2022, 19 | sourceType: 'module', 20 | globals: { 21 | ...globals.node, 22 | ...globals.es2021 23 | } 24 | } 25 | }, 26 | 27 | // TypeScript-specific rules for .ts files 28 | { 29 | files: ['**/*.ts'], 30 | rules: { 31 | // Disable rules from recommended that are too strict for this codebase 32 | '@typescript-eslint/no-explicit-any': 'off', 33 | '@typescript-eslint/no-unused-vars': 'off', 34 | '@typescript-eslint/no-require-imports': 'off', 35 | '@typescript-eslint/no-unsafe-function-type': 'off', 36 | 37 | '@typescript-eslint/adjacent-overload-signatures': 'error', 38 | '@typescript-eslint/explicit-function-return-type': [ 39 | 'warn', 40 | { 41 | allowExpressions: true 42 | } 43 | ], 44 | '@typescript-eslint/explicit-member-accessibility': 'warn', 45 | '@typescript-eslint/consistent-type-assertions': 'error', 46 | '@typescript-eslint/no-array-constructor': 'error', 47 | '@typescript-eslint/no-inferrable-types': [ 48 | 'error', 49 | { 50 | ignoreParameters: true, 51 | ignoreProperties: true 52 | } 53 | ], 54 | '@typescript-eslint/no-namespace': [ 55 | 'error', 56 | { 57 | allowDeclarations: true 58 | } 59 | ], 60 | '@typescript-eslint/no-non-null-assertion': 'error', 61 | '@typescript-eslint/parameter-properties': 'error', 62 | '@typescript-eslint/triple-slash-reference': 'error' 63 | } 64 | }, 65 | 66 | // Complexity rule for src/ files only (enforces max cyclomatic complexity of 6) 67 | { 68 | files: ['src/**/*.ts'], 69 | rules: { 70 | complexity: ['error', { max: 6 }] 71 | } 72 | }, 73 | 74 | // Jest config for test files 75 | { 76 | files: ['**/*.test.ts', 'tests/**/*.ts'], 77 | plugins: { 78 | jest 79 | }, 80 | languageOptions: { 81 | globals: { 82 | ...globals.jest 83 | } 84 | }, 85 | rules: { 86 | 'jest/no-disabled-tests': 'warn', 87 | 'jest/no-focused-tests': 'error', 88 | 'jest/no-identical-title': 'error', 89 | 'jest/prefer-to-have-length': 'warn', 90 | 'jest/valid-expect': 'error' 91 | } 92 | }, 93 | 94 | // Prettier config (must be last to override other formatting rules) 95 | // Options are read from .prettierrc for consistency with editor extensions 96 | eslintPluginPrettierRecommended 97 | ) 98 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/builders/__tests__/DefaultVisualBuilder.test.ts: -------------------------------------------------------------------------------- 1 | import { DefaultVisualBuilder } from '../DefaultVisualBuilder' 2 | import type { FlowExplicitDescription } from '../../FlowBuilderI' 3 | 4 | describe('DefaultVisualBuilder', () => { 5 | let builder: DefaultVisualBuilder 6 | 7 | beforeEach(() => { 8 | builder = new DefaultVisualBuilder() 9 | }) 10 | 11 | describe('build', () => { 12 | it('builds visual representation for a simple linear flow', () => { 13 | const schema: FlowExplicitDescription = { 14 | process: [['boxA'], ['boxB'], ['boxC']] 15 | } 16 | 17 | const result = builder.build(schema) 18 | 19 | expect(result).toContain('* process') 20 | expect(result).toContain('SERIAL') 21 | expect(result).toContain('CONCURRENT') 22 | expect(result).toContain('- boxA') 23 | expect(result).toContain('- boxB') 24 | expect(result).toContain('- boxC') 25 | }) 26 | 27 | it('builds visual representation for parallel boxes', () => { 28 | const schema: FlowExplicitDescription = { 29 | process: [['boxA', 'boxB'], ['boxC']] 30 | } 31 | 32 | const result = builder.build(schema) 33 | 34 | expect(result).toContain('- boxA') 35 | expect(result).toContain('- boxB') 36 | expect(result).toContain('- boxC') 37 | }) 38 | 39 | it('builds visual representation for nested generator flows', () => { 40 | const schema: FlowExplicitDescription = { 41 | process: [ 42 | [ 43 | { 44 | generator: [['nestedBoxA'], ['nestedBoxB']] 45 | } 46 | ], 47 | ['boxC'] 48 | ] 49 | } 50 | 51 | const result = builder.build(schema) 52 | 53 | expect(result).toContain('* process') 54 | expect(result).toContain('* generator') 55 | expect(result).toContain('- nestedBoxA') 56 | expect(result).toContain('- nestedBoxB') 57 | expect(result).toContain('- boxC') 58 | }) 59 | 60 | it('builds visual representation for empty process', () => { 61 | const schema: FlowExplicitDescription = { 62 | process: [] 63 | } 64 | 65 | const result = builder.build(schema) 66 | 67 | expect(result).toContain('* process') 68 | expect(result).toContain('SERIAL') 69 | }) 70 | 71 | it('uses asterisks and dashes for formatting', () => { 72 | const schema: FlowExplicitDescription = { 73 | process: [['boxA']] 74 | } 75 | 76 | const result = builder.build(schema) 77 | 78 | // Check formatting characters are used 79 | expect(result).toMatch(/\*+/) 80 | expect(result).toMatch(/-+/) 81 | }) 82 | 83 | it('handles deeply nested generators', () => { 84 | const schema: FlowExplicitDescription = { 85 | process: [ 86 | [ 87 | { 88 | outerGen: [ 89 | [ 90 | { 91 | innerGen: [['deepBox']] 92 | } 93 | ] 94 | ] 95 | } 96 | ] 97 | ] 98 | } 99 | 100 | const result = builder.build(schema) 101 | 102 | expect(result).toContain('* outerGen') 103 | expect(result).toContain('* innerGen') 104 | expect(result).toContain('- deepBox') 105 | }) 106 | }) 107 | }) 108 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/boxCore/boxFactory.ts: -------------------------------------------------------------------------------- 1 | import type { BoxMeta, BatchingBoxMeta } from '../BoxI' 2 | import type { Message, MessageData } from '../Message' 3 | import type { PriorityQueueI } from '../queue/PriorityQueueI' 4 | import type { ServiceProvider } from '../ServiceProvider' 5 | import type { 6 | BoxExecutiveDefinition, 7 | BoxExecutiveBatchDefinition, 8 | BoxFactorySignature, 9 | BatchingBoxFactorySignature 10 | } from './types' 11 | import { Box } from './Box' 12 | import { BatchingBox } from './BatchingBox' 13 | 14 | /** 15 | * Creates a single-message processing box factory. 16 | * 17 | * @param metadata - Information about intended operation of the code 18 | * @param processValueDef - The code of your box 19 | * @returns A box model class 20 | * @internalapi 21 | */ 22 | function boxSingleFactory( 23 | metadata: BoxMeta, 24 | processValueDef: BoxExecutiveDefinition 25 | ): BoxFactorySignature { 26 | return class extends Box { 27 | public constructor( 28 | providedName: string, 29 | serviceProvider: ServiceProvider, 30 | q?: PriorityQueueI, 31 | parameters?: any 32 | ) { 33 | super(providedName, metadata, serviceProvider, q, parameters) 34 | } 35 | protected processValue( 36 | msg: MessageData, 37 | emit: (msgs: MessageData[], priority?: number) => void 38 | ): ReturnType { 39 | return processValueDef(this.serviceParamsProvider, msg, emit) 40 | } 41 | } 42 | } 43 | 44 | /** 45 | * Creates a batch-processing box factory. 46 | * 47 | * @param metadata - Information about intended operation of the code 48 | * @param processValueDef - The code of your box 49 | * @returns A batching box model class 50 | * @internalapi 51 | */ 52 | function boxBatchingFactory( 53 | metadata: BatchingBoxMeta, 54 | processValueDef: BoxExecutiveBatchDefinition 55 | ): BatchingBoxFactorySignature { 56 | return class extends BatchingBox { 57 | public constructor( 58 | providedName: string, 59 | serviceProvider: ServiceProvider, 60 | q?: PriorityQueueI, 61 | parameters?: any 62 | ) { 63 | super(providedName, metadata, serviceProvider, q, parameters) 64 | } 65 | protected processValue(msgs: MessageData[]): ReturnType { 66 | return processValueDef(this.serviceParamsProvider, msgs) 67 | } 68 | } 69 | } 70 | 71 | /** 72 | * A basic mean of creating your own boxes. 73 | * 74 | * Each box has to be in its own file, the filename being the box's identificator (name). 75 | * The file is a JS (TS) module that `exports default` the return value of `boxFactory`. 76 | * 77 | * @param metadata - Information about intended operation of the code 78 | * @param processValueDef - The code of your box 79 | * @returns A box model. It must be the *default export* of the module. 80 | * @publicapi 81 | */ 82 | export function boxFactory( 83 | metadata: BoxMeta | BatchingBoxMeta, 84 | processValueDef: BoxExecutiveDefinition | BoxExecutiveBatchDefinition 85 | ): BoxFactorySignature | BatchingBoxFactorySignature { 86 | if ((metadata as BatchingBoxMeta).batch) { 87 | return boxBatchingFactory( 88 | metadata as BatchingBoxMeta, 89 | processValueDef as BoxExecutiveBatchDefinition 90 | ) 91 | } else { 92 | return boxSingleFactory(metadata as BoxMeta, processValueDef as BoxExecutiveDefinition) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /benchmarks/output/resultWriter.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Result writing utilities for benchmark runner 3 | */ 4 | 5 | import * as path from 'path' 6 | import * as fs from 'fs' 7 | import { BenchmarkConfig, BenchmarkResult } from '../types' 8 | import { ProgressState } from '../progress' 9 | import { getGitCommitSha } from '../utils/git' 10 | 11 | /** Directory for storing benchmark results */ 12 | const RESULTS_DIR = path.join(__dirname, '..', 'results') 13 | 14 | /** 15 | * Ensure the results directory exists 16 | */ 17 | function ensureResultsDir(): void { 18 | if (!fs.existsSync(RESULTS_DIR)) { 19 | fs.mkdirSync(RESULTS_DIR, { recursive: true }) 20 | } 21 | } 22 | 23 | /** 24 | * Save benchmark results to a JSON file 25 | */ 26 | export function saveResults(results: BenchmarkResult[]): string { 27 | ensureResultsDir() 28 | 29 | const timestamp = new Date().toISOString().replace(/[:.]/g, '-') 30 | const filename = path.join(RESULTS_DIR, `benchmark-${timestamp}.json`) 31 | 32 | fs.writeFileSync(filename, JSON.stringify(results, null, 2)) 33 | console.log(`\nResults saved to: ${filename}`) 34 | 35 | return filename 36 | } 37 | 38 | /** Partial result structure for incomplete runs */ 39 | export interface PartialResult { 40 | status: 'incomplete' 41 | reason: string 42 | flowType: string 43 | config: BenchmarkConfig 44 | elapsedMs: number 45 | sentEventCount: number 46 | messagesProcessed: number 47 | expectedMessages: number 48 | avgTimePerMessageMs: number | null 49 | messagesPerSecond: number 50 | memoryMB: number 51 | peakMemoryMB: number 52 | timestamp: string 53 | nodeVersion: string 54 | gitCommitSha: string | undefined 55 | environment: { 56 | BAKERYJS_DISABLE_EXPERIMENTAL_TRACING?: string 57 | } 58 | } 59 | 60 | /** 61 | * Save partial results for incomplete benchmark runs 62 | */ 63 | export function savePartialResults( 64 | flowType: string, 65 | config: BenchmarkConfig, 66 | state: ProgressState, 67 | reason: string 68 | ): string { 69 | const elapsed = Date.now() - state.startTime 70 | const mem = process.memoryUsage() 71 | 72 | const partialResult: PartialResult = { 73 | status: 'incomplete', 74 | reason, 75 | flowType, 76 | config, 77 | elapsedMs: elapsed, 78 | sentEventCount: state.sentCount, 79 | messagesProcessed: state.drainedCount, 80 | expectedMessages: state.expectedMessages, 81 | avgTimePerMessageMs: state.drainedCount > 0 ? elapsed / state.drainedCount : null, 82 | messagesPerSecond: state.drainedCount > 0 ? state.drainedCount / (elapsed / 1000) : 0, 83 | memoryMB: mem.heapUsed / 1024 / 1024, 84 | peakMemoryMB: state.peakMemory / 1024 / 1024, 85 | timestamp: new Date().toISOString(), 86 | nodeVersion: process.version, 87 | gitCommitSha: getGitCommitSha(), 88 | environment: { 89 | BAKERYJS_DISABLE_EXPERIMENTAL_TRACING: process.env.BAKERYJS_DISABLE_EXPERIMENTAL_TRACING 90 | } 91 | } 92 | 93 | ensureResultsDir() 94 | 95 | const filename = `partial-${flowType}-${config.itemCount}x${ 96 | config.nestedItemCount || 1 97 | }-${Date.now()}.json` 98 | const filepath = path.join(RESULTS_DIR, filename) 99 | fs.writeFileSync(filepath, JSON.stringify(partialResult, null, 2)) 100 | console.log(`\nPartial results saved to: ${filepath}`) 101 | 102 | return filepath 103 | } 104 | -------------------------------------------------------------------------------- /benchmarks/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Types for the BakeryJS benchmarking system 3 | */ 4 | 5 | export interface BenchmarkConfig { 6 | /** Number of items for the first generator */ 7 | itemCount: number 8 | /** Number of items for nested generator (complex flow only) */ 9 | nestedItemCount?: number 10 | /** Artificial delay in ms for mapper boxes (0 = no delay) */ 11 | mapperDelayMs?: number 12 | /** Number of benchmark runs to average */ 13 | runs?: number 14 | /** Whether to output verbose logs */ 15 | verbose?: boolean 16 | } 17 | 18 | export interface BenchmarkMetrics { 19 | /** Total execution time in milliseconds */ 20 | totalTimeMs: number 21 | /** Total messages processed through the drain */ 22 | messagesProcessed: number 23 | /** Average time per message in milliseconds */ 24 | avgTimePerMessageMs: number 25 | /** Memory used at end of run in MB */ 26 | memoryUsedMB: number 27 | /** Peak memory during run in MB */ 28 | peakMemoryMB: number 29 | /** Heap statistics */ 30 | heapStats?: { 31 | heapUsed: number 32 | heapTotal: number 33 | external: number 34 | } 35 | } 36 | 37 | export interface EventTimings { 38 | /** Time of first 'sent' event from run start */ 39 | firstSentMs: number 40 | /** Time of last 'sent' event from run start */ 41 | lastSentMs: number 42 | /** Time when drain completed from run start */ 43 | drainCompleteMs: number 44 | /** Total number of 'sent' events */ 45 | sentEventCount: number 46 | /** Event timeline for detailed analysis */ 47 | timeline?: EventTimelineEntry[] 48 | } 49 | 50 | export interface EventTimelineEntry { 51 | /** Timestamp relative to run start */ 52 | timestampMs: number 53 | /** Event type */ 54 | event: 'sent' | 'run' | 'drain' 55 | /** Source box */ 56 | source?: string 57 | /** Target box */ 58 | target?: string 59 | /** Batch size */ 60 | batchSize?: number 61 | } 62 | 63 | export interface BenchmarkResult { 64 | /** Name of the benchmark */ 65 | name: string 66 | /** Type of flow */ 67 | flowType: 'simple' | 'complex' 68 | /** Configuration used */ 69 | config: BenchmarkConfig 70 | /** Performance metrics */ 71 | metrics: BenchmarkMetrics 72 | /** Event timing information */ 73 | eventTimings: EventTimings 74 | /** Timestamp when benchmark was run */ 75 | timestamp: string 76 | /** Node.js version */ 77 | nodeVersion: string 78 | /** Git commit SHA of the repository */ 79 | gitCommitSha?: string 80 | /** Environment flags */ 81 | environment: { 82 | BAKERYJS_DISABLE_EXPERIMENTAL_TRACING?: string 83 | } 84 | } 85 | 86 | export interface BenchmarkSummary { 87 | /** Results for each configuration */ 88 | results: BenchmarkResult[] 89 | /** Comparison analysis */ 90 | analysis?: { 91 | /** Scaling factor (time increase per 10x messages) */ 92 | scalingFactor: number 93 | /** Whether scaling appears linear */ 94 | isLinearScaling: boolean 95 | /** Simple vs complex flow overhead ratio */ 96 | complexityOverheadRatio: number 97 | } 98 | } 99 | 100 | /** 101 | * Message data types for benchmark boxes 102 | */ 103 | export interface BenchmarkGeneratorInput { 104 | itemCount?: number 105 | nestedItemCount?: number 106 | } 107 | 108 | export interface BenchmarkGeneratorOutput { 109 | item: number 110 | generatorId: string 111 | timestamp: number 112 | } 113 | 114 | export interface NestedGeneratorOutput { 115 | item: number 116 | parentItem: number 117 | generatorId: string 118 | timestamp: number 119 | } 120 | 121 | export interface MapperOutput { 122 | processed: boolean 123 | processorId: string 124 | processedAt: number 125 | } 126 | -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T09-21-40-812Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-10x5", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 10, 7 | "nestedItemCount": 5 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 40, 11 | "messagesProcessed": 50, 12 | "avgTimePerMessageMs": 0.8, 13 | "memoryUsedMB": 3.6990432739257812, 14 | "peakMemoryMB": 72.7908935546875, 15 | "heapStats": { 16 | "heapUsed": 62464752, 17 | "heapTotal": 115965952, 18 | "external": 2787654 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 23, 23 | "lastSentMs": 39, 24 | "drainCompleteMs": 40, 25 | "sentEventCount": 504 26 | }, 27 | "timestamp": "2025-12-14T09:21:40.498Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "0ec047f1512e2cd20969527bbd8ea6aeee45c0fc", 30 | "environment": {} 31 | }, 32 | { 33 | "name": "complex-10x10", 34 | "flowType": "complex", 35 | "config": { 36 | "itemCount": 10, 37 | "nestedItemCount": 10 38 | }, 39 | "metrics": { 40 | "totalTimeMs": 27, 41 | "messagesProcessed": 100, 42 | "avgTimePerMessageMs": 0.27, 43 | "memoryUsedMB": 3.0790634155273438, 44 | "peakMemoryMB": 61.12340545654297, 45 | "heapStats": { 46 | "heapUsed": 69878728, 47 | "heapTotal": 125140992, 48 | "external": 2787695 49 | } 50 | }, 51 | "eventTimings": { 52 | "firstSentMs": 9, 53 | "lastSentMs": 26, 54 | "drainCompleteMs": 27, 55 | "sentEventCount": 954 56 | }, 57 | "timestamp": "2025-12-14T09:21:40.562Z", 58 | "nodeVersion": "v24.12.0", 59 | "gitCommitSha": "0ec047f1512e2cd20969527bbd8ea6aeee45c0fc", 60 | "environment": {} 61 | }, 62 | { 63 | "name": "complex-50x10", 64 | "flowType": "complex", 65 | "config": { 66 | "itemCount": 50, 67 | "nestedItemCount": 10 68 | }, 69 | "metrics": { 70 | "totalTimeMs": 54, 71 | "messagesProcessed": 500, 72 | "avgTimePerMessageMs": 0.108, 73 | "memoryUsedMB": -4.2136993408203125, 74 | "peakMemoryMB": 73.78018951416016, 75 | "heapStats": { 76 | "heapUsed": 69547432, 77 | "heapTotal": 128024576, 78 | "external": 2787654 79 | } 80 | }, 81 | "eventTimings": { 82 | "firstSentMs": 8, 83 | "lastSentMs": 53, 84 | "drainCompleteMs": 54, 85 | "sentEventCount": 4754 86 | }, 87 | "timestamp": "2025-12-14T09:21:40.659Z", 88 | "nodeVersion": "v24.12.0", 89 | "gitCommitSha": "0ec047f1512e2cd20969527bbd8ea6aeee45c0fc", 90 | "environment": {} 91 | }, 92 | { 93 | "name": "complex-100x10", 94 | "flowType": "complex", 95 | "config": { 96 | "itemCount": 100, 97 | "nestedItemCount": 10 98 | }, 99 | "metrics": { 100 | "totalTimeMs": 85, 101 | "messagesProcessed": 1000, 102 | "avgTimePerMessageMs": 0.085, 103 | "memoryUsedMB": -5.9658050537109375, 104 | "peakMemoryMB": 84.95553588867188, 105 | "heapStats": { 106 | "heapUsed": 67831256, 107 | "heapTotal": 133791744, 108 | "external": 2787654 109 | } 110 | }, 111 | "eventTimings": { 112 | "firstSentMs": 6, 113 | "lastSentMs": 83, 114 | "drainCompleteMs": 85, 115 | "sentEventCount": 9504 116 | }, 117 | "timestamp": "2025-12-14T09:21:40.781Z", 118 | "nodeVersion": "v24.12.0", 119 | "gitCommitSha": "0ec047f1512e2cd20969527bbd8ea6aeee45c0fc", 120 | "environment": {} 121 | } 122 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-14T10-48-38-374Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-10x5", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 10, 7 | "nestedItemCount": 5 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 154, 11 | "messagesProcessed": 50, 12 | "avgTimePerMessageMs": 3.08, 13 | "memoryUsedMB": 28.163230895996094, 14 | "peakMemoryMB": 177.41468048095703, 15 | "heapStats": { 16 | "heapUsed": 188639040, 17 | "heapTotal": 289947648, 18 | "external": 7911442 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 140, 23 | "lastSentMs": 154, 24 | "drainCompleteMs": 154, 25 | "sentEventCount": 504 26 | }, 27 | "timestamp": "2025-12-14T10:48:38.031Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "24a8fa4bdfd60d8daf6ecae461f90c89ddf52015", 30 | "environment": {} 31 | }, 32 | { 33 | "name": "complex-10x10", 34 | "flowType": "complex", 35 | "config": { 36 | "itemCount": 10, 37 | "nestedItemCount": 10 38 | }, 39 | "metrics": { 40 | "totalTimeMs": 30, 41 | "messagesProcessed": 100, 42 | "avgTimePerMessageMs": 0.3, 43 | "memoryUsedMB": -14.120170593261719, 44 | "peakMemoryMB": 193.5002670288086, 45 | "heapStats": { 46 | "heapUsed": 178022448, 47 | "heapTotal": 303579136, 48 | "external": 7911483 49 | } 50 | }, 51 | "eventTimings": { 52 | "firstSentMs": 10, 53 | "lastSentMs": 29, 54 | "drainCompleteMs": 30, 55 | "sentEventCount": 954 56 | }, 57 | "timestamp": "2025-12-14T10:48:38.115Z", 58 | "nodeVersion": "v24.12.0", 59 | "gitCommitSha": "24a8fa4bdfd60d8daf6ecae461f90c89ddf52015", 60 | "environment": {} 61 | }, 62 | { 63 | "name": "complex-50x10", 64 | "flowType": "complex", 65 | "config": { 66 | "itemCount": 50, 67 | "nestedItemCount": 10 68 | }, 69 | "metrics": { 70 | "totalTimeMs": 55, 71 | "messagesProcessed": 500, 72 | "avgTimePerMessageMs": 0.11, 73 | "memoryUsedMB": 22.832542419433594, 74 | "peakMemoryMB": 201.73692321777344, 75 | "heapStats": { 76 | "heapUsed": 206067592, 77 | "heapTotal": 307773440, 78 | "external": 7903291 79 | } 80 | }, 81 | "eventTimings": { 82 | "firstSentMs": 9, 83 | "lastSentMs": 53, 84 | "drainCompleteMs": 55, 85 | "sentEventCount": 4754 86 | }, 87 | "timestamp": "2025-12-14T10:48:38.208Z", 88 | "nodeVersion": "v24.12.0", 89 | "gitCommitSha": "24a8fa4bdfd60d8daf6ecae461f90c89ddf52015", 90 | "environment": {} 91 | }, 92 | { 93 | "name": "complex-100x10", 94 | "flowType": "complex", 95 | "config": { 96 | "itemCount": 100, 97 | "nestedItemCount": 10 98 | }, 99 | "metrics": { 100 | "totalTimeMs": 90, 101 | "messagesProcessed": 1000, 102 | "avgTimePerMessageMs": 0.09, 103 | "memoryUsedMB": 17.13011932373047, 104 | "peakMemoryMB": 217.6428451538086, 105 | "heapStats": { 106 | "heapUsed": 228386824, 107 | "heapTotal": 311181312, 108 | "external": 7903250 109 | } 110 | }, 111 | "eventTimings": { 112 | "firstSentMs": 7, 113 | "lastSentMs": 89, 114 | "drainCompleteMs": 90, 115 | "sentEventCount": 9504 116 | }, 117 | "timestamp": "2025-12-14T10:48:38.337Z", 118 | "nodeVersion": "v24.12.0", 119 | "gitCommitSha": "24a8fa4bdfd60d8daf6ecae461f90c89ddf52015", 120 | "environment": {} 121 | } 122 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T06-30-50-933Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-10x5", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 10, 7 | "nestedItemCount": 5 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 140, 11 | "messagesProcessed": 50, 12 | "avgTimePerMessageMs": 2.8, 13 | "memoryUsedMB": 18.539833068847656, 14 | "peakMemoryMB": 176.68624114990234, 15 | "heapStats": { 16 | "heapUsed": 185267992, 17 | "heapTotal": 289423360, 18 | "external": 7911980 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 136, 23 | "lastSentMs": 140, 24 | "drainCompleteMs": 140, 25 | "sentEventCount": 504 26 | }, 27 | "timestamp": "2025-12-15T06:30:50.732Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 30 | "environment": {} 31 | }, 32 | { 33 | "name": "complex-10x10", 34 | "flowType": "complex", 35 | "config": { 36 | "itemCount": 10, 37 | "nestedItemCount": 10 38 | }, 39 | "metrics": { 40 | "totalTimeMs": 13, 41 | "messagesProcessed": 100, 42 | "avgTimePerMessageMs": 0.13, 43 | "memoryUsedMB": 15.315299987792969, 44 | "peakMemoryMB": 195.99929809570312, 45 | "heapStats": { 46 | "heapUsed": 205519752, 47 | "heapTotal": 289947648, 48 | "external": 7912021 49 | } 50 | }, 51 | "eventTimings": { 52 | "firstSentMs": 10, 53 | "lastSentMs": 13, 54 | "drainCompleteMs": 13, 55 | "sentEventCount": 954 56 | }, 57 | "timestamp": "2025-12-15T06:30:50.784Z", 58 | "nodeVersion": "v24.12.0", 59 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 60 | "environment": {} 61 | }, 62 | { 63 | "name": "complex-50x10", 64 | "flowType": "complex", 65 | "config": { 66 | "itemCount": 50, 67 | "nestedItemCount": 10 68 | }, 69 | "metrics": { 70 | "totalTimeMs": 21, 71 | "messagesProcessed": 500, 72 | "avgTimePerMessageMs": 0.042, 73 | "memoryUsedMB": -15.680282592773438, 74 | "peakMemoryMB": 199.9048309326172, 75 | "heapStats": { 76 | "heapUsed": 193175920, 77 | "heapTotal": 298336256, 78 | "external": 7912062 79 | } 80 | }, 81 | "eventTimings": { 82 | "firstSentMs": 8, 83 | "lastSentMs": 21, 84 | "drainCompleteMs": 21, 85 | "sentEventCount": 4754 86 | }, 87 | "timestamp": "2025-12-15T06:30:50.836Z", 88 | "nodeVersion": "v24.12.0", 89 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 90 | "environment": {} 91 | }, 92 | { 93 | "name": "complex-100x10", 94 | "flowType": "complex", 95 | "config": { 96 | "itemCount": 100, 97 | "nestedItemCount": 10 98 | }, 99 | "metrics": { 100 | "totalTimeMs": 28, 101 | "messagesProcessed": 1000, 102 | "avgTimePerMessageMs": 0.028, 103 | "memoryUsedMB": -0.8545379638671875, 104 | "peakMemoryMB": 188.338623046875, 105 | "heapStats": { 106 | "heapUsed": 196847144, 107 | "heapTotal": 303316992, 108 | "external": 7903829 109 | } 110 | }, 111 | "eventTimings": { 112 | "firstSentMs": 7, 113 | "lastSentMs": 28, 114 | "drainCompleteMs": 28, 115 | "sentEventCount": 9504 116 | }, 117 | "timestamp": "2025-12-15T06:30:50.899Z", 118 | "nodeVersion": "v24.12.0", 119 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 120 | "environment": {} 121 | } 122 | ] -------------------------------------------------------------------------------- /benchmarks/results/benchmark-2025-12-15T06-31-06-110Z.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "complex-10x5", 4 | "flowType": "complex", 5 | "config": { 6 | "itemCount": 10, 7 | "nestedItemCount": 5 8 | }, 9 | "metrics": { 10 | "totalTimeMs": 136, 11 | "messagesProcessed": 50, 12 | "avgTimePerMessageMs": 2.72, 13 | "memoryUsedMB": 18.65538787841797, 14 | "peakMemoryMB": 176.94794464111328, 15 | "heapStats": { 16 | "heapUsed": 185542408, 17 | "heapTotal": 289161216, 18 | "external": 7912028 19 | } 20 | }, 21 | "eventTimings": { 22 | "firstSentMs": 132, 23 | "lastSentMs": 136, 24 | "drainCompleteMs": 136, 25 | "sentEventCount": 504 26 | }, 27 | "timestamp": "2025-12-15T06:31:05.914Z", 28 | "nodeVersion": "v24.12.0", 29 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 30 | "environment": {} 31 | }, 32 | { 33 | "name": "complex-10x10", 34 | "flowType": "complex", 35 | "config": { 36 | "itemCount": 10, 37 | "nestedItemCount": 10 38 | }, 39 | "metrics": { 40 | "totalTimeMs": 12, 41 | "messagesProcessed": 100, 42 | "avgTimePerMessageMs": 0.12, 43 | "memoryUsedMB": 15.320381164550781, 44 | "peakMemoryMB": 196.26443481445312, 45 | "heapStats": { 46 | "heapUsed": 205797768, 47 | "heapTotal": 289947648, 48 | "external": 7912069 49 | } 50 | }, 51 | "eventTimings": { 52 | "firstSentMs": 8, 53 | "lastSentMs": 11, 54 | "drainCompleteMs": 12, 55 | "sentEventCount": 954 56 | }, 57 | "timestamp": "2025-12-15T06:31:05.962Z", 58 | "nodeVersion": "v24.12.0", 59 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 60 | "environment": {} 61 | }, 62 | { 63 | "name": "complex-50x10", 64 | "flowType": "complex", 65 | "config": { 66 | "itemCount": 50, 67 | "nestedItemCount": 10 68 | }, 69 | "metrics": { 70 | "totalTimeMs": 20, 71 | "messagesProcessed": 500, 72 | "avgTimePerMessageMs": 0.04, 73 | "memoryUsedMB": -15.568092346191406, 74 | "peakMemoryMB": 200.1802520751953, 75 | "heapStats": { 76 | "heapUsed": 193582360, 77 | "heapTotal": 298336256, 78 | "external": 7912110 79 | } 80 | }, 81 | "eventTimings": { 82 | "firstSentMs": 8, 83 | "lastSentMs": 20, 84 | "drainCompleteMs": 20, 85 | "sentEventCount": 4754 86 | }, 87 | "timestamp": "2025-12-15T06:31:06.018Z", 88 | "nodeVersion": "v24.12.0", 89 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 90 | "environment": {} 91 | }, 92 | { 93 | "name": "complex-100x10", 94 | "flowType": "complex", 95 | "config": { 96 | "itemCount": 100, 97 | "nestedItemCount": 10 98 | }, 99 | "metrics": { 100 | "totalTimeMs": 29, 101 | "messagesProcessed": 1000, 102 | "avgTimePerMessageMs": 0.029, 103 | "memoryUsedMB": -0.3310089111328125, 104 | "peakMemoryMB": 189.00605010986328, 105 | "heapStats": { 106 | "heapUsed": 197366400, 107 | "heapTotal": 302268416, 108 | "external": 7903877 109 | } 110 | }, 111 | "eventTimings": { 112 | "firstSentMs": 7, 113 | "lastSentMs": 29, 114 | "drainCompleteMs": 29, 115 | "sentEventCount": 9504 116 | }, 117 | "timestamp": "2025-12-15T06:31:06.088Z", 118 | "nodeVersion": "v24.12.0", 119 | "gitCommitSha": "34d7f5a882c727476ef18caf2736d6d65b93452a", 120 | "environment": {} 121 | } 122 | ] -------------------------------------------------------------------------------- /profiling/analyzer/thresholds.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Configurable Threshold Constants for Profile Analysis 3 | * 4 | * These thresholds control what is considered a hotspot, bottleneck, or regression. 5 | */ 6 | 7 | /** Default thresholds for analysis */ 8 | export interface AnalysisThresholds { 9 | /** Minimum self-time percentage to flag as a hotspot (default: 5%) */ 10 | hotspotSelfTimePercent: number 11 | 12 | /** Minimum total-time percentage to flag as a bottleneck (default: 10%) */ 13 | bottleneckTotalTimePercent: number 14 | 15 | /** Minimum percentage increase to flag as a regression (default: 10%) */ 16 | regressionChangePercent: number 17 | 18 | /** Number of top functions to include in reports (default: 10) */ 19 | topN: number 20 | 21 | /** Minimum hit count to consider a function significant */ 22 | minHitCount: number 23 | } 24 | 25 | /** Default analysis thresholds */ 26 | export const DEFAULT_THRESHOLDS: AnalysisThresholds = { 27 | hotspotSelfTimePercent: 5.0, 28 | bottleneckTotalTimePercent: 10.0, 29 | regressionChangePercent: 10.0, 30 | topN: 10, 31 | minHitCount: 10 32 | } 33 | 34 | /** 35 | * Create thresholds from partial overrides 36 | */ 37 | export function createThresholds(overrides: Partial = {}): AnalysisThresholds { 38 | return { 39 | ...DEFAULT_THRESHOLDS, 40 | ...overrides 41 | } 42 | } 43 | 44 | /** 45 | * Source file patterns for filtering by code category 46 | */ 47 | export const SOURCE_PATTERNS = { 48 | /** BakeryJS core source files */ 49 | bakeryJs: ['/src/lib/bakeryjs/', '/src/'], 50 | 51 | /** TracingModel specifically */ 52 | tracingModel: ['TracingModel.ts'], 53 | 54 | /** Flow-related code */ 55 | flow: ['Flow.ts', 'FlowBuilder'], 56 | 57 | /** Box-related code */ 58 | box: ['Box.ts', 'BoxFactory', 'Box/'], 59 | 60 | /** Benchmark-specific code (not core BakeryJS) */ 61 | benchmark: ['/benchmarks/'], 62 | 63 | /** Node.js internal patterns */ 64 | nodeInternals: ['node:', 'internal/', 'v8/', 'native '], 65 | 66 | /** Third-party libraries */ 67 | nodeModules: ['node_modules/'] 68 | } 69 | 70 | /** 71 | * Categorize a file path into a code category 72 | */ 73 | export function categorizeFile(url: string): string { 74 | if (!url) { 75 | return 'native' 76 | } 77 | 78 | // Check TracingModel first (most specific) 79 | if (SOURCE_PATTERNS.tracingModel.some(p => url.includes(p))) { 80 | return 'TracingModel' 81 | } 82 | 83 | // Check Flow 84 | if (SOURCE_PATTERNS.flow.some(p => url.includes(p))) { 85 | return 'Flow' 86 | } 87 | 88 | // Check Box 89 | if (SOURCE_PATTERNS.box.some(p => url.includes(p))) { 90 | return 'Box' 91 | } 92 | 93 | // Check other BakeryJS code 94 | if (SOURCE_PATTERNS.bakeryJs.some(p => url.includes(p))) { 95 | return 'Other BakeryJS' 96 | } 97 | 98 | // Check benchmark code 99 | if (SOURCE_PATTERNS.benchmark.some(p => url.includes(p))) { 100 | return 'Benchmark' 101 | } 102 | 103 | // Check node_modules 104 | if (SOURCE_PATTERNS.nodeModules.some(p => url.includes(p))) { 105 | return 'Dependencies' 106 | } 107 | 108 | // Check Node.js internals 109 | if (SOURCE_PATTERNS.nodeInternals.some(p => url.includes(p))) { 110 | return 'Node.js Internals' 111 | } 112 | 113 | return 'Other' 114 | } 115 | 116 | /** 117 | * Check if a file is part of BakeryJS core (not benchmarks or dependencies) 118 | */ 119 | export function isBakeryJsCore(url: string): boolean { 120 | if (!url) return false 121 | 122 | // Must be in src/ but not in benchmarks 123 | return ( 124 | SOURCE_PATTERNS.bakeryJs.some(p => url.includes(p)) && 125 | !SOURCE_PATTERNS.benchmark.some(p => url.includes(p)) 126 | ) 127 | } 128 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/ComponentFactory.ts: -------------------------------------------------------------------------------- 1 | import { VError } from 'verror' 2 | import type { BatchingBoxInterface, BoxInterface } from './BoxI' 3 | import type ComponentFactoryI from './ComponentFactoryI' 4 | import type { PriorityQueueI } from './queue/PriorityQueueI' 5 | import type { Message } from './Message' 6 | import type { ServiceProvider } from './ServiceProvider' 7 | import { scanComponentsPath } from './scanComponentsPath' 8 | import Debug from 'debug' 9 | 10 | const debug = Debug('bakeryjs:componentProvider') 11 | 12 | function boxNotFoundError(name: string, baseURIs: string | string[]): Error { 13 | const joinedUris = typeof baseURIs == 'string' ? baseURIs : baseURIs.join(',') 14 | return new VError( 15 | { 16 | name: 'BoxNotFound', 17 | info: { 18 | requestedBoxName: name, 19 | factoryBaseUri: baseURIs 20 | } 21 | }, 22 | "Box '%s' not found in %s.", 23 | name, 24 | joinedUris 25 | ) 26 | } 27 | 28 | export class ComponentFactory implements ComponentFactoryI { 29 | private availableComponents: { [s: string]: string } = {} 30 | private readonly serviceProvider: ServiceProvider 31 | public readonly baseURI: string 32 | 33 | public constructor(componentsPath: string, serviceProvider: ServiceProvider) { 34 | this.baseURI = `file://${componentsPath}` 35 | this.availableComponents = scanComponentsPath(componentsPath) 36 | debug(this.availableComponents) 37 | this.serviceProvider = serviceProvider 38 | } 39 | 40 | public async create( 41 | name: string, 42 | queue?: PriorityQueueI, 43 | parameters?: any 44 | ): Promise { 45 | if (!this.availableComponents[name]) { 46 | throw boxNotFoundError(name, this.baseURI) 47 | } 48 | try { 49 | // TODO: (code detail) Is it necessary to always import the file? 50 | const box = await import(this.availableComponents[name]) 51 | return new box.default(name, this.serviceProvider, queue, parameters) as 52 | | BoxInterface 53 | | BatchingBoxInterface 54 | } catch (error) { 55 | throw new VError( 56 | { 57 | name: 'ComponentLoadError', 58 | cause: error instanceof Error ? error : new Error(String(error)), 59 | info: { 60 | componentName: name 61 | } 62 | }, 63 | 'Error loading component %s', 64 | name 65 | ) 66 | } 67 | } 68 | } 69 | 70 | export class MultiComponentFactory implements ComponentFactoryI { 71 | protected readonly factories: ComponentFactory[] 72 | public constructor() { 73 | this.factories = [] 74 | } 75 | 76 | public push(factory: ComponentFactory): void { 77 | this.factories.unshift(factory) 78 | } 79 | 80 | public async create( 81 | name: string, 82 | queue?: PriorityQueueI, 83 | parameters?: any 84 | ): Promise { 85 | const futureBoxes = this.factories.map(async factory => { 86 | try { 87 | return await factory.create(name, queue, parameters) 88 | } catch (reason) { 89 | const error = reason instanceof Error ? reason : new Error(String(reason)) 90 | if (VError.hasCauseWithName(error, 'BoxNotFound')) { 91 | return 92 | } 93 | 94 | throw new VError( 95 | { 96 | name: 'FactoryException', 97 | message: 'ComponentFactory.create(%s) failed.', 98 | info: { 99 | factoryBaseURI: factory.baseURI, 100 | requestedBoxName: name 101 | }, 102 | cause: error 103 | }, 104 | name 105 | ) 106 | } 107 | }) 108 | 109 | const resolvedBoxes = await Promise.all(futureBoxes) 110 | const result = resolvedBoxes.find((resp: any) => resp !== undefined) 111 | if (result) { 112 | return result 113 | } 114 | 115 | throw boxNotFoundError( 116 | name, 117 | this.factories.map(f => f.baseURI) 118 | ) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # BakeryJS Example Project 2 | 3 | This is a minimal example of BakeryJS project, with generator and processor components. Take a look at project structure and individual files. Especially take a look at: 4 | 5 | - [index.js](index.js) – Shows how to run the program and data flow description 6 | - [helloworld generator](components/generators/helloworld.js) 7 | - [wordcount processor](components/processors/wordcount.js) 8 | 9 | ## Project Structure 10 | 11 | - [index.js](index.js) – Entry point for the project 12 | - [components/](components/) – Root directory of components (boxes) available for flows; the components are further separated into: 13 | - [generators/](components/generators/) – Components which generate data for the flow (e.g. consume from queue or just make things up) 14 | - [processors/](components/processors/) – Components which convert, enrich and generally process data in the flow. 15 | 16 | This is a recommended structure of `components` directory, but it is up to you. Each component is added to flow based on its `name` property in metadata. 17 | 18 | ## Try It Out 19 | 20 | ``` 21 | npm install 22 | npm start 23 | ``` 24 | 25 | This will output the data flow, traces of messages as they are sent between components and messages which are "drained": 26 | 27 | ``` 28 | dispatch on flow description: 29 | building flow from SchemaObject 30 | * process ********************************************************************** 31 | * SERIAL ----------------------------------------------------------------------- 32 | * | CONCURRENT ----------------------------------------------------------------- 33 | * | - helloworld 34 | * | ---------------------------------------------------------------------------- 35 | * | CONCURRENT ----------------------------------------------------------------- 36 | * | - wordcount 37 | * | - punctcount 38 | * | ---------------------------------------------------------------------------- 39 | * | CONCURRENT ----------------------------------------------------------------- 40 | * | - checksum 41 | * | ---------------------------------------------------------------------------- 42 | * ------------------------------------------------------------------------------ 43 | ******************************************************************************** 44 | 45 | Program run -----> 46 | 10:27:32 AM Sent: _root_ --> helloworld (1) 47 | 10:27:32 AM Sent: helloworld --> punctcount (1) 48 | 10:27:32 AM Sent: helloworld --> wordcount (1) 49 | 10:27:32 AM Sent: punctcount --> checksum (1) 50 | 10:27:32 AM Sent: wordcount --> checksum (1) 51 | Drain received a message { jobId: '0', 52 | msg: 'Hello world!', 53 | punct: 3, 54 | words: 3, 55 | checksum: 7.242640687119286 } 56 | 10:27:32 AM Sent: helloworld --> punctcount (2) 57 | 10:27:32 AM Sent: helloworld --> wordcount (2) 58 | 10:27:32 AM Sent: punctcount --> checksum (1) 59 | 10:27:32 AM Sent: wordcount --> checksum (1) 60 | 10:27:32 AM Sent: punctcount --> checksum (1) 61 | 10:27:32 AM Sent: wordcount --> checksum (1) 62 | Drain received a message { jobId: '0', 63 | msg: '¡Hola mundo!', 64 | punct: 3, 65 | words: 4, 66 | checksum: 8.65685424949238 } 67 | Drain received a message { jobId: '0', 68 | msg: 'Ahoj světe!', 69 | punct: 4, 70 | words: 4, 71 | checksum: 9.65685424949238 } 72 | 10:27:33 AM Sent: helloworld --> punctcount (2) 73 | 10:27:33 AM Sent: helloworld --> wordcount (2) 74 | 10:27:33 AM Sent: punctcount --> checksum (1) 75 | 10:27:33 AM Sent: wordcount --> checksum (1) 76 | 10:27:33 AM Sent: punctcount --> checksum (1) 77 | 10:27:33 AM Sent: wordcount --> checksum (1) 78 | Drain received a message { jobId: '0', 79 | msg: 'Hallo Welt!', 80 | punct: 3, 81 | words: 3, 82 | checksum: 7.242640687119286 } 83 | Drain received a message { jobId: '0', 84 | msg: 'Bonjour monde!', 85 | punct: 3, 86 | words: 3, 87 | checksum: 7.242640687119286 } 88 | 10:27:33 AM Sent: helloworld --> punctcount (1) 89 | 10:27:33 AM Sent: helloworld --> wordcount (1) 90 | 10:27:33 AM Sent: punctcount --> checksum (1) 91 | 10:27:33 AM Sent: wordcount --> checksum (1) 92 | ``` 93 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/Message.ts: -------------------------------------------------------------------------------- 1 | import Debug from 'debug' 2 | 3 | const debug = Debug('bakeryjs:message') 4 | 5 | export type MessageData = { [key: string]: any } 6 | 7 | let messageId = 0 8 | 9 | /** 10 | * ### Identifiable message 11 | * 12 | * The Identifiable with data and their accessors and the flag that the generation is not finished. 13 | */ 14 | export interface Message { 15 | readonly id: string 16 | readonly parent: Message | undefined 17 | create(values?: MessageData): Message 18 | getInput(requires: string[]): MessageData 19 | setOutput(provides: string[], output: MessageData): void 20 | } 21 | 22 | /** 23 | * One piece of data flowing inside the Flow through Boxes. 24 | * 25 | * # Behaviour 26 | * 1. Object that is identified by an *id*. The *id* is unique at lest within the flow being currently executed. 27 | * 2. Such an object can have its parent, the object it has been generated from. 28 | * 3. A new descendant is created by calling the method *create*. 29 | * 30 | * # Intended use: 31 | * - A Message is Identifiable. 32 | * - Message holds the computed information in fields. The information in a field is *immutable* once written. 33 | * - As Message flows through Boxes, each Box adds one or more field with arbitrary information. 34 | * - The trace of the Message passage is reported by the flow executor into an API passed into Program 35 | * 36 | * ## Joints of Edges 37 | * Several clones of the Message pass through the flow "in parallel". We need to merge 38 | * these clones into a single Message, eventually. As Messages can be (possibly) prioritized, we have to perform a join 39 | * with respect to *ids*. 40 | * 41 | * ## Generating Messages in a *Generator* 42 | * A generated Message must have an (abstract) link to its *parent* for the purpose of aggregation. The *parent* *id* plays 43 | * a role of the GROUP BY expression. The trace of the Message passage is reported by the flow executor into an API passed into Program 44 | * 45 | * @internalapi 46 | */ 47 | export class DataMessage implements Message { 48 | private readonly _id: string 49 | protected data: MessageData 50 | public readonly parent: Message | undefined 51 | 52 | public constructor(initData?: MessageData, parent?: Message) { 53 | this._id = `${messageId++}` 54 | this.parent = parent 55 | this.data = initData ? initData : {} 56 | } 57 | 58 | public get id(): string { 59 | return (this.parent ? `${this.parent.id}` : '') + '/' + this._id 60 | } 61 | 62 | public create(values?: MessageData): Message { 63 | const newData = values 64 | ? Object.create(this.data, Object.getOwnPropertyDescriptors(values)) 65 | : Object.create(this.data) 66 | return new DataMessage(newData, this) 67 | } 68 | 69 | // TODO: (code detail) the flow executor should create a Data Access Object that will guard the fields and 70 | // pass the DAO into the box. The factory of the DAO could be a method of the Message. 71 | public getInput(requires: string[]): MessageData { 72 | const input: MessageData = {} 73 | for (const r of requires) { 74 | input[r] = this.data[r] 75 | } 76 | debug(`set input: ${JSON.stringify(input)}`) 77 | 78 | return input 79 | } 80 | 81 | public setOutput(provides: string[], output: MessageData): void { 82 | const currentKeys = Object.keys(this.data) 83 | const intersectionKeys = currentKeys.filter((key: string) => provides.indexOf(key) !== -1) 84 | if (intersectionKeys.length > 0) { 85 | throw new Error( 86 | `Cannot provide some data because the message already contains following results "${intersectionKeys.join( 87 | '", "' 88 | )}".` 89 | ) 90 | } 91 | 92 | debug(`set output: ${JSON.stringify(output)}`) 93 | for (const p of provides) { 94 | this.data[p] = output[p] 95 | } 96 | } 97 | 98 | public export(): MessageData { 99 | const protoChain: MessageData[] = [] 100 | let obj = this.data 101 | while (obj !== Object.prototype) { 102 | protoChain.unshift(obj) 103 | obj = Object.getPrototypeOf(obj) as MessageData 104 | } 105 | return Object.assign({}, ...protoChain) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/lib/bakeryjs/errors/BoxErrorFactory.ts: -------------------------------------------------------------------------------- 1 | import VError from 'verror' 2 | import type { MessageData } from '../Message' 3 | 4 | /** 5 | * Minimal metadata required for error reporting. 6 | */ 7 | export interface BoxMetaInfo { 8 | requires: string[] 9 | provides: string[] 10 | emits?: string[] 11 | } 12 | 13 | /** 14 | * Information about a box for error reporting. 15 | */ 16 | export interface BoxInfo { 17 | name: string 18 | meta: BoxMetaInfo 19 | } 20 | 21 | /** 22 | * Processing mode for error context. 23 | */ 24 | export type ProcessingMode = 'mapper' | 'generator' | 'aggregator' 25 | 26 | /** 27 | * Centralized factory for creating box-related errors. 28 | * Provides consistent error formatting and information across the codebase. 29 | * 30 | * @publicapi 31 | */ 32 | export class BoxErrorFactory { 33 | /** 34 | * Creates an error for when a box encounters an exception during processing. 35 | * 36 | * @param box - Information about the box that failed 37 | * @param mode - The processing mode (mapper, generator, aggregator) 38 | * @param cause - The original error that was thrown 39 | * @param value - The input value being processed when the error occurred 40 | */ 41 | public static invocationError( 42 | box: BoxInfo, 43 | mode: ProcessingMode, 44 | cause: Error, 45 | value?: MessageData 46 | ): VError { 47 | return new VError( 48 | { 49 | name: 'BoxInvocationException', 50 | cause, 51 | info: { 52 | mode, 53 | box: { name: box.name, meta: box.meta }, 54 | value 55 | } 56 | }, 57 | "The box '%s' in a %s mode encountered an exception.", 58 | box.name, 59 | mode 60 | ) 61 | } 62 | 63 | /** 64 | * Creates an error for when box parameters fail validation. 65 | * 66 | * @param schema - The JSON schema that was used for validation 67 | * @param parameters - The parameters that failed validation 68 | * @param validationErrors - The validation errors from the validator 69 | */ 70 | public static validationError( 71 | schema: object | string, 72 | parameters: unknown, 73 | validationErrors: unknown 74 | ): VError { 75 | return new VError( 76 | { 77 | name: 'BoxParametersValidationError', 78 | info: { 79 | schema, 80 | parameters, 81 | validationErrors 82 | } 83 | }, 84 | 'Box parameters must conform to the schema defined in the box. Schema: %s, parameters: %s', 85 | JSON.stringify(schema), 86 | JSON.stringify(parameters) 87 | ) 88 | } 89 | 90 | /** 91 | * Creates an error for when a generator misbehaves (e.g., emits after resolution). 92 | * 93 | * @param box - Information about the box that misbehaved 94 | * @param description - A description of the misbehavior 95 | * @param value - The input value being processed when the error occurred 96 | */ 97 | public static misbehaveError(box: BoxInfo, description: string, value?: MessageData): VError { 98 | return new VError( 99 | { 100 | name: 'GeneratorMisbehaveException', 101 | info: { 102 | mode: 'generator' as ProcessingMode, 103 | box: { name: box.name, meta: box.meta }, 104 | value, 105 | description 106 | } 107 | }, 108 | 'Generator %s emitted messages after its promise had been resolved.', 109 | box.name 110 | ) 111 | } 112 | 113 | /** 114 | * Creates an error for when a mapper tries to emit (which is not allowed). 115 | * 116 | * @param boxName - The name of the box that tried to emit 117 | */ 118 | public static inconsistentBoxError(boxName: string): VError { 119 | return new VError( 120 | { 121 | name: 'InconsistentBoxError', 122 | info: { name: boxName } 123 | }, 124 | "Box '%s': Can't invoke `emitCallback` unless being a generator/aggregator! Either set metadata filed 'emits' or 'aggregates'.", 125 | boxName 126 | ) 127 | } 128 | 129 | /** 130 | * Converts an unknown error to an Error instance. 131 | * Useful for handling non-Error throws. 132 | * 133 | * @param error - The unknown error value 134 | */ 135 | public static toError(error: unknown): Error { 136 | return error instanceof Error ? error : new Error(String(error)) 137 | } 138 | } 139 | --------------------------------------------------------------------------------