├── .eslintignore ├── .eslintrc.json ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature-request.md ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── codecov.yml │ ├── commitlint.yml │ ├── housekeeping.yml │ ├── pr-description-enforcer.yml │ ├── release-please.yml │ ├── semantic-pr.yml │ ├── tests.yml │ └── verify.yml ├── .gitignore ├── .husky ├── commit-msg └── pre-commit ├── .nvmrc ├── .prettierignore ├── .prettierrc ├── .vscode ├── launch.json └── settings.json ├── CHANGELOG.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── commitlint.config.js ├── jest.config.ts ├── package-lock.json ├── package.json ├── sonar-project.properties ├── src ├── bindings │ ├── common.test.ts │ ├── common.ts │ └── index.ts ├── common │ ├── index.ts │ ├── logger.test.ts │ ├── logger.ts │ ├── types.ts │ └── utils │ │ ├── batch.test.ts │ │ ├── batch.ts │ │ ├── common.test.ts │ │ ├── common.ts │ │ ├── index.ts │ │ └── step.ts ├── errors │ ├── batch.ts │ ├── binding_not_found.ts │ ├── index.ts │ ├── info.ts │ ├── return_result.ts │ ├── status.ts │ ├── step_creation.ts │ ├── step_execution.ts │ ├── utils.ts │ ├── workflow_creation.ts │ └── workflow_execution.ts ├── index.ts ├── steps │ ├── base │ │ ├── batch │ │ │ ├── default_batch_workflow_executor.ts │ │ │ ├── factory.ts │ │ │ ├── simple_batch_executor.ts │ │ │ └── step_executor.ts │ │ ├── custom │ │ │ ├── factory.ts │ │ │ └── step_executor.ts │ │ ├── executors │ │ │ ├── base.ts │ │ │ ├── index.ts │ │ │ └── workflow_step.ts │ │ ├── factory.test.ts │ │ ├── factory.ts │ │ ├── index.ts │ │ ├── simple │ │ │ ├── executors │ │ │ │ ├── external_workflow.ts │ │ │ │ ├── function.ts │ │ │ │ ├── identity.ts │ │ │ │ ├── index.ts │ │ │ │ └── template │ │ │ │ │ ├── factory.ts │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── jsonata.ts │ │ │ │ │ └── jsontemplate.ts │ │ │ ├── factory.ts │ │ │ └── index.ts │ │ └── utils.ts │ ├── composed │ │ ├── executors │ │ │ ├── composable.ts │ │ │ ├── conditional.ts │ │ │ ├── custom_input.ts │ │ │ ├── debuggable.ts │ │ │ ├── error_wrap.ts │ │ │ ├── index.ts │ │ │ └── loop.ts │ │ ├── factory.ts │ │ └── index.ts │ ├── factory.ts │ ├── index.ts │ └── types.ts └── workflow │ ├── default_executor.ts │ ├── engine.ts │ ├── factory.ts │ ├── index.ts │ ├── output_validator.ts │ └── utils.ts ├── stryker.conf.json ├── test ├── custom_scenarios │ └── loop_over_input │ │ └── wrapped_error │ │ ├── external_workflow.yaml │ │ └── workflow.yaml ├── e2e-custom.test.ts ├── e2e.test.ts ├── scenario.test.ts ├── scenarios │ ├── basic_workflow │ │ ├── data.json │ │ └── workflow.yaml │ ├── batch_step │ │ ├── bindings.ts │ │ ├── data.json │ │ ├── using_executor.yaml │ │ └── workflow.yaml │ ├── bindings │ │ ├── array_bindings.yaml │ │ ├── bindings_using_current_dir.yaml │ │ ├── data.json │ │ ├── execution_bindings.yaml │ │ ├── external_library.yaml │ │ ├── functions.ts │ │ ├── invalid_binding.ts │ │ ├── invalid_binding.yaml │ │ ├── non_existant_binding.yaml │ │ ├── ops.ts │ │ └── workflow.yaml │ ├── bindings_paths │ │ ├── data.json │ │ ├── functions.ts │ │ ├── ops.ts │ │ └── workflow.yaml │ ├── bindings_provider │ │ ├── bad_binding_from_provider.yaml │ │ ├── bindings.ts │ │ ├── data.ts │ │ ├── provider.ts │ │ └── workflow.yaml │ ├── common_bindings │ │ ├── assert_throw_using_custom_error.yaml │ │ ├── assert_throw_using_string.yaml │ │ ├── bindings.ts │ │ ├── data.json │ │ ├── get_by_paths.yaml │ │ ├── get_one_by_paths.yaml │ │ ├── logger.yaml │ │ ├── sha256-json-template.yaml │ │ ├── sha256.yaml │ │ ├── sum.yaml │ │ ├── to_milli_seconds.yaml │ │ └── to_seconds.yaml │ ├── compile_time_expressions │ │ ├── data.json │ │ ├── functions.ts │ │ ├── ops.ts │ │ └── workflow.yaml │ ├── conditions │ │ ├── data.json │ │ ├── else_step.yaml │ │ ├── using_context.yaml │ │ └── using_outputs.yaml │ ├── context │ │ ├── data.json │ │ ├── jsontemplate.yaml │ │ └── workflow.yaml │ ├── create │ │ └── data.ts │ ├── custom_executor │ │ ├── bad_executor.yaml │ │ ├── chained_executor.yaml │ │ ├── custom_executors.ts │ │ ├── data.ts │ │ ├── non_existing_executor.yaml │ │ └── workflow.yaml │ ├── custom_step │ │ ├── bindings.ts │ │ ├── data.json │ │ ├── executor.yaml │ │ └── provider.yaml │ ├── debug_step │ │ ├── data.json │ │ └── workflow.yaml │ ├── execute_steps │ │ ├── data.json │ │ └── workflow.yaml │ ├── exit_actions │ │ ├── data.json │ │ └── workflow.yaml │ ├── external_workflows │ │ ├── bad_workflow.yaml │ │ ├── bindings.ts │ │ ├── data.json │ │ ├── external_workflow.yaml │ │ ├── external_workflow_parent_binding.yaml │ │ ├── external_workflow_value_binding.yaml │ │ ├── external_workflow_with_errors.yaml │ │ └── workflow.yaml │ ├── input_template │ │ ├── data.json │ │ ├── jsontemplate.yaml │ │ └── workflow.yaml │ ├── invalid_workflows │ │ ├── blank_externalworkflow_step.yaml │ │ ├── blank_function_step.yaml │ │ ├── blank_template_path_step.yaml │ │ ├── blank_template_step.yaml │ │ ├── blank_workflow_step_path.yaml │ │ ├── data.json │ │ ├── duplicate_step_names.yaml │ │ ├── duplicate_step_names_in_workflowstep.yaml │ │ ├── empty_steps.yaml │ │ ├── empty_workflow.yaml │ │ ├── invalid_batch_step_executor.yaml │ │ ├── invalid_batch_step_executor_and_batches.yaml │ │ ├── invalid_batch_step_filter_output_reference.yaml │ │ ├── invalid_batch_step_map_output_reference.yaml │ │ ├── invalid_batch_step_no_executor_or_batches.yaml │ │ ├── invalid_batch_step_with_loop.yaml │ │ ├── invalid_binding.yaml │ │ ├── invalid_custom_step_executor.yaml │ │ ├── invalid_custom_step_executor_and_provider.yaml │ │ ├── invalid_custom_step_no_executor_or_provider.yaml │ │ ├── invalid_custom_step_provider.yaml │ │ ├── invalid_else_simple_output_reference.yaml │ │ ├── invalid_else_workflow_output_reference.yaml │ │ ├── invalid_function_step.yaml │ │ ├── invalid_input_template_output_references.yaml │ │ ├── invalid_loop_condition_output_references.yaml │ │ ├── invalid_loop_condition_step.yaml │ │ ├── invalid_nested_workflow_step.yaml │ │ ├── invalid_simple_mutliple_output_references.yaml │ │ ├── invalid_simple_output_reference.yaml │ │ ├── invalid_step_in_workflowstep.yaml │ │ ├── invalid_step_name_with_spaces.yaml │ │ ├── invalid_step_name_with_special_chars.yaml │ │ ├── invalid_step_nobody.yaml │ │ ├── invalid_step_noname.yaml │ │ ├── invalid_step_noname_with_workflow_name.yaml │ │ ├── invalid_template.yaml │ │ ├── invalid_usage_else_step.yaml │ │ ├── invalid_usage_nested_else_step.yaml │ │ ├── invalid_usage_oncomplete.yaml │ │ ├── invalid_workflow_inner_simple_output_reference.yaml │ │ ├── invalid_workflow_inner_workflow_output_reference.yaml │ │ ├── invalid_workflow_outer_workflow_output_reference.yaml │ │ ├── invalid_workflow_self_output_reference.yaml │ │ ├── invalid_workflowstep.yaml │ │ ├── invalid_workflowstep_from_path.yaml │ │ └── workflow_step_in_workflow_step.yaml │ ├── jsontemplate │ │ ├── data.json │ │ └── workflow.yaml │ ├── loop_over_input │ │ ├── data.ts │ │ ├── loop_condition.yaml │ │ └── workflow.yaml │ ├── mappings │ │ ├── data.ts │ │ └── workflow.yaml │ ├── multiplexing │ │ ├── data.json │ │ └── workflow.yaml │ ├── outputs │ │ ├── complex_output_reference.yaml │ │ ├── data.json │ │ └── workflow.yaml │ ├── override_bindings │ │ ├── bindings.ts │ │ ├── data.json │ │ ├── external_workflow.yaml │ │ ├── workflow.yaml │ │ └── workflow_step.yaml │ ├── return_within_template │ │ ├── data.json │ │ └── workflow.yaml │ ├── simple_steps │ │ ├── bindings.ts │ │ ├── data.json │ │ ├── divide_workflow.yaml │ │ ├── multiply.jsonata │ │ └── workflow.yaml │ ├── throw_error │ │ ├── bad_workflow.yaml │ │ ├── bindings.ts │ │ ├── data.json │ │ └── workflow.yaml │ ├── to_array │ │ ├── data.json │ │ └── workflow.yaml │ └── workflow_steps │ │ ├── data.json │ │ ├── functions.ts │ │ ├── validation_workflow_step.yaml │ │ └── workflow.yaml ├── types.ts └── utils │ ├── common.ts │ ├── index.ts │ └── scenario.ts ├── tsconfig.build.json └── tsconfig.json /.eslintignore: -------------------------------------------------------------------------------- 1 | build 2 | test 3 | coverage 4 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true 5 | }, 6 | "plugins": ["unused-imports", "@typescript-eslint"], 7 | "extends": [ 8 | "airbnb-base", 9 | "airbnb-typescript/base", 10 | "plugin:sonarjs/recommended", 11 | "plugin:@typescript-eslint/recommended", 12 | "prettier" 13 | ], 14 | "parser": "@typescript-eslint/parser", 15 | "overrides": [], 16 | "parserOptions": { 17 | "requireConfigFile": false, 18 | "ecmaVersion": 12, 19 | "sourceType": "module", 20 | "project": "./tsconfig.json" 21 | }, 22 | "ignorePatterns": ["jest.config.ts", "commitlint.config.js"], 23 | "rules": { 24 | "import/prefer-default-export": "off", 25 | "@typescript-eslint/no-explicit-any": "off", 26 | "no-restricted-syntax": "off", 27 | "import/no-cycle": "off", 28 | "no-unused-vars": "error" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '[ISSUE]' 5 | labels: bug 6 | assignees: '' 7 | --- 8 | 9 | **Describe the issue** 10 | Enter a clear and concise description of what the bug/issue is. 11 | 12 | **To Reproduce** 13 | Mention the steps to reproduce the behavior that causes the bug/issue: 14 | 15 | **Expected behavior** 16 | A clear and concise description of what you expected to happen. 17 | 18 | **Screenshots** 19 | If applicable, add screenshots to help explain your problem. 20 | 21 | **Required information (please complete the following information):** 22 | 23 | - Package version: [e.g., v0.1.1] 24 | 25 | **Additional context** 26 | Add any other context about the problem here. 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest an idea for this project 4 | title: '[ENHANCEMENT]' 5 | labels: enhancement 6 | assignees: '' 7 | --- 8 | 9 | **Is your feature request related to a problem? Please describe.** 10 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 11 | 12 | **Describe the solution you'd like** 13 | A clear and concise description of what you want to happen. 14 | 15 | **Describe alternatives you've considered** 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | **Additional context** 19 | Add any other context or screenshots about the feature request here. 20 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'github-actions' 4 | directory: '/' 5 | schedule: 6 | interval: 'weekly' 7 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## What are the changes introduced in this PR? 2 | 3 | Write a brief explainer on your code changes. 4 | 5 | ## What is the related Linear task? 6 | 7 | Resolves INT-XXX 8 | 9 | ## Please explain the objectives of your changes below 10 | 11 | Put down any required details on the broader aspect of your changes. If there are any dependent changes, **mandatorily** mention them here 12 | 13 | ### Any changes to existing capabilities/behaviour, mention the reason & what are the changes ? 14 | 15 | N/A 16 | 17 | ### Any new dependencies introduced with this change? 18 | 19 | N/A 20 | 21 | ### Any new generic utility introduced or modified. Please explain the changes. 22 | 23 | N/A 24 | 25 | ### Any technical or performance related pointers to consider with the change? 26 | 27 | N/A 28 | 29 | @coderabbitai review 30 |
31 | 32 | ### Developer checklist 33 | 34 | - [ ] My code follows the style guidelines of this project 35 | 36 | - [ ] **No breaking changes are being introduced.** 37 | 38 | - [ ] All related docs linked with the PR? 39 | 40 | - [ ] All changes manually tested? 41 | 42 | - [ ] Any documentation changes needed with this change? 43 | 44 | - [ ] Is the PR limited to 10 file changes? 45 | 46 | - [ ] Is the PR limited to one linear task? 47 | 48 | - [ ] Are relevant unit and component test-cases added? 49 | 50 | ### Reviewer checklist 51 | 52 | - [ ] Is the type of change in the PR title appropriate as per the changes? 53 | 54 | - [ ] Verified that there are no credentials or confidential data exposed with the changes. 55 | -------------------------------------------------------------------------------- /.github/workflows/codecov.yml: -------------------------------------------------------------------------------- 1 | # https://github.com/marketplace/actions/jest-coverage-report 2 | name: 'Code Coverage' 3 | on: 4 | pull_request: 5 | branches: 6 | - '*' 7 | jobs: 8 | coverage: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: ArtiomTr/jest-coverage-report-action@v2 13 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | name: Commitlint 2 | 3 | on: [push] 4 | 5 | jobs: 6 | commitlint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v4 11 | with: 12 | fetch-depth: 0 13 | 14 | - name: Setup Node 15 | uses: actions/setup-node@v4 16 | with: 17 | node-version-file: '.nvmrc' 18 | cache: 'npm' 19 | 20 | - name: Install Dependencies 21 | run: npm ci 22 | 23 | - name: Print versions 24 | run: | 25 | git --version 26 | node --version 27 | npm --version 28 | npx commitlint --version 29 | 30 | # Run the commitlint action, considering its own dependencies and yours as well 🚀 31 | # `github.workspace` is the path to your repository. 32 | - uses: wagoid/commitlint-github-action@v6 33 | env: 34 | NODE_PATH: ${{ github.workspace }}/node_modules 35 | with: 36 | commitDepth: 1 37 | -------------------------------------------------------------------------------- /.github/workflows/housekeeping.yml: -------------------------------------------------------------------------------- 1 | name: Handle stale PRs 2 | 3 | on: 4 | schedule: 5 | - cron: '42 1 * * *' 6 | 7 | jobs: 8 | prs: 9 | name: Clean up stale PRs 10 | runs-on: ubuntu-latest 11 | 12 | permissions: 13 | pull-requests: write 14 | 15 | steps: 16 | - uses: actions/stale@v9 17 | with: 18 | repo-token: ${{ secrets.GITHUB_TOKEN }} 19 | operations-per-run: 200 20 | stale-pr-message: 'This PR is considered to be stale. It has been open 20 days with no further activity thus it is going to be closed in 7 days. To avoid such a case please consider removing the stale label manually or add a comment to the PR.' 21 | days-before-pr-stale: 20 22 | days-before-pr-close: 7 23 | stale-pr-label: 'Stale' 24 | 25 | branches: 26 | name: Cleanup old branches 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v4 31 | 32 | - name: Run delete-old-branches-action 33 | uses: beatlabs/delete-old-branches-action@v0.0.9 34 | with: 35 | repo_token: ${{ github.token }} 36 | date: '6 months ago' 37 | dry_run: false 38 | delete_tags: false 39 | extra_protected_branch_regex: ^(main|master|release.*|rudder-saas)$ 40 | exclude_open_pr_branches: true 41 | -------------------------------------------------------------------------------- /.github/workflows/pr-description-enforcer.yml: -------------------------------------------------------------------------------- 1 | name: 'Pull Request Description' 2 | on: 3 | pull_request: 4 | types: 5 | - opened 6 | - edited 7 | - reopened 8 | 9 | jobs: 10 | enforce: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: rudderlabs/pr-description-enforcer@v1.1.0 15 | with: 16 | repo-token: '${{ secrets.GITHUB_TOKEN }}' 17 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - 'main' 5 | - 'release/*' 6 | 7 | name: release-please 8 | 9 | jobs: 10 | release-please: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Extract Branch Name 14 | shell: bash 15 | run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT 16 | id: extract_branch 17 | 18 | - uses: googleapis/release-please-action@v4 19 | id: release 20 | with: 21 | token: ${{ github.token }} 22 | release-type: node 23 | 24 | # The logic below handles the npm publication: 25 | - name: Checkout 26 | uses: actions/checkout@v4 27 | # these if statements ensure that a publication only occurs when 28 | # a new release is created: 29 | if: ${{ steps.release.outputs.release_created }} 30 | 31 | - name: Setup Node 32 | uses: actions/setup-node@v4 33 | with: 34 | node-version-file: '.nvmrc' 35 | cache: 'npm' 36 | if: ${{ steps.release.outputs.release_created }} 37 | 38 | - name: Install Dependencies 39 | run: npm ci 40 | if: ${{ steps.release.outputs.release_created }} 41 | 42 | - name: Build Package 43 | run: npm run build 44 | if: ${{ steps.release.outputs.release_created }} 45 | 46 | - name: Configure NPM 47 | run: npm set //registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }} 48 | if: ${{ steps.release.outputs.release_created }} 49 | 50 | - name: Publish Package to NPM 51 | run: npm publish 52 | env: 53 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 54 | if: ${{ steps.release.outputs.release_created }} 55 | -------------------------------------------------------------------------------- /.github/workflows/semantic-pr.yml: -------------------------------------------------------------------------------- 1 | name: 'Semantic Pull Requests' 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - labeled 9 | - unlabeled 10 | - converted_to_draft 11 | - ready_for_review 12 | - synchronize 13 | 14 | jobs: 15 | main: 16 | name: Validate PR Title 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: amannn/action-semantic-pull-request@v5 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | with: 23 | types: | 24 | fix 25 | feat 26 | chore 27 | refactor 28 | exp 29 | docs 30 | test 31 | requireScope: false 32 | subjectPattern: ^(?![A-Z]).+$ 33 | subjectPatternError: | 34 | The subject "{subject}" found in the pull request title "{title}" 35 | didn't match the configured pattern. Please ensure that the subject 36 | doesn't start with an uppercase character. 37 | # For work-in-progress PRs you can typically use draft pull requests 38 | # from GitHub. However, private repositories on the free plan don't have 39 | # this option and therefore this action allows you to opt-in to using the 40 | # special "[WIP]" prefix to indicate this state. This will avoid the 41 | # validation of the PR title and the pull request checks remain pending. 42 | # Note that a second check will be reported if this is enabled. 43 | wip: true 44 | # When using "Squash and merge" on a PR with only one commit, GitHub 45 | # will suggest using that commit message instead of the PR title for the 46 | # merge commit, and it's easy to commit this by mistake. Enable this option 47 | # to also validate the commit message for one commit PRs. 48 | validateSingleCommit: false 49 | # Related to `validateSingleCommit` you can opt-in to validate that the PR 50 | # title matches a single commit to avoid confusion. 51 | validateSingleCommitMatchesPrTitle: false 52 | # If the PR contains one of these labels, the validation is skipped. 53 | # Multiple labels can be separated by newlines. 54 | # If you want to rerun the validation when labels change, you might want 55 | # to use the `labeled` and `unlabeled` event triggers in your workflow. 56 | ignoreLabels: | 57 | bot 58 | dependencies 59 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | run-tests: 7 | name: Run Tests 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v4 12 | 13 | - name: Setup Node 14 | uses: actions/setup-node@v4 15 | with: 16 | node-version-file: '.nvmrc' 17 | cache: 'npm' 18 | 19 | - name: Install Dependencies 20 | run: npm ci 21 | 22 | - name: Run Tests 23 | run: npm run test:no-logging 24 | 25 | - name: Run Lint Checks 26 | run: | 27 | npm run check:lint 28 | 29 | - name: Upload coverage reports to Codecov 30 | uses: codecov/codecov-action@v5 31 | with: 32 | token: ${{ secrets.CODECOV_TOKEN }} 33 | directory: ./reports/coverage 34 | 35 | - name: Update sonar-project.properties 36 | run: | 37 | # Retrieve the version from package.json 38 | version=$(node -e "console.log(require('./package.json').version)") 39 | # Update the sonar-project.properties file with the version 40 | sed -i "s/sonar.projectVersion=.*$/sonar.projectVersion=$version/" sonar-project.properties 41 | 42 | - name: Fix filesystem paths in generated reports 43 | if: always() 44 | run: | 45 | sed -i 's+home/runner/work/rudder-workflow-engine/rudder-workflow-engine+/github/workspace+g' reports/coverage/lcov.info 46 | sed -i 's+/home/runner/work/rudder-workflow-engine/rudder-workflow-engine+/github/workspace+g' reports/eslint.json 47 | 48 | - name: SonarCloud Scan 49 | if: always() 50 | uses: SonarSource/sonarqube-scan-action@v4 51 | env: 52 | GITHUB_TOKEN: ${{ secrets.PAT }} 53 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 54 | -------------------------------------------------------------------------------- /.github/workflows/verify.yml: -------------------------------------------------------------------------------- 1 | name: Verify 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | formatting-lint: 8 | name: Check for formatting & lint errors 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | with: 15 | # Make sure the actual branch is checked out when running on pull requests 16 | ref: ${{ github.head_ref }} 17 | 18 | - name: Setup Node 19 | uses: actions/setup-node@v4 20 | with: 21 | node-version-file: .nvmrc 22 | cache: 'npm' 23 | 24 | - name: Install Dependencies 25 | run: npm ci 26 | 27 | - name: Run Lint Checks 28 | run: | 29 | npm run lint 30 | 31 | - run: git diff --exit-code 32 | 33 | - name: Error message 34 | if: ${{ failure() }} 35 | run: | 36 | echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.' 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | reports 13 | .stryker-tmp 14 | 15 | # Runtime data 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | 21 | # Directory for instrumented libs generated by jscoverage/JSCover 22 | lib-cov 23 | 24 | # Coverage directory used by tools like istanbul 25 | coverage 26 | *.lcov 27 | 28 | # nyc test coverage 29 | .nyc_output 30 | 31 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 32 | .grunt 33 | 34 | # Bower dependency directory (https://bower.io/) 35 | bower_components 36 | 37 | # node-waf configuration 38 | .lock-wscript 39 | 40 | # Compiled binary addons (https://nodejs.org/api/addons.html) 41 | build/Release 42 | 43 | # Dependency directories 44 | node_modules/ 45 | jspm_packages/ 46 | 47 | # Snowpack dependency directory (https://snowpack.dev/) 48 | web_modules/ 49 | 50 | # TypeScript cache 51 | *.tsbuildinfo 52 | 53 | # Optional npm cache directory 54 | .npm 55 | 56 | # Optional eslint cache 57 | .eslintcache 58 | 59 | # Optional stylelint cache 60 | .stylelintcache 61 | 62 | # Microbundle cache 63 | .rpt2_cache/ 64 | .rts2_cache_cjs/ 65 | .rts2_cache_es/ 66 | .rts2_cache_umd/ 67 | 68 | # Optional REPL history 69 | .node_repl_history 70 | 71 | # Output of 'npm pack' 72 | *.tgz 73 | 74 | # Yarn Integrity file 75 | .yarn-integrity 76 | 77 | # dotenv environment variable files 78 | .env 79 | .env.development.local 80 | .env.test.local 81 | .env.production.local 82 | .env.local 83 | 84 | # parcel-bundler cache (https://parceljs.org/) 85 | .cache 86 | .parcel-cache 87 | 88 | # Next.js build output 89 | .next 90 | out 91 | 92 | # Nuxt.js build / generate output 93 | .nuxt 94 | dist 95 | 96 | # Gatsby files 97 | .cache/ 98 | # Comment in the public line in if your project uses Gatsby and not Next.js 99 | # https://nextjs.org/blog/next-9-1#public-directory-support 100 | # public 101 | 102 | # vuepress build output 103 | .vuepress/dist 104 | 105 | # vuepress v2.x temp and cache directory 106 | .temp 107 | .cache 108 | 109 | # Docusaurus cache and generated files 110 | .docusaurus 111 | 112 | # Serverless directories 113 | .serverless/ 114 | 115 | # FuseBox cache 116 | .fusebox/ 117 | 118 | # DynamoDB Local files 119 | .dynamodb/ 120 | 121 | # TernJS port file 122 | .tern-port 123 | 124 | # Stores VSCode versions used for testing VSCode extensions 125 | .vscode-test 126 | 127 | # yarn v2 128 | .yarn/cache 129 | .yarn/unplugged 130 | .yarn/build-state.yml 131 | .yarn/install-state.gz 132 | .pnp.* 133 | 134 | # build files 135 | build/ 136 | 137 | # mac files 138 | .DS_Store -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npx commitlint --edit $1 5 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npm test 5 | npm run lint:fix 6 | npm run lint:check 7 | npm run lint-staged 8 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 20.18.3 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | build/ 2 | node_modules/ 3 | coverage/ 4 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "all", 3 | "tabWidth": 2, 4 | "semi": true, 5 | "singleQuote": true, 6 | "printWidth": 100 7 | } 8 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "runtimeExecutable": "/usr/local/bin/node", 9 | "type": "node", 10 | "request": "launch", 11 | "name": "Jest Scenario", 12 | "program": "${workspaceFolder}/node_modules/.bin/jest", 13 | "args": [ 14 | "test/scenario.test.ts", 15 | "--config", 16 | "jest.config.ts", 17 | "--scenario=${input:scenario}", 18 | "--index=${input:index}" 19 | ], 20 | "console": "integratedTerminal", 21 | "internalConsoleOptions": "neverOpen", 22 | "windows": { 23 | "program": "${workspaceFolder}/node_modules/jest/bin/jest" 24 | } 25 | }, 26 | { 27 | "runtimeExecutable": "/usr/local/bin/node", 28 | "type": "node", 29 | "request": "launch", 30 | "name": "Jest Current File", 31 | "program": "${workspaceFolder}/node_modules/.bin/jest", 32 | "args": ["${relativeFile}", "--config", "jest.config.ts"], 33 | "console": "integratedTerminal", 34 | "internalConsoleOptions": "neverOpen", 35 | "windows": { 36 | "program": "${workspaceFolder}/node_modules/jest/bin/jest" 37 | } 38 | }, 39 | { 40 | "runtimeExecutable": "/usr/local/bin/node", 41 | "type": "node", 42 | "request": "launch", 43 | "name": "Jest Scenarios", 44 | "program": "${workspaceFolder}/node_modules/.bin/jest", 45 | "args": ["test/e2e.test.ts", "--config", "jest.config.ts", "--scenarios=${input:scenarios}"], 46 | "console": "integratedTerminal", 47 | "internalConsoleOptions": "neverOpen", 48 | "windows": { 49 | "program": "${workspaceFolder}/node_modules/jest/bin/jest" 50 | } 51 | } 52 | ], 53 | "inputs": [ 54 | { 55 | "id": "scenarios", 56 | "type": "promptString", 57 | "description": "Enter Scenarios", 58 | "default": "all" 59 | }, 60 | { 61 | "id": "scenario", 62 | "type": "promptString", 63 | "description": "Enter Scenario", 64 | "default": "basic_workflow" 65 | }, 66 | { 67 | "id": "index", 68 | "type": "promptString", 69 | "description": "Enter test index", 70 | "default": "0" 71 | } 72 | ] 73 | } 74 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "prettier.requireConfig": true, 3 | "prettier.configPath": ".prettierrc", 4 | "[typescript]": { 5 | "editor.defaultFormatter": "esbenp.prettier-vscode", 6 | "editor.formatOnSave": true 7 | }, 8 | "[javascript]": { 9 | "editor.defaultFormatter": "esbenp.prettier-vscode", 10 | "editor.formatOnSave": true 11 | }, 12 | "[jsonc]": { 13 | "editor.defaultFormatter": "esbenp.prettier-vscode", 14 | "editor.formatOnSave": true 15 | }, 16 | "[yaml]": { 17 | "editor.defaultFormatter": "esbenp.prettier-vscode", 18 | "editor.formatOnSave": true 19 | }, 20 | "editor.codeActionsOnSave": { 21 | "source.organizeImports": "never" 22 | }, 23 | "eslint.validate": ["javascript", "typescript"] 24 | } 25 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @koladilip @saikumarrs @sanpj2292 2 | * @rudderlabs/integrations-platform 3 | * @rudderlabs/integrations 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | - Using welcoming and inclusive language 18 | - Being respectful of differing viewpoints and experiences 19 | - Gracefully accepting constructive criticism 20 | - Focusing on what is best for the community 21 | - Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | - The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | - Trolling, insulting/derogatory comments, and personal or political attacks 28 | - Public or private harassment 29 | - Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | - Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at **contact@rudderstack.com**. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [https://www.contributor-covenant.org][contributor-covenant]. 72 | 73 | For answers to common questions about this code of conduct, see the [FAQs][faqs]. 74 | 75 | 76 | 77 | [homepage]: https://www.contributor-covenant.org 78 | [contributor-covenant]: https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 79 | [faqs]: https://www.contributor-covenant.org/faq 80 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to RudderStack 2 | 3 | Thanks for taking the time and for your help in improving this project! 4 | 5 | ## Table of contents 6 | 7 | - [**RudderStack Contributor Agreement**](#rudderstack-contributor-agreement) 8 | - [**How you can contribute to RudderStack**](#how-you-can-contribute-to-rudderstack) 9 | - [**Committing**](#committing) 10 | - [**Getting help**](#getting-help) 11 | 12 | ## RudderStack Contributor Agreement 13 | 14 | To contribute to this project, we need you to sign the [**Contributor License Agreement (“CLA”)**][CLA] for the first commit you make. By agreeing to the [**CLA**][CLA], we can add you to list of approved contributors and review the changes proposed by you. 15 | 16 | ## How you can contribute to RudderStack 17 | 18 | If you come across any issues or bugs, or have any suggestions for improvement, you can navigate to the specific file in the [**repo**](https://github.com/rudderlabs/rudder-repo-template), make the change, and raise a PR. 19 | 20 | You can also contribute to any open-source RudderStack project. View our [**GitHub page**](https://github.com/rudderlabs) to see all the different projects. 21 | 22 | ## Committing 23 | 24 | We prefer squash or rebase commits so that all changes from a branch are committed to master as a single commit. All pull requests are squashed when merged, but rebasing prior to merge gives you better control over the commit message. 25 | 26 | ## Getting help 27 | 28 | For any questions, concerns, or queries, you can start by asking a question in our [**Slack**](https://rudderstack.com/join-rudderstack-slack-community/) community. 29 | 30 | ### We look forward to your feedback on improving this project! 31 | 32 | 33 | 34 | [issue]: https://github.com/rudderlabs/rudder-server/issues/new 35 | [CLA]: https://rudderlabs.wufoo.com/forms/rudderlabs-contributor-license-agreement 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 RudderStack 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: ['@commitlint/config-conventional'], 3 | }; 4 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@rudderstack/workflow-engine", 3 | "version": "0.8.19", 4 | "description": "A generic workflow execution engine", 5 | "main": "build/index.js", 6 | "types": "build/index.d.ts", 7 | "publishConfig": { 8 | "access": "public" 9 | }, 10 | "scripts": { 11 | "test": "jest --coverage --verbose", 12 | "test:no-logging": "LOG_LEVEL=100 jest --coverage --verbose", 13 | "build": "tsc --project tsconfig.build.json", 14 | "clean": "rm -rf build", 15 | "build:clean": "npm run clean && npm run build", 16 | "lint:fix": "eslint . --fix", 17 | "lint:check": "eslint . || exit 1", 18 | "lint-staged": "lint-staged", 19 | "format": "prettier --write '**/*.ts' '**/*.json'", 20 | "lint": "npm run format && npm run lint:fix", 21 | "prepare": "husky install", 22 | "jest:scenarios": "jest e2e.test.ts --verbose", 23 | "test:scenario": "jest test/scenario.test.ts --verbose", 24 | "test:stryker": "stryker run", 25 | "check:lint": "eslint . -f json -o reports/eslint.json || exit 0" 26 | }, 27 | "repository": { 28 | "type": "git", 29 | "url": "git+https://github.com/rudderlabs/rudder-workflow-engine.git" 30 | }, 31 | "keywords": [ 32 | "rudder", 33 | "rudderstack", 34 | "cdp", 35 | "workflow", 36 | "engine" 37 | ], 38 | "author": "RudderStack", 39 | "license": "MIT", 40 | "bugs": { 41 | "url": "https://github.com/rudderlabs/rudder-workflow-engine/issues" 42 | }, 43 | "homepage": "https://github.com/rudderlabs/rudder-workflow-engine#readme", 44 | "devDependencies": { 45 | "@babel/eslint-parser": "^7.19.1", 46 | "@commitlint/cli": "^17.4.4", 47 | "@commitlint/config-conventional": "^17.4.4", 48 | "@stryker-mutator/core": "^6.4.1", 49 | "@types/jest": "^29.4.0", 50 | "@types/lodash": "^4.14.197", 51 | "@types/mocha": "^10.0.1", 52 | "@types/node": "^18.14.6", 53 | "@typescript-eslint/eslint-plugin": "^5.62.0", 54 | "@typescript-eslint/parser": "^5.62.0", 55 | "commander": "^10.0.0", 56 | "eslint": "^8.35.0", 57 | "eslint-config-airbnb-base": "^15.0.0", 58 | "eslint-config-airbnb-typescript": "^17.1.0", 59 | "eslint-config-prettier": "^8.7.0", 60 | "eslint-config-standard-with-typescript": "^34.0.0", 61 | "eslint-plugin-import": "^2.27.5", 62 | "eslint-plugin-promise": "^6.1.1", 63 | "eslint-plugin-sonarjs": "^0.23.0", 64 | "eslint-plugin-unused-imports": "^2.0.0", 65 | "husky": "^8.0.3", 66 | "jest": "^29.4.3", 67 | "jest-html-reporter": "^3.10.2", 68 | "lint-staged": "^15.2.10", 69 | "prettier": "^2.8.4", 70 | "ts-jest": "^29.0.5", 71 | "ts-node": "^10.9.1", 72 | "typescript": "^4.9.5" 73 | }, 74 | "dependencies": { 75 | "@aws-crypto/sha256-js": "^5.2.0", 76 | "@rudderstack/json-template-engine": "^0.19.5", 77 | "jsonata": "^2.0.5", 78 | "lodash": "^4.17.21", 79 | "object-sizeof": "^2.6.5", 80 | "yaml": "^2.6.0" 81 | }, 82 | "lint-staged": { 83 | "*.(ts|json|yaml)": "prettier --write" 84 | }, 85 | "files": [ 86 | "build/**/*.[jt]s", 87 | "CHANGELOG.md" 88 | ] 89 | } 90 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.log.level=INFO 2 | sonar.verbose=false 3 | sonar.qualitygate.wait=false 4 | 5 | # Project details 6 | sonar.projectKey=rudderlabs_rudder-workflow-engine 7 | sonar.organization=rudderlabs 8 | sonar.projectName=rudder-workflow-engine 9 | sonar.projectVersion=0.6.10 10 | 11 | # Meta-data for the project 12 | sonar.links.scm=https://github.com/rudderlabs/rudder-workflow-engine 13 | sonar.links.issue=https://github.com/rudderlabs/rudder-workflow-engine/issues 14 | 15 | # Path to reports 16 | sonar.javascript.lcov.reportPaths=reports/coverage/lcov.info 17 | sonar.eslint.reportPaths=reports/eslint.json 18 | 19 | # Path to sources 20 | sonar.sources=src 21 | sonar.inclusions=**/*.ts 22 | sonar.exclusions=**/*.json,**/*.html,**/*.png,**/*.jpg,**/*.gif,**/*.svg,**/*.yml 23 | # Path to tests 24 | sonar.tests=test 25 | sonar.test.inclusions=**/*.test.ts 26 | sonar.test.exclusions=**/*.json,**/*.html,**/*.png,**/*.jpg,**/*.gif,**/*.svg 27 | sonar.coverage.exclusions=test/**/*,**/*.json,**/*.html,**/*.png,**/*.jpg,**/*.gif,**/*.svg 28 | 29 | # Source encoding 30 | sonar.sourceEncoding=UTF-8 31 | 32 | # Exclusions for copy-paste detection 33 | sonar.cpd.exclusions=test/**/* -------------------------------------------------------------------------------- /src/bindings/common.test.ts: -------------------------------------------------------------------------------- 1 | import { SHA256, containsAll, toArray, toMilliseconds, toSeconds } from './common'; 2 | 3 | describe('Common Bindings', () => { 4 | describe('toMilliseconds', () => { 5 | it('should return milliseconds of timestamp', () => { 6 | expect(toMilliseconds('2022-04-26T09:35:24.561Z')).toBe(1650965724561); 7 | }); 8 | it('should return NaN for invalid timestamp string', () => { 9 | expect(toMilliseconds('invalid timestamp')).toBeUndefined(); 10 | }); 11 | it('should return undefined for blank timestamp string', () => { 12 | expect(toMilliseconds('')).toBeUndefined(); 13 | }); 14 | }); 15 | describe('toSeconds', () => { 16 | it('should return seconds of timestamp', () => { 17 | expect(toSeconds('2022-04-26T09:35:24.561Z')).toBe(1650965724); 18 | }); 19 | it('should return NaN for invalid timestamp string', () => { 20 | expect(toSeconds('invalid timestamp')).toBeUndefined(); 21 | }); 22 | it('should return undefined for blank timestamp string', () => { 23 | expect(toSeconds('')).toBeUndefined(); 24 | }); 25 | }); 26 | describe('toArray', () => { 27 | it('should return array if input is array', () => { 28 | expect(toArray([1, 2])).toEqual([1, 2]); 29 | }); 30 | 31 | it('should return array if input is single value', () => { 32 | expect(toArray({ a: 1 })).toEqual([{ a: 1 }]); 33 | }); 34 | it('should return undefined if input is undefined', () => { 35 | expect(toArray(undefined)).toBeUndefined(); 36 | }); 37 | }); 38 | describe('SHA256', () => { 39 | it('should return sha256 hashed data if input is string or number', () => { 40 | expect(SHA256('value')).toEqual( 41 | 'cd42404d52ad55ccfa9aca4adc828aa5800ad9d385a0671fbcbf724118320619', 42 | ); 43 | }); 44 | it('should return undefined if input is undefined', () => { 45 | expect(SHA256(undefined)).toEqual(undefined); 46 | }); 47 | }); 48 | 49 | describe('containsAll', () => { 50 | it('should return true if array2 contains all of array1', () => { 51 | expect(containsAll(['pizza', 'cola'], ['pizza', 'cake', 'cola'])).toEqual(true); 52 | }); 53 | it('should return false if array2 does not contain all of array1', () => { 54 | expect(containsAll(['pizza', 'cola', 'cheese'], ['pizza', 'cake', 'cola'])).toEqual(false); 55 | }); 56 | it('should return true if array1 is empty array', () => { 57 | expect(containsAll([], ['pizza', 'cake', 'cola'])).toEqual(true); 58 | }); 59 | it('should return false if array2 empty array', () => { 60 | expect(containsAll(['pizza', 'cola', 'cheese'], [])).toEqual(false); 61 | }); 62 | it('should return true if both array1 and array2 empty array', () => { 63 | expect(containsAll([], [])).toEqual(true); 64 | }); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /src/bindings/common.ts: -------------------------------------------------------------------------------- 1 | import { Sha256 } from '@aws-crypto/sha256-js'; 2 | import { at, identity } from 'lodash'; 3 | import { ReturnResultError, StatusError } from '../errors'; 4 | 5 | export { debug, error, info, warn } from '../common/logger'; 6 | 7 | export { chunk, sum } from 'lodash'; 8 | 9 | export function values(obj: Record): any[] { 10 | return Object.values(obj); 11 | } 12 | 13 | export function getByPaths(obj: any, paths: string | string[]): any { 14 | if (!obj || !paths) { 15 | return undefined; 16 | } 17 | const result = at(obj, paths).filter(identity); 18 | return Array.isArray(paths) ? result : result[0]; 19 | } 20 | 21 | export function toArray(obj: any): any[] | undefined { 22 | if (obj === undefined) { 23 | return obj; 24 | } 25 | return Array.isArray(obj) ? obj : [obj]; 26 | } 27 | 28 | export function getOneByPaths(obj: any, paths: string | string[]): any { 29 | const newPaths = toArray(paths); 30 | if (!obj || !newPaths?.length) { 31 | return undefined; 32 | } 33 | return at(obj, newPaths.shift())[0] ?? getOneByPaths(obj, newPaths); 34 | } 35 | 36 | export function doReturn(obj?: any) { 37 | throw new ReturnResultError(obj); 38 | } 39 | 40 | export function assertThrow(val: any, error: Error | string) { 41 | if (!val) { 42 | if (typeof error === 'string') { 43 | throw new Error(error); 44 | } 45 | throw error; 46 | } 47 | } 48 | 49 | export function doThrow(message: string, status = 500) { 50 | throw new StatusError(message, Number(status)); 51 | } 52 | 53 | export function toMilliseconds(timestamp: string): number | undefined { 54 | const time = new Date(timestamp).getTime(); 55 | if (!time) { 56 | return undefined; 57 | } 58 | return time; 59 | } 60 | 61 | export function toSeconds(timestamp: string): number | undefined { 62 | const timeInMillis = toMilliseconds(timestamp); 63 | if (!timeInMillis) { 64 | return undefined; 65 | } 66 | return Math.floor(timeInMillis / 1000); 67 | } 68 | 69 | export function SHA256(text: string | number | undefined) { 70 | if (!text) { 71 | return undefined; 72 | } 73 | const hash = new Sha256(); 74 | hash.update(`${text}`); 75 | const digest = hash.digestSync(); 76 | return Buffer.from(digest).toString('hex'); 77 | } 78 | 79 | // Check if arr1 is subset of arr2 80 | export function containsAll(arr1: any[], arr2: any[]): boolean { 81 | return arr1.every((element) => arr2.includes(element)); 82 | } 83 | -------------------------------------------------------------------------------- /src/bindings/index.ts: -------------------------------------------------------------------------------- 1 | export * from './common'; 2 | -------------------------------------------------------------------------------- /src/common/index.ts: -------------------------------------------------------------------------------- 1 | export * from './logger'; 2 | export * from './types'; 3 | export * from './utils'; 4 | -------------------------------------------------------------------------------- /src/common/logger.test.ts: -------------------------------------------------------------------------------- 1 | import { getInitialLogLevel } from './logger'; 2 | import { LogLevel } from './types'; 3 | 4 | describe('Logger', () => { 5 | describe('getInitialLogLevel', () => { 6 | it('should set log level', () => { 7 | const oldLogLevel = process.env.LOG_LEVEL; 8 | process.env.LOG_LEVEL = LogLevel.DEBUG.toString(); 9 | expect(getInitialLogLevel()).toEqual(LogLevel.DEBUG); 10 | process.env.LOG_LEVEL = oldLogLevel; 11 | }); 12 | }); 13 | }); 14 | -------------------------------------------------------------------------------- /src/common/logger.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { LogLevel } from './types'; 3 | 4 | export function getInitialLogLevel() { 5 | if (typeof process === 'object' && process.env.LOG_LEVEL) { 6 | return parseInt(process.env.LOG_LEVEL, 10); 7 | } 8 | return LogLevel.INFO; 9 | } 10 | 11 | let logLevel: LogLevel = getInitialLogLevel(); 12 | 13 | const mustLog = (...args) => { 14 | console.log(...args); 15 | }; 16 | 17 | const getLogLevel = () => logLevel; 18 | 19 | const setLogLevel = (newLevel: LogLevel) => { 20 | logLevel = newLevel; 21 | }; 22 | 23 | export const debug = (...args) => { 24 | if (LogLevel.DEBUG >= logLevel) { 25 | console.debug(...args); 26 | } 27 | }; 28 | 29 | export const info = (...args) => { 30 | if (LogLevel.INFO >= logLevel) { 31 | console.info(...args); 32 | } 33 | }; 34 | 35 | export const warn = (...args) => { 36 | if (LogLevel.WARN >= logLevel) { 37 | console.warn(...args); 38 | } 39 | }; 40 | 41 | export const error = (...args) => { 42 | if (LogLevel.ERROR >= logLevel) { 43 | console.error(...args); 44 | } 45 | }; 46 | 47 | export const logger = { 48 | setLogLevel, 49 | getLogLevel, 50 | debug, 51 | mustLog, 52 | info, 53 | warn, 54 | error, 55 | }; 56 | -------------------------------------------------------------------------------- /src/common/types.ts: -------------------------------------------------------------------------------- 1 | import { type StepExecutionError } from '../errors'; 2 | 3 | export interface Executor { 4 | execute(input: any, bindings?: ExecutionBindings): Promise; 5 | } 6 | 7 | export enum LogLevel { 8 | DEBUG = 0, 9 | INFO, 10 | WARN, 11 | ERROR, 12 | } 13 | export type PathBinding = { 14 | // exported value's name in bindings 15 | // if not specified then all paths will be exported 16 | name?: string | string[]; 17 | // the file from which the export has to be looked at 18 | // defaults to bindings.js / bindings.ts 19 | path?: string; 20 | // Export all when name specified 21 | exportAll?: true; 22 | }; 23 | 24 | export type ValueBinding = { 25 | name: string; 26 | value: any; 27 | }; 28 | 29 | export type ParentBinding = { 30 | name: string; 31 | fromParent: true; 32 | }; 33 | 34 | export type Binding = PathBinding | ValueBinding | ParentBinding; 35 | export type ExecutionBindings = { 36 | [key: string]: any; 37 | outputs: Record; 38 | context: Record; 39 | setContext: (string, any) => void; 40 | }; 41 | 42 | export enum TemplateType { 43 | JSONATA = 'jsonata', 44 | JSON_TEMPLATE = 'jsontemplate', 45 | } 46 | export type StepOutput = { 47 | error?: { 48 | message: string; 49 | status: number; 50 | originalError: Error; 51 | error: StepExecutionError; 52 | }; 53 | skipped?: boolean; 54 | output?: any; 55 | outputs?: Record; 56 | }; 57 | 58 | export type LoopStepOutput = { 59 | output: StepOutput[]; 60 | }; 61 | 62 | export type BatchResult = { 63 | key: string; 64 | items: any[]; 65 | indices: number[]; 66 | }; 67 | 68 | export type BatchStepOutput = { 69 | output: BatchResult[]; 70 | }; 71 | 72 | export enum StepType { 73 | Simple = 'simple', 74 | Workflow = 'workflow', 75 | Batch = 'batch', 76 | Custom = 'custom', 77 | Unknown = 'unknown', 78 | } 79 | 80 | export enum StepExitAction { 81 | Return = 'return', 82 | Continue = 'continue', 83 | } 84 | export type StepFunction = (input: any, bindings: ExecutionBindings) => StepOutput; 85 | 86 | export type ExternalWorkflow = { 87 | path: string; 88 | // root path for resolving dependencies 89 | rootPath?: string; 90 | bindings?: Binding[]; 91 | }; 92 | 93 | export type StepCommon = { 94 | name: string; 95 | description?: string; 96 | type?: StepType; 97 | condition?: string; 98 | else?: Step; 99 | inputTemplate?: string; 100 | loopOverInput?: boolean; 101 | loopCondition?: string; 102 | onComplete?: StepExitAction; 103 | onError?: StepExitAction; 104 | debug?: boolean; 105 | identity?: boolean; 106 | }; 107 | 108 | export type SimpleStep = StepCommon & { 109 | // One of the template, templatePath, externalWorkflowPath, Function are required for simple steps 110 | template?: string; 111 | templatePath?: string; 112 | mappings?: boolean; 113 | // external workflow is executed independently and we can access only final output 114 | externalWorkflow?: ExternalWorkflow; 115 | // Function must be passed using bindings 116 | functionName?: string; 117 | }; 118 | 119 | export type BatchConfig = { 120 | options?: { 121 | size?: number; 122 | length?: number; 123 | }; 124 | disabled?: true; 125 | filter?: string; 126 | map?: string; 127 | key: string; 128 | }; 129 | 130 | export type BatchStep = StepCommon & { 131 | batches?: BatchConfig[]; 132 | // Executor must be passed using bindings 133 | executor?: string; 134 | }; 135 | 136 | export type CustomStep = StepCommon & { 137 | // provider must be passed using bindings 138 | provider?: string; 139 | // Executor must be passed using bindings 140 | executor?: string; 141 | params?: Record; 142 | }; 143 | 144 | export type WorkflowStep = StepCommon & { 145 | // One of the template, templatePath, Function are required for simple steps 146 | // One of the steps, workflowStepPath are required for workflow steps 147 | steps?: SimpleStep[]; 148 | workflowStepPath?: string; 149 | }; 150 | 151 | export type Step = SimpleStep | WorkflowStep | BatchStep | CustomStep; 152 | 153 | export type Workflow = { 154 | name: string; 155 | bindings?: Binding[]; 156 | steps: Step[]; 157 | templateType?: TemplateType; 158 | // Executor name will be searched in the bindings 159 | executor?: string; 160 | }; 161 | 162 | export type WorkflowOutput = { 163 | output?: any; 164 | outputs?: Record; 165 | status?: number; 166 | error?: any; 167 | }; 168 | 169 | export type WorkflowOptions = { 170 | bindingsPaths?: string[]; 171 | rootPath: string; 172 | creationTimeBindings?: Record; 173 | templateType?: TemplateType; 174 | executor?: WorkflowExecutor; 175 | bindingProvider?: WorkflowBindingProvider; 176 | }; 177 | 178 | export type WorkflowOptionsInternal = WorkflowOptions & { 179 | currentBindings: Record; 180 | parentBindings?: Record; 181 | }; 182 | 183 | export interface WorkflowExecutor { 184 | execute( 185 | engine: WorkflowEngine, 186 | input: any, 187 | bindings?: Record, 188 | ): Promise; 189 | } 190 | 191 | export interface WorkflowBindingProvider { 192 | provide(name: string): Promise; 193 | } 194 | 195 | export interface StepExecutor extends Executor { 196 | /** 197 | * Returns the name of the step which executor is operating 198 | */ 199 | getStepName(): string; 200 | /** 201 | * Returns the step which executor is operating 202 | */ 203 | getStep(): Step; 204 | /** 205 | * Executes the step 206 | */ 207 | execute(input: any, executionBindings: ExecutionBindings): Promise; 208 | } 209 | 210 | export interface WorkflowEngine extends Executor { 211 | getName(): string; 212 | getBindings(): Record; 213 | getStepExecutors(): StepExecutor[]; 214 | getStepExecutor(stepName: string): StepExecutor; 215 | execute(input: any, executionBindings?: Record): Promise; 216 | } 217 | -------------------------------------------------------------------------------- /src/common/utils/batch.test.ts: -------------------------------------------------------------------------------- 1 | import { BatchUtils } from './batch'; 2 | 3 | describe('Batch utils', () => { 4 | describe('chunkArrayBySizeAndLength', () => { 5 | const array = Array.from('abcdefghijk'); 6 | it('should chunk by length = 2', () => { 7 | expect(BatchUtils.chunkArrayBySizeAndLength(array, { maxItems: 2 })).toEqual({ 8 | indices: [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10]], 9 | items: [['a', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j'], ['k']], 10 | }); 11 | }); 12 | it('should chunk by length = 3', () => { 13 | expect(BatchUtils.chunkArrayBySizeAndLength(array, { maxItems: 3 })).toEqual({ 14 | items: [ 15 | ['a', 'b', 'c'], 16 | ['d', 'e', 'f'], 17 | ['g', 'h', 'i'], 18 | ['j', 'k'], 19 | ], 20 | indices: [ 21 | [0, 1, 2], 22 | [3, 4, 5], 23 | [6, 7, 8], 24 | [9, 10], 25 | ], 26 | }); 27 | }); 28 | 29 | it('should chunk by size = 32', () => { 30 | expect(BatchUtils.chunkArrayBySizeAndLength(array, { maxSizeInBytes: 32 })).toEqual({ 31 | items: [['a', 'b'], ['c', 'd'], ['e', 'f'], ['g', 'h'], ['i', 'j'], ['k']], 32 | indices: [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10]], 33 | }); 34 | }); 35 | it('should chunk by size = 48', () => { 36 | expect(BatchUtils.chunkArrayBySizeAndLength(array, { maxSizeInBytes: 48 })).toEqual({ 37 | items: [ 38 | ['a', 'b', 'c'], 39 | ['d', 'e', 'f'], 40 | ['g', 'h', 'i'], 41 | ['j', 'k'], 42 | ], 43 | indices: [ 44 | [0, 1, 2], 45 | [3, 4, 5], 46 | [6, 7, 8], 47 | [9, 10], 48 | ], 49 | }); 50 | }); 51 | it('should chunk without size and length', () => { 52 | expect(BatchUtils.chunkArrayBySizeAndLength(array)).toEqual({ 53 | items: [['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']], 54 | indices: [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], 55 | }); 56 | }); 57 | it('should chunk empty array', () => { 58 | expect(BatchUtils.chunkArrayBySizeAndLength([])).toEqual({ 59 | items: [], 60 | indices: [], 61 | }); 62 | }); 63 | }); 64 | 65 | describe('validateBatchResults', () => { 66 | it('should not throw error if batchResults is valid', () => { 67 | expect(() => { 68 | BatchUtils.validateBatchResults( 69 | [1, 2, 3], 70 | [ 71 | { 72 | indices: [0, 1, 2], 73 | items: [1, 2, 3], 74 | key: 'key', 75 | }, 76 | ], 77 | ); 78 | }).not.toThrow(); 79 | }); 80 | it('should throw error if batchResults is not array', () => { 81 | expect(() => { 82 | BatchUtils.validateBatchResults([], {} as any); 83 | }).toThrow(); 84 | }); 85 | it('should throw error if batchResults does not contain all indices', () => { 86 | expect(() => { 87 | BatchUtils.validateBatchResults( 88 | [1, 2, 3], 89 | [ 90 | { 91 | indices: [1, 2], 92 | items: [1, 2], 93 | key: 'key', 94 | }, 95 | ], 96 | ); 97 | }).toThrow(); 98 | }); 99 | it('should throw error if batchResults does not contain same number of items as input', () => { 100 | expect(() => { 101 | BatchUtils.validateBatchResults( 102 | [1, 2, 3], 103 | [ 104 | { 105 | indices: [0, 1, 2], 106 | items: [1, 2], 107 | key: 'key', 108 | }, 109 | ], 110 | ); 111 | }).toThrow(); 112 | }); 113 | 114 | it('should throw error if batchResults does not contain same number of indices as input', () => { 115 | expect(() => { 116 | BatchUtils.validateBatchResults( 117 | [1, 2, 3], 118 | [ 119 | { 120 | indices: [0, 1], 121 | items: [1, 2, 3], 122 | key: 'key', 123 | }, 124 | ], 125 | ); 126 | }).toThrow(); 127 | }); 128 | it('should throw error if batchResults does not contain same of number of indices as items', () => { 129 | expect(() => { 130 | BatchUtils.validateBatchResults( 131 | [1, 2, 3], 132 | [ 133 | { 134 | indices: [0, 1], 135 | items: [1], 136 | key: 'key', 137 | }, 138 | { 139 | indices: [2], 140 | items: [2, 3], 141 | key: 'key', 142 | }, 143 | ], 144 | ); 145 | }).toThrow(); 146 | }); 147 | it('should throw error if batchResults does not valid key', () => { 148 | expect(() => { 149 | BatchUtils.validateBatchResults( 150 | [1, 2, 3], 151 | [ 152 | { 153 | indices: [0, 1, 2], 154 | items: [1, 2, 3], 155 | key: '', 156 | }, 157 | ], 158 | ); 159 | }).toThrow(); 160 | }); 161 | }); 162 | }); 163 | -------------------------------------------------------------------------------- /src/common/utils/batch.ts: -------------------------------------------------------------------------------- 1 | import sizeof from 'object-sizeof'; 2 | import { BatchError } from '../../errors/batch'; 3 | import { BatchResult } from '../types'; 4 | 5 | export class BatchUtils { 6 | static chunkArrayBySizeAndLength( 7 | array: any[], 8 | options: { maxSizeInBytes?: number; maxItems?: number } = {}, 9 | ): { items: any[][]; indices: number[][] } { 10 | const items: any[][] = []; 11 | const indices: number[][] = []; 12 | if (!array?.length) { 13 | return { items, indices }; 14 | } 15 | const { maxSizeInBytes, maxItems } = options; 16 | let currentChunk: any[] = [array[0]]; 17 | let currentIndices: number[] = [0]; 18 | let currentSize = maxSizeInBytes ? sizeof(array[0]) : 0; 19 | for (let idx = 1; idx < array.length; idx += 1) { 20 | const item = array[idx]; 21 | const itemSizeInBytes = maxSizeInBytes ? sizeof(item) : 0; 22 | if ( 23 | this.isSizeLimitReached(itemSizeInBytes, currentSize, maxSizeInBytes) || 24 | this.isLengthLimitReached(currentChunk.length, maxItems) 25 | ) { 26 | items.push(currentChunk); 27 | indices.push(currentIndices); 28 | currentChunk = []; 29 | currentIndices = []; 30 | currentSize = 0; 31 | } 32 | currentChunk.push(item); 33 | currentIndices.push(idx); 34 | currentSize += itemSizeInBytes; 35 | } 36 | 37 | if (currentChunk.length > 0) { 38 | items.push(currentChunk); 39 | indices.push(currentIndices); 40 | } 41 | 42 | return { items, indices }; 43 | } 44 | 45 | static validateBatchResults(input: any[], batchResults: BatchResult[]) { 46 | if (!Array.isArray(batchResults)) { 47 | throw new BatchError('batch step requires array output from batch executor'); 48 | } 49 | const batchIndices = batchResults.reduce( 50 | (acc, batchResult) => acc.concat(batchResult.indices), 51 | [] as number[], 52 | ); 53 | batchIndices.sort((a, b) => a - b); 54 | batchIndices.forEach((index, idx) => { 55 | if (index !== idx) { 56 | throw new BatchError('batch step requires return all indices'); 57 | } 58 | }); 59 | 60 | const batchItems = batchResults.reduce( 61 | (acc, batchResult) => acc.concat(batchResult.items), 62 | [] as any[], 63 | ); 64 | if (batchIndices.length !== input.length || batchItems.length !== input.length) { 65 | throw new BatchError( 66 | 'batch step requires batch executor to return same number of items as input', 67 | ); 68 | } 69 | batchResults.forEach((batchResult) => { 70 | if (!batchResult.key) { 71 | throw new BatchError( 72 | 'batch step requires batch executor to return key for each batch result', 73 | ); 74 | } 75 | if (batchResult.indices.length !== batchResult.items.length) { 76 | throw new BatchError( 77 | 'batch step requires batch executor to return same number of items and indices', 78 | ); 79 | } 80 | }); 81 | } 82 | 83 | private static isSizeLimitReached( 84 | itemSizeInBytes: number, 85 | currentSize: number, 86 | maxSizeInBytes?: number, 87 | ) { 88 | return maxSizeInBytes && currentSize + itemSizeInBytes > maxSizeInBytes; 89 | } 90 | 91 | private static isLengthLimitReached(currentLength: number, maxLength?: number) { 92 | return maxLength && currentLength + 1 > maxLength; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/common/utils/common.test.ts: -------------------------------------------------------------------------------- 1 | import { CommonUtils } from './common'; 2 | 3 | describe('Common utils', () => { 4 | describe('findDuplicateStrings', () => { 5 | it('should return duplicate strings when duplicates are present', () => { 6 | expect(CommonUtils.findDuplicateStrings(['a', 'b', 'c', 'a', 'c'])).toEqual(['a', 'c']); 7 | }); 8 | it('should return empty array when no duplicates are present', () => { 9 | expect(CommonUtils.findDuplicateStrings(['a', 'b', 'c'])).toEqual([]); 10 | }); 11 | it('should return empty array when empty array is given as input', () => { 12 | expect(CommonUtils.findDuplicateStrings([])).toEqual([]); 13 | }); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /src/common/utils/common.ts: -------------------------------------------------------------------------------- 1 | export class CommonUtils { 2 | static findDuplicateStrings(strings: string[]): Array { 3 | return Array.from(new Set(strings.filter((item, index) => strings.indexOf(item) !== index))); 4 | } 5 | 6 | static async readFile(path: string): Promise { 7 | const readFile = await import('fs/promises').then((m) => m.readFile); 8 | return readFile(path, { encoding: 'utf-8' }); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/common/utils/index.ts: -------------------------------------------------------------------------------- 1 | export * from './batch'; 2 | export * from './common'; 3 | export * from './step'; 4 | -------------------------------------------------------------------------------- /src/common/utils/step.ts: -------------------------------------------------------------------------------- 1 | import { StepCreationError } from '../../errors'; 2 | import { 3 | BatchStep, 4 | CustomStep, 5 | SimpleStep, 6 | Step, 7 | StepExitAction, 8 | StepType, 9 | WorkflowStep, 10 | } from '../types'; 11 | import { CommonUtils } from './common'; 12 | 13 | const stepNameRegex = /^[a-zA-Z][0-9a-zA-Z]*$/; 14 | export class StepUtils { 15 | static getStepType(step: Step): StepType { 16 | if (StepUtils.isBatchStep(step)) { 17 | return StepType.Batch; 18 | } 19 | 20 | if (StepUtils.isCustomStep(step)) { 21 | return StepType.Custom; 22 | } 23 | 24 | if (StepUtils.isWorkflowStep(step)) { 25 | return StepType.Workflow; 26 | } 27 | if (StepUtils.isSimpleStep(step)) { 28 | return StepType.Simple; 29 | } 30 | return StepType.Unknown; 31 | } 32 | 33 | static isBatchStep(step: BatchStep) { 34 | return step.type === StepType.Batch; 35 | } 36 | 37 | static isCustomStep(step: BatchStep) { 38 | return step.type === StepType.Custom; 39 | } 40 | 41 | static isWorkflowStep(step: WorkflowStep): boolean { 42 | return !!step.steps?.length || !!step.workflowStepPath; 43 | } 44 | 45 | static isSimpleStep(step: SimpleStep): boolean { 46 | return ( 47 | !!step.identity || 48 | !!step.template || 49 | !!step.templatePath || 50 | !!step.functionName || 51 | !!step.externalWorkflow 52 | ); 53 | } 54 | 55 | static populateElseStep(step: Step) { 56 | if (step.else) { 57 | // eslint-disable-next-line no-param-reassign 58 | step.else.type = StepUtils.getStepType(step.else); 59 | this.populateElseStep(step.else); 60 | } 61 | } 62 | 63 | static populateSteps(steps: Step[]) { 64 | for (const step of steps) { 65 | step.type = StepUtils.getStepType(step); 66 | this.populateElseStep(step); 67 | } 68 | } 69 | 70 | private static checkForStepNameDuplicates(steps: Step[]) { 71 | const duplicateNames = CommonUtils.findDuplicateStrings(steps.map((step) => step.name)); 72 | if (duplicateNames.length > 0) { 73 | throw new StepCreationError(`found duplicate step names: ${duplicateNames}`); 74 | } 75 | } 76 | 77 | static validateSteps(steps: Step[], notAllowedTypes?: string[]) { 78 | const notAllowed = notAllowedTypes ?? []; 79 | notAllowed.push(StepType.Unknown); 80 | this.checkForStepNameDuplicates(steps); 81 | for (let i = 0; i < steps.length; i += 1) { 82 | StepUtils.validateStep(steps[i], i, notAllowed); 83 | } 84 | } 85 | 86 | private static validateStepName(step: Step, index: number) { 87 | if (!step.name) { 88 | throw new StepCreationError(`step#${index} should have a name`); 89 | } 90 | 91 | if (!stepNameRegex.exec(step.name)) { 92 | throw new StepCreationError('step name is invalid', step.name); 93 | } 94 | } 95 | 96 | private static validateStepType(step: Step, index: number, notAllowed: string[]) { 97 | if (notAllowed.includes(step.type as StepType)) { 98 | throw new StepCreationError('unsupported step type', step.name); 99 | } 100 | } 101 | 102 | private static validateElseStep(step: Step, index: number, notAllowed: string[]) { 103 | if (step.else) { 104 | if (!step.condition) { 105 | throw new StepCreationError('else step should be used in a step with condition', step.name); 106 | } else { 107 | this.validateStep(step.else, index, notAllowed); 108 | } 109 | } 110 | } 111 | 112 | private static validateLoopStep(step: Step) { 113 | if (step.loopCondition && !step.loopOverInput) { 114 | throw new StepCreationError('loopCondition should be used with loopOverInput', step.name); 115 | } 116 | 117 | if (step.loopOverInput && step.type === StepType.Batch) { 118 | throw new StepCreationError('loopOverInput is not supported for batch step', step.name); 119 | } 120 | } 121 | 122 | private static validateOnComplete(step: Step) { 123 | if (step.onComplete === StepExitAction.Return && !step.condition) { 124 | throw new StepCreationError( 125 | '"onComplete = return" should be used in a step with condition', 126 | step.name, 127 | ); 128 | } 129 | } 130 | 131 | static validateStep(step: Step, index: number, notAllowed: string[]) { 132 | this.validateStepName(step, index); 133 | 134 | this.validateStepType(step, index, notAllowed); 135 | 136 | this.validateElseStep(step, index, notAllowed); 137 | 138 | this.validateLoopStep(step); 139 | 140 | this.validateOnComplete(step); 141 | 142 | if (step.type === StepType.Batch) { 143 | this.ValidateBatchStep(step as BatchStep); 144 | } 145 | 146 | if (step.type === StepType.Custom) { 147 | this.ValidateCustomStep(step as CustomStep); 148 | } 149 | } 150 | 151 | static ValidateBatchStep(step: BatchStep) { 152 | if (!step.batches && !step.executor) { 153 | throw new StepCreationError('batches or executor is required for batch step', step.name); 154 | } 155 | 156 | if (step.batches && step.executor) { 157 | throw new StepCreationError('only one of batches or executor should be specified', step.name); 158 | } 159 | } 160 | 161 | static ValidateCustomStep(step: CustomStep) { 162 | if (!step.provider && !step.executor) { 163 | throw new StepCreationError('provider or executor is required for custom step', step.name); 164 | } 165 | if (step.executor && step.provider) { 166 | throw new StepCreationError( 167 | 'only one of provider or executor should be specified', 168 | step.name, 169 | ); 170 | } 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/errors/batch.ts: -------------------------------------------------------------------------------- 1 | export class BatchError extends Error {} 2 | -------------------------------------------------------------------------------- /src/errors/binding_not_found.ts: -------------------------------------------------------------------------------- 1 | export class BindingNotFoundError extends Error { 2 | constructor(bindingName: string) { 3 | super(`Binding not found: ${bindingName}`); 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/errors/index.ts: -------------------------------------------------------------------------------- 1 | export * from './batch'; 2 | export * from './binding_not_found'; 3 | export * from './info'; 4 | export * from './return_result'; 5 | export * from './status'; 6 | export * from './step_creation'; 7 | export * from './step_execution'; 8 | export * from './utils'; 9 | export * from './workflow_creation'; 10 | export * from './workflow_execution'; 11 | -------------------------------------------------------------------------------- /src/errors/info.ts: -------------------------------------------------------------------------------- 1 | export type ErrorInfo = { 2 | stepName?: string; 3 | childStepName?: string; 4 | error?: Error; 5 | }; 6 | -------------------------------------------------------------------------------- /src/errors/return_result.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable max-classes-per-file */ 2 | 3 | export class ReturnResultError extends Error { 4 | result: any; 5 | 6 | constructor(result: any) { 7 | super(); 8 | this.result = result; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/errors/status.ts: -------------------------------------------------------------------------------- 1 | export class StatusError extends Error { 2 | status: number; 3 | 4 | constructor(message: string, status: number) { 5 | super(message); 6 | this.status = +status; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/errors/step_creation.ts: -------------------------------------------------------------------------------- 1 | import { StatusError } from './status'; 2 | 3 | export class StepCreationError extends StatusError { 4 | stepName?: string; 5 | 6 | childStepName?: string; 7 | 8 | constructor(message: string, stepName?: string, childStepName?: string) { 9 | super(message, 400); 10 | this.stepName = stepName; 11 | this.childStepName = childStepName; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/errors/step_execution.ts: -------------------------------------------------------------------------------- 1 | import { ErrorInfo } from './info'; 2 | import { StatusError } from './status'; 3 | 4 | export class StepExecutionError extends StatusError { 5 | stepName?: string; 6 | 7 | childStepName?: string; 8 | 9 | error: Error; 10 | 11 | originalError: Error; 12 | 13 | constructor(message: string, status: number, info?: ErrorInfo) { 14 | super(message, status); 15 | this.stepName = info?.stepName; 16 | this.childStepName = info?.childStepName; 17 | this.error = info?.error ?? this; 18 | this.originalError = (this.error as any).originalError ?? info?.error; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/errors/utils.ts: -------------------------------------------------------------------------------- 1 | import { StepExecutionError } from './step_execution'; 2 | import { WorkflowExecutionError } from './workflow_execution'; 3 | 4 | export class ErrorUtils { 5 | static isAssertError(error: any) { 6 | return error.token === 'assert'; 7 | } 8 | 9 | static getErrorStatus(error: any) { 10 | return error?.response?.status ?? error?.status ?? 500; 11 | } 12 | 13 | static createStepExecutionError( 14 | error: Error, 15 | stepName: string, 16 | childStepName?: string, 17 | ): StepExecutionError { 18 | return new StepExecutionError(error.message, ErrorUtils.getErrorStatus(error), { 19 | stepName, 20 | childStepName, 21 | error, 22 | }); 23 | } 24 | 25 | static createWorkflowExecutionError(error: Error, workflowName: string): WorkflowExecutionError { 26 | if (error instanceof StepExecutionError) { 27 | return new WorkflowExecutionError( 28 | error.message, 29 | ErrorUtils.getErrorStatus(error), 30 | workflowName, 31 | { 32 | stepName: error.stepName, 33 | childStepName: error.childStepName, 34 | error: error.error, 35 | }, 36 | ); 37 | } 38 | 39 | return new WorkflowExecutionError( 40 | error.message, 41 | ErrorUtils.getErrorStatus(error), 42 | workflowName, 43 | { 44 | error, 45 | }, 46 | ); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/errors/workflow_creation.ts: -------------------------------------------------------------------------------- 1 | import { StepCreationError } from './step_creation'; 2 | 3 | export class WorkflowCreationError extends StepCreationError { 4 | workflowName: string; 5 | 6 | constructor(message: string, workflowName: string, stepName?: string, childStepName?: string) { 7 | super(message, stepName, childStepName); 8 | this.workflowName = workflowName; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/errors/workflow_execution.ts: -------------------------------------------------------------------------------- 1 | import { ErrorInfo } from './info'; 2 | import { StepExecutionError } from './step_execution'; 3 | 4 | export class WorkflowExecutionError extends StepExecutionError { 5 | workflowName: string; 6 | 7 | constructor(message: string, status: number, workflowName: string, info?: ErrorInfo) { 8 | super(message, status, info); 9 | this.workflowName = workflowName; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './common'; 2 | export * from './errors'; 3 | export * from './steps'; 4 | export * from './workflow'; 5 | -------------------------------------------------------------------------------- /src/steps/base/batch/default_batch_workflow_executor.ts: -------------------------------------------------------------------------------- 1 | import { BatchResult, ExecutionBindings } from '../../../common/types'; 2 | import { BatchExecutor } from '../../types'; 3 | 4 | export class DefaultBatchWorkflowExecutor implements BatchExecutor { 5 | readonly executors: BatchExecutor[]; 6 | 7 | constructor(executors: BatchExecutor[]) { 8 | this.executors = executors; 9 | } 10 | 11 | async execute(input: any[], bindings: ExecutionBindings): Promise { 12 | const result: BatchResult[][] = await Promise.all( 13 | this.executors.map((executor) => executor.execute(input, bindings)), 14 | ); 15 | return result.flat(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/steps/base/batch/factory.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BatchConfig, 3 | BatchStep, 4 | SimpleStep, 5 | StepExecutor, 6 | StepType, 7 | WorkflowOptionsInternal, 8 | } from '../../../common'; 9 | import { StepCreationError } from '../../../errors'; 10 | import { BatchExecutor } from '../../types'; 11 | import { DefaultBatchWorkflowExecutor } from './default_batch_workflow_executor'; 12 | import { SimpleBatchExecutor } from './simple_batch_executor'; 13 | import { BatchStepExecutor } from './step_executor'; 14 | import { StepExecutorFactory } from '../../factory'; 15 | 16 | export class BatchStepExecutorFactory { 17 | static async create( 18 | step: BatchStep, 19 | options: WorkflowOptionsInternal, 20 | ): Promise { 21 | if (step.executor) { 22 | const executor = options.currentBindings[step.executor] as BatchExecutor; 23 | if (typeof executor?.execute !== 'function') { 24 | throw new StepCreationError(`Invalid batch executor: ${step.executor}`, step.name); 25 | } 26 | return new BatchStepExecutor(step, executor); 27 | } 28 | const defaultExecutor = await this.createDefaultBatchWorkflowExecutor(step, options); 29 | return new BatchStepExecutor(step, defaultExecutor); 30 | } 31 | 32 | static async createFilterMapExecutor( 33 | stepName: string, 34 | config: BatchConfig, 35 | options: WorkflowOptionsInternal, 36 | ): Promise { 37 | const filterStep: SimpleStep = { 38 | name: stepName, 39 | type: StepType.Simple, 40 | loopOverInput: true, 41 | }; 42 | 43 | if (config.filter) { 44 | filterStep.loopCondition = config.filter; 45 | } 46 | if (config.map) { 47 | filterStep.template = config.map; 48 | } else { 49 | filterStep.identity = true; 50 | } 51 | return StepExecutorFactory.create(filterStep, options); 52 | } 53 | 54 | static async createSimpleBatchExecutors( 55 | step: BatchStep, 56 | options: WorkflowOptionsInternal, 57 | ): Promise { 58 | const batches = step.batches as BatchConfig[]; 59 | return Promise.all( 60 | batches.map(async (config: BatchConfig) => { 61 | let filterMapExector: StepExecutor | undefined; 62 | if (config.filter) { 63 | filterMapExector = await this.createFilterMapExecutor( 64 | `${step.name}-batch-${config.key}`, 65 | config, 66 | options, 67 | ); 68 | } 69 | return new SimpleBatchExecutor(config, filterMapExector); 70 | }), 71 | ); 72 | } 73 | 74 | static async createDefaultBatchWorkflowExecutor( 75 | step: BatchStep, 76 | options: WorkflowOptionsInternal, 77 | ): Promise { 78 | const executors = await this.createSimpleBatchExecutors(step, options); 79 | return new DefaultBatchWorkflowExecutor(executors); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/steps/base/batch/simple_batch_executor.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BatchConfig, 3 | BatchResult, 4 | ExecutionBindings, 5 | LoopStepOutput, 6 | StepExecutor, 7 | } from '../../../common/types'; 8 | import { BatchUtils } from '../../../common/utils/batch'; 9 | import { BatchExecutor } from '../../types'; 10 | 11 | export class SimpleBatchExecutor implements BatchExecutor { 12 | readonly config: BatchConfig; 13 | 14 | readonly filterMapExector?: StepExecutor; 15 | 16 | constructor(config: BatchConfig, filterMapExector?: StepExecutor) { 17 | this.config = config; 18 | this.filterMapExector = filterMapExector; 19 | } 20 | 21 | async execute(input: any[], bindings: ExecutionBindings): Promise { 22 | const { filteredInput, filteredIndices } = await this.handleFilteringAndMapping( 23 | input, 24 | bindings, 25 | ); 26 | if (this.config.disabled) { 27 | return this.handleBatchingDisabled(filteredInput, filteredIndices); 28 | } 29 | return this.handleBatching(filteredInput, filteredIndices); 30 | } 31 | 32 | private handleBatching(filteredInput: any[], filteredIndices: number[]): BatchResult[] { 33 | const { items: itemArrays, indices } = BatchUtils.chunkArrayBySizeAndLength(filteredInput, { 34 | maxSizeInBytes: this.config.options?.size, 35 | maxItems: this.config.options?.length, 36 | }); 37 | return itemArrays.map((items, index) => ({ 38 | items, 39 | indices: indices[index].map((i) => filteredIndices[i]), 40 | key: this.config.key, 41 | })); 42 | } 43 | 44 | private handleBatchingDisabled(filteredInput: any[], filteredIndices: number[]): BatchResult[] { 45 | return filteredInput.map((item, index) => ({ 46 | items: [item], 47 | indices: [filteredIndices[index]], 48 | key: this.config.key, 49 | })); 50 | } 51 | 52 | private async handleFilteringAndMapping(input: any[], bindings: ExecutionBindings) { 53 | let filteredInput: any[] = input; 54 | let filteredIndices: number[] = Array.from(input.keys()); 55 | 56 | if (this.filterMapExector) { 57 | // Filter map executor internally invokes the loop step executor 58 | const filterResult = (await this.filterMapExector.execute(input, bindings)) as LoopStepOutput; 59 | filteredIndices = filteredIndices.filter((_, index) => !filterResult.output[index].skipped); 60 | filteredInput = filteredIndices.map((index) => filterResult.output[index].output); 61 | } 62 | return { filteredInput, filteredIndices }; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/steps/base/batch/step_executor.ts: -------------------------------------------------------------------------------- 1 | import { BatchStep, BatchStepOutput, ExecutionBindings } from '../../../common/types'; 2 | import { BatchUtils } from '../../../common/utils'; 3 | import { BatchError, ErrorUtils } from '../../../errors'; 4 | import { BatchExecutor } from '../../types'; 5 | import { BaseStepExecutor } from '../executors/base'; 6 | 7 | export class BatchStepExecutor extends BaseStepExecutor { 8 | readonly executor: BatchExecutor; 9 | 10 | async execute(input: any, bindings: ExecutionBindings): Promise { 11 | try { 12 | if (!Array.isArray(input)) { 13 | throw new BatchError('batch step requires array input'); 14 | } 15 | const batchResults = await this.executor.execute(input, bindings); 16 | BatchUtils.validateBatchResults(input, batchResults); 17 | return { 18 | output: batchResults, 19 | }; 20 | } catch (e: any) { 21 | throw ErrorUtils.createStepExecutionError(e, this.getStepName()); 22 | } 23 | } 24 | 25 | constructor(step: BatchStep, executor: BatchExecutor) { 26 | super(step); 27 | this.executor = executor; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/steps/base/custom/factory.ts: -------------------------------------------------------------------------------- 1 | import { CustomStep, StepExecutor, WorkflowOptionsInternal } from '../../../common/types'; 2 | import { StepCreationError } from '../../../errors'; 3 | import { CustomStepExecutor, CustomStepExecutorProvider } from '../../types'; 4 | import { BaseCustomStepExecutor } from './step_executor'; 5 | 6 | export class CustomStepExecutorFactory { 7 | static async create(step: CustomStep, options: WorkflowOptionsInternal): Promise { 8 | const executor = await this.getExecutor(step, options); 9 | return new BaseCustomStepExecutor(step, executor); 10 | } 11 | 12 | private static async getExecutor( 13 | step: CustomStep, 14 | options: WorkflowOptionsInternal, 15 | ): Promise { 16 | if (step.provider) { 17 | return this.getExecutorFromProvider(step, options); 18 | } 19 | const executor = options.currentBindings[step.executor as string] as CustomStepExecutor; 20 | if (typeof executor?.execute !== 'function') { 21 | throw new StepCreationError(`Invalid custom step executor: ${step.executor}`, step.name); 22 | } 23 | return executor; 24 | } 25 | 26 | private static getExecutorFromProvider( 27 | step: CustomStep, 28 | options: WorkflowOptionsInternal, 29 | ): Promise { 30 | const provider = options.currentBindings[step.provider as string] as CustomStepExecutorProvider; 31 | if (typeof provider?.provide !== 'function') { 32 | throw new StepCreationError(`Invalid custom step provider: ${step.provider}`, step.name); 33 | } 34 | return provider.provide(step); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/steps/base/custom/step_executor.ts: -------------------------------------------------------------------------------- 1 | import { CustomStep, ExecutionBindings, StepOutput } from '../../../common/types'; 2 | import { CustomStepExecutor } from '../../types'; 3 | import { BaseStepExecutor } from '../executors'; 4 | 5 | export class BaseCustomStepExecutor extends BaseStepExecutor { 6 | readonly executor: CustomStepExecutor; 7 | 8 | constructor(step: CustomStep, executor: CustomStepExecutor) { 9 | super(step); 10 | this.executor = executor; 11 | } 12 | 13 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 14 | const output = await this.executor.execute( 15 | input, 16 | executionBindings, 17 | (this.step as CustomStep).params, 18 | ); 19 | return { output }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/steps/base/executors/base.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, Step, StepExecutor, StepOutput } from '../../../common/types'; 2 | 3 | export abstract class BaseStepExecutor implements StepExecutor { 4 | protected readonly step: Step; 5 | 6 | constructor(step: Step) { 7 | this.step = step; 8 | } 9 | 10 | getStep(): Step { 11 | return this.step; 12 | } 13 | 14 | getStepName(): string { 15 | return this.step.name; 16 | } 17 | 18 | abstract execute(input: any, executionBindings: ExecutionBindings): Promise; 19 | } 20 | -------------------------------------------------------------------------------- /src/steps/base/executors/index.ts: -------------------------------------------------------------------------------- 1 | export * from './base'; 2 | export * from './workflow_step'; 3 | -------------------------------------------------------------------------------- /src/steps/base/executors/workflow_step.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ExecutionBindings, 3 | StepExecutor, 4 | StepExitAction, 5 | StepOutput, 6 | WorkflowStep, 7 | } from '../../../common/types'; 8 | import { ErrorUtils } from '../../../errors'; 9 | import { BaseStepExecutor } from './base'; 10 | 11 | export class WorkflowStepExecutor extends BaseStepExecutor { 12 | private readonly stepExecutors: StepExecutor[]; 13 | 14 | constructor(step: WorkflowStep, stepExecutors: StepExecutor[]) { 15 | super(step); 16 | this.stepExecutors = stepExecutors; 17 | } 18 | 19 | async executeChildStep( 20 | childExector: StepExecutor, 21 | input: any, 22 | executionBindings: ExecutionBindings, 23 | ): Promise { 24 | try { 25 | return await childExector.execute(input, executionBindings); 26 | } catch (error: any) { 27 | throw ErrorUtils.createStepExecutionError( 28 | error, 29 | this.getStepName(), 30 | childExector.getStepName(), 31 | ); 32 | } 33 | } 34 | 35 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 36 | const workflowStepName = this.getStepName(); 37 | const newExecutionBindings = executionBindings; 38 | newExecutionBindings.outputs[workflowStepName] = {}; 39 | let finalOutput: any; 40 | 41 | for (const childExecutor of this.stepExecutors) { 42 | const childStep = childExecutor.getStep(); 43 | // eslint-disable-next-line no-await-in-loop 44 | const { skipped, output } = await this.executeChildStep( 45 | childExecutor, 46 | input, 47 | executionBindings, 48 | ); 49 | if (!skipped) { 50 | newExecutionBindings.outputs[workflowStepName][childExecutor.getStepName()] = output; 51 | finalOutput = output; 52 | if (childStep.onComplete === StepExitAction.Return) { 53 | break; 54 | } 55 | } 56 | } 57 | return { outputs: executionBindings.outputs[workflowStepName], output: finalOutput }; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/steps/base/factory.test.ts: -------------------------------------------------------------------------------- 1 | import { StepType } from '../../common/types'; 2 | import { BaseStepExecutorFactory } from './factory'; 3 | 4 | describe('BaseStepExecutorFactory: ', () => { 5 | describe('create:', () => { 6 | it('should throw error when step type is unknown', () => { 7 | expect(() => 8 | BaseStepExecutorFactory.create( 9 | { type: StepType.Unknown, name: 'unknowStep' }, 10 | { rootPath: __dirname, currentBindings: {} }, 11 | ), 12 | ).toThrow(); 13 | }); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /src/steps/base/factory.ts: -------------------------------------------------------------------------------- 1 | import { 2 | SimpleStep, 3 | Step, 4 | StepExecutor, 5 | StepType, 6 | WorkflowOptionsInternal, 7 | WorkflowStep, 8 | } from '../../common'; 9 | import { StepCreationError } from '../../errors'; 10 | import { BatchStepExecutorFactory } from './batch/factory'; 11 | import { CustomStepExecutorFactory } from './custom/factory'; 12 | import { WorkflowStepExecutor } from './executors/workflow_step'; 13 | import { SimpleStepExecutorFactory } from './simple'; 14 | import { BaseStepUtils } from './utils'; 15 | import { StepExecutorFactory } from '../factory'; 16 | 17 | export class BaseStepExecutorFactory { 18 | static create(step: Step, options: WorkflowOptionsInternal): Promise { 19 | switch (step.type) { 20 | case StepType.Simple: 21 | return SimpleStepExecutorFactory.create(step, options); 22 | case StepType.Workflow: 23 | return this.createWorkflowStepExecutor(step, options); 24 | case StepType.Batch: 25 | return BatchStepExecutorFactory.create(step, options); 26 | case StepType.Custom: 27 | return CustomStepExecutorFactory.create(step, options); 28 | default: 29 | throw new StepCreationError(`Unknown step type: ${step.type}`); 30 | } 31 | } 32 | 33 | static async createWorkflowStepExecutor( 34 | step: WorkflowStep, 35 | options: WorkflowOptionsInternal, 36 | ): Promise { 37 | try { 38 | const newStep = await BaseStepUtils.prepareWorkflowStep(step, options); 39 | const simpleStepExecutors = await this.createSimpleStepExecutors(newStep, options); 40 | return new WorkflowStepExecutor(newStep, simpleStepExecutors); 41 | } catch (error: any) { 42 | throw new StepCreationError(error.message, step.name, error.stepName); 43 | } 44 | } 45 | 46 | static async createSimpleStepExecutors( 47 | workflowStep: WorkflowStep, 48 | options: WorkflowOptionsInternal, 49 | ): Promise { 50 | const steps = workflowStep.steps as SimpleStep[]; 51 | return Promise.all(steps.map((step) => StepExecutorFactory.create(step, options))); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/steps/base/index.ts: -------------------------------------------------------------------------------- 1 | export * from './executors'; 2 | export * from './factory'; 3 | export * from './simple'; 4 | export * from './utils'; 5 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/external_workflow.ts: -------------------------------------------------------------------------------- 1 | import { SimpleStep, StepOutput, WorkflowEngine } from '../../../../common'; 2 | import { ErrorUtils } from '../../../../errors'; 3 | import { BaseStepExecutor } from '../../executors/base'; 4 | 5 | export class ExternalWorkflowStepExecutor extends BaseStepExecutor { 6 | private readonly workflowEngine: WorkflowEngine; 7 | 8 | constructor(workflowEngine: WorkflowEngine, step: SimpleStep) { 9 | super(step); 10 | this.workflowEngine = workflowEngine; 11 | } 12 | 13 | async execute(input: any): Promise { 14 | try { 15 | return await this.workflowEngine.execute(input); 16 | } catch (error: any) { 17 | throw ErrorUtils.createStepExecutionError(error, this.getStepName()); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/function.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, SimpleStep, StepFunction, StepOutput } from '../../../../common'; 2 | import { StepCreationError } from '../../../../errors'; 3 | import { BaseStepExecutor } from '../../executors/base'; 4 | 5 | export class FunctionStepExecutor extends BaseStepExecutor { 6 | private readonly fn: StepFunction; 7 | 8 | constructor(step: SimpleStep, bindings: Record) { 9 | super(step); 10 | this.fn = FunctionStepExecutor.extractFunction( 11 | step.functionName as string, 12 | bindings, 13 | step.name, 14 | ); 15 | } 16 | 17 | private static extractFunction( 18 | functionName: string, 19 | bindings: Record, 20 | stepName: string, 21 | ): StepFunction { 22 | if (typeof bindings[functionName] !== 'function') { 23 | throw new StepCreationError(`Invalid functionName: ${functionName}`, stepName); 24 | } 25 | return bindings[functionName] as StepFunction; 26 | } 27 | 28 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 29 | return this.fn(input, executionBindings); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/identity.ts: -------------------------------------------------------------------------------- 1 | import { StepOutput } from '../../../../common'; 2 | import { BaseStepExecutor } from '../../executors'; 3 | 4 | export class IdentityStepExecutor extends BaseStepExecutor { 5 | // eslint-disable-next-line class-methods-use-this 6 | async execute(input: any): Promise { 7 | return { output: input }; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/index.ts: -------------------------------------------------------------------------------- 1 | export * from './external_workflow'; 2 | export * from './function'; 3 | export * from './template'; 4 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/template/factory.ts: -------------------------------------------------------------------------------- 1 | import { SimpleStep, TemplateType, WorkflowOptionsInternal } from '../../../../../common'; 2 | import { TemplateStepExecutor } from '../../../../types'; 3 | import { JsonataStepExecutor } from './jsonata'; 4 | import { JsonTemplateStepExecutor } from './jsontemplate'; 5 | 6 | export class TemplateStepExecutorFactory { 7 | static create( 8 | step: SimpleStep, 9 | template: string, 10 | options: WorkflowOptionsInternal, 11 | ): TemplateStepExecutor { 12 | if (options.templateType === TemplateType.JSONATA) { 13 | return new JsonataStepExecutor(step, template); 14 | } 15 | return new JsonTemplateStepExecutor(step, template, options.currentBindings); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/template/index.ts: -------------------------------------------------------------------------------- 1 | export * from './factory'; 2 | export * from './jsonata'; 3 | export * from './jsontemplate'; 4 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/template/jsonata.ts: -------------------------------------------------------------------------------- 1 | import jsonata from 'jsonata'; 2 | import { ExecutionBindings, Step, StepOutput } from '../../../../../common'; 3 | import { ErrorUtils, StatusError } from '../../../../../errors'; 4 | import { BaseStepExecutor } from '../../../executors/base'; 5 | 6 | export class JsonataStepExecutor extends BaseStepExecutor { 7 | private readonly templateExpression: jsonata.Expression; 8 | 9 | constructor(step: Step, template: string) { 10 | super(step); 11 | this.templateExpression = jsonata(template); 12 | } 13 | 14 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 15 | const output = await JsonataStepExecutor.evaluateJsonataExpr( 16 | this.templateExpression, 17 | input, 18 | executionBindings, 19 | ); 20 | return { output: JsonataStepExecutor.cleanUpArrays(output) }; 21 | } 22 | 23 | /** 24 | * JSONata adds custom properties to arrays for internal processing 25 | * hence it fails the comparison so we need to cleanup. 26 | * Reference: https://github.com/jsonata-js/jsonata/issues/296 27 | */ 28 | private static cleanUpArrays(obj: any) { 29 | let newObj = obj; 30 | if (Array.isArray(obj)) { 31 | newObj = obj.map((val) => this.cleanUpArrays(val)); 32 | } else if (newObj instanceof Object) { 33 | Object.keys(newObj).forEach((key) => { 34 | newObj[key] = this.cleanUpArrays(obj[key]); 35 | }); 36 | } 37 | return newObj; 38 | } 39 | 40 | static async evaluateJsonataExpr( 41 | expr: jsonata.Expression, 42 | data: any, 43 | bindings: Record, 44 | ): Promise { 45 | try { 46 | return await expr.evaluate(data, bindings); 47 | } catch (error: any) { 48 | if (error.token === 'doReturn') { 49 | return error.result; 50 | } 51 | if (ErrorUtils.isAssertError(error)) { 52 | throw new StatusError(error.message, 400); 53 | } 54 | throw error; 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/steps/base/simple/executors/template/jsontemplate.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Expression, 3 | FlatMappingPaths, 4 | JsonTemplateEngine, 5 | PathType, 6 | } from '@rudderstack/json-template-engine'; 7 | import { ExecutionBindings, SimpleStep, StepOutput } from '../../../../../common'; 8 | import { BaseStepExecutor } from '../../../executors/base'; 9 | 10 | export class JsonTemplateStepExecutor extends BaseStepExecutor { 11 | private readonly templateEngine: JsonTemplateEngine; 12 | 13 | static parse(template: string, mappings?: boolean, bindings?: Record): Expression { 14 | if (mappings) { 15 | try { 16 | const mappingPaths = JSON.parse(template) as FlatMappingPaths[]; 17 | return JsonTemplateEngine.parse(mappingPaths, { defaultPathType: PathType.JSON }); 18 | } catch (e) { 19 | // parse as template 20 | } 21 | } 22 | return JsonTemplateEngine.parse(template, { 23 | defaultPathType: PathType.SIMPLE, 24 | compileTimeBindings: bindings, 25 | }); 26 | } 27 | 28 | constructor(step: SimpleStep, template: string, bindings?: Record) { 29 | super(step); 30 | const expression = JsonTemplateStepExecutor.parse(template, step.mappings, bindings); 31 | this.templateEngine = JsonTemplateEngine.create(expression, { 32 | compileTimeBindings: bindings, 33 | defaultPathType: PathType.SIMPLE, 34 | }); 35 | } 36 | 37 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 38 | const output = await this.templateEngine.evaluate(input, executionBindings); 39 | return { output }; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/steps/base/simple/factory.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'path'; 2 | import { ExternalWorkflow, SimpleStep, WorkflowOptionsInternal } from '../../../common/types'; 3 | import { CommonUtils } from '../../../common/utils'; 4 | import { WorkflowEngineFactory, WorkflowUtils } from '../../../workflow'; 5 | import { BaseStepExecutor } from '../executors'; 6 | import { ExternalWorkflowStepExecutor } from './executors'; 7 | import { FunctionStepExecutor } from './executors/function'; 8 | import { IdentityStepExecutor } from './executors/identity'; 9 | import { TemplateStepExecutorFactory } from './executors/template'; 10 | 11 | export class SimpleStepExecutorFactory { 12 | static async create( 13 | step: SimpleStep, 14 | options: WorkflowOptionsInternal, 15 | ): Promise { 16 | if (step.identity) { 17 | return new IdentityStepExecutor(step); 18 | } 19 | 20 | if (step.externalWorkflow) { 21 | return this.createExternalWorkflowEngineExecutor(step, options); 22 | } 23 | 24 | if (step.functionName) { 25 | return new FunctionStepExecutor(step, options.currentBindings); 26 | } 27 | 28 | if (step.templatePath) { 29 | // eslint-disable-next-line no-param-reassign 30 | step.template = await this.extractTemplate(options.rootPath, step.templatePath); 31 | } 32 | return TemplateStepExecutorFactory.create(step, step.template as string, options); 33 | } 34 | 35 | private static extractTemplate(rootPath: string, templatePath: string): Promise { 36 | return CommonUtils.readFile(join(rootPath, templatePath)); 37 | } 38 | 39 | private static async createExternalWorkflowEngineExecutor( 40 | step: SimpleStep, 41 | options: WorkflowOptionsInternal, 42 | ): Promise { 43 | const externalWorkflowConfig = step.externalWorkflow as ExternalWorkflow; 44 | const externalWorkflowPath = join(options.rootPath, externalWorkflowConfig.path); 45 | const externalWorkflowRootPath = join(options.rootPath, externalWorkflowConfig.rootPath ?? ''); 46 | const externalWorkflow = await WorkflowUtils.createWorkflowFromFilePath(externalWorkflowPath); 47 | externalWorkflow.bindings = (externalWorkflow.bindings ?? []).concat( 48 | externalWorkflowConfig.bindings ?? [], 49 | ); 50 | const externalWorkflowEngine = await WorkflowEngineFactory.create(externalWorkflow, { 51 | ...options, 52 | parentBindings: options.currentBindings, 53 | rootPath: externalWorkflowRootPath, 54 | }); 55 | return new ExternalWorkflowStepExecutor(externalWorkflowEngine, step); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/steps/base/simple/index.ts: -------------------------------------------------------------------------------- 1 | export * from './executors'; 2 | export * from './factory'; 3 | -------------------------------------------------------------------------------- /src/steps/base/utils.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'path'; 2 | import { StepType, WorkflowOptionsInternal, WorkflowStep } from '../../common'; 3 | import { StepUtils } from '../../common/utils'; 4 | import { StepCreationError } from '../../errors'; 5 | import { WorkflowUtils } from '../../workflow/utils'; 6 | 7 | export class BaseStepUtils { 8 | static async prepareWorkflowStep( 9 | step: WorkflowStep, 10 | options: WorkflowOptionsInternal, 11 | ): Promise { 12 | let newStep = step; 13 | if (step.workflowStepPath) { 14 | const workflowStepPath = join(options.rootPath, step.workflowStepPath); 15 | const workflowStepFromPath = await WorkflowUtils.createFromFilePath( 16 | workflowStepPath, 17 | ); 18 | newStep = { ...workflowStepFromPath, ...step }; 19 | } 20 | BaseStepUtils.validateWorkflowStep(newStep); 21 | return newStep; 22 | } 23 | 24 | static validateWorkflowStep(workflowStep: WorkflowStep) { 25 | if (!workflowStep.steps?.length) { 26 | throw new StepCreationError('Invalid workflow step configuration', workflowStep.name); 27 | } 28 | StepUtils.populateSteps(workflowStep.steps); 29 | StepUtils.validateSteps(workflowStep.steps, [StepType.Workflow]); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/steps/composed/executors/composable.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, Step, StepExecutor, StepOutput } from '../../../common/types'; 2 | 3 | /** 4 | * ComposableStepExecutor allows compose more logic 5 | * on top the given step executor. 6 | */ 7 | export class ComposableStepExecutor implements StepExecutor { 8 | private readonly stepExecutor: StepExecutor; 9 | 10 | constructor(stepExecutor: StepExecutor) { 11 | this.stepExecutor = stepExecutor; 12 | } 13 | 14 | getStep(): Step { 15 | return this.stepExecutor.getStep(); 16 | } 17 | 18 | getStepName(): string { 19 | return this.stepExecutor.getStepName(); 20 | } 21 | 22 | execute(input: any, executionBindings: ExecutionBindings): Promise { 23 | return this.stepExecutor.execute(input, executionBindings); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/steps/composed/executors/conditional.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, StepExecutor, StepOutput } from '../../../common/types'; 2 | import { ComposableStepExecutor } from './composable'; 3 | 4 | export class ConditionalStepExecutor extends ComposableStepExecutor { 5 | private readonly conditionExecutor: StepExecutor; 6 | 7 | private readonly thenExecutor: StepExecutor; 8 | 9 | private readonly elseExecutor?: StepExecutor; 10 | 11 | constructor( 12 | conditionExecutor: StepExecutor, 13 | thenExecutor: StepExecutor, 14 | elseExecutor?: StepExecutor, 15 | ) { 16 | super(thenExecutor); 17 | this.conditionExecutor = conditionExecutor; 18 | this.thenExecutor = thenExecutor; 19 | this.elseExecutor = elseExecutor; 20 | } 21 | 22 | private async shouldExecuteStep( 23 | input: any, 24 | executionBindings: ExecutionBindings, 25 | ): Promise { 26 | const result = await this.conditionExecutor.execute(input, executionBindings); 27 | return result.output; 28 | } 29 | 30 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 31 | const shouldExecuteStep = await this.shouldExecuteStep(input, executionBindings); 32 | if (shouldExecuteStep) { 33 | return this.thenExecutor.execute(input, executionBindings); 34 | } 35 | if (this.elseExecutor) { 36 | return this.elseExecutor.execute(input, executionBindings); 37 | } 38 | return { skipped: true }; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/steps/composed/executors/custom_input.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, StepExecutor, StepOutput } from '../../../common/types'; 2 | import { TemplateStepExecutor } from '../../types'; 3 | import { ComposableStepExecutor } from './composable'; 4 | 5 | /** 6 | * CustomInputStepExecutor customizes the input and 7 | * then invokes step executor with the new input. 8 | */ 9 | export class CustomInputStepExecutor extends ComposableStepExecutor { 10 | private readonly inputTemplateExecutor: TemplateStepExecutor; 11 | 12 | constructor(inputTemplateExecutor: TemplateStepExecutor, nextExecutor: StepExecutor) { 13 | super(nextExecutor); 14 | this.inputTemplateExecutor = inputTemplateExecutor; 15 | } 16 | 17 | private async prepareData(input: any, executionBindings: ExecutionBindings): Promise { 18 | const result = await this.inputTemplateExecutor.execute(input, executionBindings); 19 | return result.output; 20 | } 21 | 22 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 23 | const customInput = await this.prepareData(input, executionBindings); 24 | return super.execute(customInput, executionBindings); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/steps/composed/executors/debuggable.ts: -------------------------------------------------------------------------------- 1 | import { logger } from '../../../common'; 2 | import { ExecutionBindings, StepOutput } from '../../../common/types'; 3 | import { ComposableStepExecutor } from './composable'; 4 | 5 | /** 6 | * DebuggableStepExecutor logs the input and output of step and also 7 | * helps to set break points for debugging. 8 | */ 9 | export class DebuggableStepExecutor extends ComposableStepExecutor { 10 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 11 | logger.mustLog('input = ', JSON.stringify(input)); 12 | logger.mustLog('bindings = ', JSON.stringify(executionBindings)); 13 | const output = await super.execute(input, executionBindings); 14 | logger.mustLog('output = ', JSON.stringify(output)); 15 | return output; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/steps/composed/executors/error_wrap.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, StepOutput } from '../../../common/types'; 2 | import { ErrorUtils, StepExecutionError } from '../../../errors'; 3 | import { ComposableStepExecutor } from './composable'; 4 | 5 | export class ErrorWrapStepExecutor extends ComposableStepExecutor { 6 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 7 | try { 8 | return await super.execute(input, executionBindings); 9 | } catch (error: any) { 10 | if (error instanceof StepExecutionError) { 11 | throw error; 12 | } 13 | throw ErrorUtils.createStepExecutionError(error, this.getStepName()); 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/steps/composed/executors/index.ts: -------------------------------------------------------------------------------- 1 | export * from './composable'; 2 | export * from './conditional'; 3 | export * from './custom_input'; 4 | export * from './debuggable'; 5 | export * from './error_wrap'; 6 | export * from './loop'; 7 | -------------------------------------------------------------------------------- /src/steps/composed/executors/loop.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, LoopStepOutput, StepOutput } from '../../../common/types'; 2 | import { ErrorUtils, StepExecutionError } from '../../../errors'; 3 | import { ComposableStepExecutor } from './composable'; 4 | 5 | export class LoopStepExecutor extends ComposableStepExecutor { 6 | private async executeForInputElement( 7 | element: any, 8 | executionBindings: ExecutionBindings, 9 | ): Promise { 10 | try { 11 | return await super.execute(element, executionBindings); 12 | } catch (error: any) { 13 | const stepExecutionError = ErrorUtils.createStepExecutionError(error, this.getStepName()); 14 | return { 15 | error: { 16 | message: stepExecutionError.message, 17 | status: stepExecutionError.status, 18 | error: stepExecutionError, 19 | originalError: stepExecutionError.originalError, 20 | }, 21 | }; 22 | } 23 | } 24 | 25 | async execute(input: any, executionBindings: ExecutionBindings): Promise { 26 | if (!Array.isArray(input)) { 27 | throw new StepExecutionError('loopOverInput requires array input', 400, { 28 | stepName: this.getStepName(), 29 | }); 30 | } 31 | const promises: Promise[] = new Array(input.length); 32 | for (let i = 0; i < input.length; i += 1) { 33 | promises[i] = this.executeForInputElement(input[i], executionBindings); 34 | } 35 | return { output: await Promise.all(promises) }; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/steps/composed/factory.ts: -------------------------------------------------------------------------------- 1 | import { StepExecutor, WorkflowOptionsInternal } from '../../common/types'; 2 | import { TemplateStepExecutorFactory } from '../base/simple/executors/template'; 3 | import { ConditionalStepExecutor } from './executors/conditional'; 4 | import { CustomInputStepExecutor } from './executors/custom_input'; 5 | import { DebuggableStepExecutor } from './executors/debuggable'; 6 | import { ErrorWrapStepExecutor } from './executors/error_wrap'; 7 | import { LoopStepExecutor } from './executors/loop'; 8 | import { StepExecutorFactory } from '../factory'; 9 | 10 | export class ComposableExecutorFactory { 11 | static async create( 12 | stepExecutor: StepExecutor, 13 | options: WorkflowOptionsInternal, 14 | ): Promise { 15 | const step = stepExecutor.getStep(); 16 | let composedStepExecutor = stepExecutor; 17 | if (step.loopOverInput) { 18 | composedStepExecutor = this.createLoopStepExecutor(composedStepExecutor, options); 19 | } 20 | 21 | if (step.inputTemplate) { 22 | composedStepExecutor = this.createCustomInputStepExecutor(composedStepExecutor, options); 23 | } 24 | 25 | if (step.condition) { 26 | composedStepExecutor = await this.createConditionalStepExecutor( 27 | composedStepExecutor, 28 | options, 29 | ); 30 | } 31 | 32 | if (step.debug) { 33 | composedStepExecutor = new DebuggableStepExecutor(composedStepExecutor); 34 | } 35 | composedStepExecutor = new ErrorWrapStepExecutor(composedStepExecutor); 36 | return composedStepExecutor; 37 | } 38 | 39 | static createCustomInputStepExecutor( 40 | stepExecutor: StepExecutor, 41 | options: WorkflowOptionsInternal, 42 | ): CustomInputStepExecutor { 43 | const step = stepExecutor.getStep(); 44 | const templateExecutor = TemplateStepExecutorFactory.create( 45 | step, 46 | step.inputTemplate as string, 47 | options, 48 | ); 49 | return new CustomInputStepExecutor(templateExecutor, stepExecutor); 50 | } 51 | 52 | static async createConditionalStepExecutor( 53 | thenExecutor: StepExecutor, 54 | options: WorkflowOptionsInternal, 55 | ): Promise { 56 | const step = thenExecutor.getStep(); 57 | const condtionalExecutor = TemplateStepExecutorFactory.create( 58 | step, 59 | step.condition as string, 60 | options, 61 | ); 62 | let elseExecutor: StepExecutor | undefined; 63 | if (step.else) { 64 | elseExecutor = await StepExecutorFactory.create(step.else, options); 65 | } 66 | return new ConditionalStepExecutor(condtionalExecutor, thenExecutor, elseExecutor); 67 | } 68 | 69 | static createLoopStepExecutor( 70 | stepExecutor: StepExecutor, 71 | options: WorkflowOptionsInternal, 72 | ): LoopStepExecutor { 73 | const step = stepExecutor.getStep(); 74 | let wrappedStepExecutor = stepExecutor; 75 | if (step.loopCondition) { 76 | const condtionalExecutor = TemplateStepExecutorFactory.create( 77 | step, 78 | step.loopCondition, 79 | options, 80 | ); 81 | wrappedStepExecutor = new ConditionalStepExecutor(condtionalExecutor, wrappedStepExecutor); 82 | } 83 | return new LoopStepExecutor(wrappedStepExecutor); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/steps/composed/index.ts: -------------------------------------------------------------------------------- 1 | export * from './executors'; 2 | export * from './factory'; 3 | -------------------------------------------------------------------------------- /src/steps/factory.ts: -------------------------------------------------------------------------------- 1 | import { Step, StepExecutor, WorkflowOptionsInternal } from '../common'; 2 | import { StepCreationError } from '../errors'; 3 | import { BaseStepExecutorFactory } from './base/factory'; 4 | import { ComposableExecutorFactory } from './composed'; 5 | 6 | export class StepExecutorFactory { 7 | static async create(step: Step, options: WorkflowOptionsInternal): Promise { 8 | try { 9 | let stepExecutor: StepExecutor = await BaseStepExecutorFactory.create(step, options); 10 | stepExecutor = await ComposableExecutorFactory.create(stepExecutor, options); 11 | return stepExecutor; 12 | } catch (error: any) { 13 | if (error instanceof StepCreationError) { 14 | throw error; 15 | } 16 | throw new StepCreationError(error.message, step.name); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/steps/index.ts: -------------------------------------------------------------------------------- 1 | export * from './base'; 2 | export * from './composed'; 3 | export * from './factory'; 4 | export * from './types'; 5 | -------------------------------------------------------------------------------- /src/steps/types.ts: -------------------------------------------------------------------------------- 1 | import { BatchResult, CustomStep, ExecutionBindings } from '../common'; 2 | import { 3 | type JsonTemplateStepExecutor, 4 | type JsonataStepExecutor, 5 | } from './base/simple/executors/template'; 6 | 7 | export interface BatchExecutor { 8 | execute(input: any[], bindings: ExecutionBindings): Promise; 9 | } 10 | 11 | export interface CustomStepExecutor { 12 | execute(input: any, bindings: ExecutionBindings, params?: Record): Promise; 13 | } 14 | export interface CustomStepExecutorProvider { 15 | provide(step: CustomStep): Promise; 16 | } 17 | 18 | export type TemplateStepExecutor = JsonTemplateStepExecutor | JsonataStepExecutor; 19 | -------------------------------------------------------------------------------- /src/workflow/default_executor.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ExecutionBindings, 3 | StepExitAction, 4 | WorkflowEngine, 5 | WorkflowExecutor, 6 | WorkflowOutput, 7 | logger, 8 | } from '../common'; 9 | import { ErrorUtils, WorkflowExecutionError } from '../errors'; 10 | 11 | interface DefaultWorkflowExecutorOptions { 12 | // In chainOutputs use set then => input -> step -> step2 -> step3 -> output 13 | chainOutputs?: boolean; 14 | } 15 | 16 | export class DefaultWorkflowExecutor implements WorkflowExecutor { 17 | readonly options: DefaultWorkflowExecutorOptions; 18 | 19 | constructor(options?: DefaultWorkflowExecutorOptions) { 20 | this.options = options ?? {}; 21 | } 22 | 23 | static readonly INSTANCE = new DefaultWorkflowExecutor(); 24 | 25 | private static handleError(error: any, workflowName: string, stepName: string) { 26 | throw new WorkflowExecutionError( 27 | error.message, 28 | ErrorUtils.getErrorStatus(error), 29 | workflowName, 30 | { 31 | stepName, 32 | childStepName: error.childStepName, 33 | error: error.error, 34 | }, 35 | ); 36 | } 37 | 38 | async execute( 39 | engine: WorkflowEngine, 40 | input: any, 41 | bindings?: Record, 42 | ): Promise { 43 | const context = {}; 44 | const executionBindings: ExecutionBindings = { 45 | ...engine.getBindings(), 46 | ...bindings, 47 | outputs: {}, 48 | context, 49 | setContext: (key, value) => { 50 | context[key] = value; 51 | }, 52 | originalInput: input, 53 | }; 54 | 55 | let prevStepOutput: any; 56 | let currStepInput: any = input; 57 | const stepExecutors = engine.getStepExecutors(); 58 | 59 | for (const stepExecutor of stepExecutors) { 60 | const step = stepExecutor.getStep(); 61 | try { 62 | // eslint-disable-next-line no-await-in-loop 63 | const { skipped, output } = await stepExecutor.execute(currStepInput, executionBindings); 64 | if (!skipped) { 65 | prevStepOutput = output; 66 | executionBindings.outputs[step.name] = output; 67 | if (this.options.chainOutputs) { 68 | currStepInput = prevStepOutput; 69 | } 70 | if (step.onComplete === StepExitAction.Return) { 71 | break; 72 | } 73 | } 74 | } catch (error) { 75 | logger.error(`step: ${step.name} failed with error:`, error); 76 | if (step.onError !== StepExitAction.Continue) { 77 | DefaultWorkflowExecutor.handleError(error, engine.getName(), step.name); 78 | } 79 | } 80 | } 81 | 82 | return { output: prevStepOutput, outputs: executionBindings.outputs }; 83 | } 84 | } 85 | 86 | export const chainExecutor = new DefaultWorkflowExecutor({ chainOutputs: true }); 87 | -------------------------------------------------------------------------------- /src/workflow/engine.ts: -------------------------------------------------------------------------------- 1 | import { StepExecutor, WorkflowEngine, WorkflowExecutor, WorkflowOutput } from '../common'; 2 | import { ErrorUtils } from '../errors'; 3 | 4 | export class DefaultWorkflowEngine implements WorkflowEngine { 5 | private readonly name: string; 6 | 7 | private readonly bindings: Record; 8 | 9 | private readonly stepExecutors: StepExecutor[]; 10 | 11 | private readonly executor: WorkflowExecutor; 12 | 13 | constructor( 14 | name: string, 15 | executor: WorkflowExecutor, 16 | bindings: Record, 17 | stepExecutors: StepExecutor[], 18 | ) { 19 | this.bindings = bindings; 20 | this.name = name; 21 | this.stepExecutors = stepExecutors; 22 | this.executor = executor; 23 | } 24 | 25 | getName(): string { 26 | return this.name; 27 | } 28 | 29 | getBindings(): Record { 30 | return this.bindings; 31 | } 32 | 33 | getStepExecutors(): StepExecutor[] { 34 | return this.stepExecutors; 35 | } 36 | 37 | getStepExecutor(stepName: string): StepExecutor { 38 | const stepExecutor = this.stepExecutors.find((executor) => executor.getStepName() === stepName); 39 | 40 | if (!stepExecutor) { 41 | throw new Error(`${stepName} was not found`); 42 | } 43 | 44 | return stepExecutor; 45 | } 46 | 47 | async execute(input: any, executionBindings?: Record): Promise { 48 | try { 49 | return await this.executor.execute(this, input, executionBindings); 50 | } catch (error: any) { 51 | throw ErrorUtils.createWorkflowExecutionError(error, this.name); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/workflow/factory.ts: -------------------------------------------------------------------------------- 1 | import { StepExecutorFactory } from '../steps/factory'; 2 | import * as libraryBindings from '../bindings'; 3 | import { 4 | Binding, 5 | Step, 6 | StepExecutor, 7 | Workflow, 8 | WorkflowEngine, 9 | WorkflowOptions, 10 | WorkflowOptionsInternal, 11 | } from '../common/types'; 12 | import { StepUtils } from '../common/utils'; 13 | import { StepCreationError, WorkflowCreationError } from '../errors'; 14 | import { DefaultWorkflowEngine } from './engine'; 15 | import { WorkflowUtils } from './utils'; 16 | 17 | export class WorkflowEngineFactory { 18 | private static prepareWorkflow(workflow: Workflow, options: WorkflowOptions) { 19 | WorkflowUtils.validateWorkflow(workflow); 20 | // eslint-disable-next-line no-param-reassign 21 | options.templateType = workflow.templateType ?? options.templateType; 22 | StepUtils.populateSteps(workflow.steps); 23 | StepUtils.validateSteps(workflow.steps); 24 | } 25 | 26 | static async create( 27 | workflow: Workflow, 28 | options: WorkflowOptions | WorkflowOptionsInternal, 29 | ): Promise { 30 | try { 31 | this.prepareWorkflow(workflow, options); 32 | const optionsInteranl = options as WorkflowOptionsInternal; 33 | const bindings = await this.prepareBindings(workflow.bindings ?? [], optionsInteranl); 34 | optionsInteranl.currentBindings = bindings; 35 | const executor = await WorkflowUtils.getExecutor(workflow, optionsInteranl); 36 | const stepExecutors = await this.createStepExecutors(workflow.steps, optionsInteranl); 37 | WorkflowUtils.validate(workflow); 38 | return new DefaultWorkflowEngine(workflow.name, executor, bindings, stepExecutors); 39 | } catch (error: any) { 40 | if (error instanceof WorkflowCreationError) { 41 | throw error; 42 | } 43 | 44 | if (error instanceof StepCreationError) { 45 | throw new WorkflowCreationError( 46 | error.message, 47 | workflow.name, 48 | error.stepName, 49 | error.childStepName, 50 | ); 51 | } 52 | 53 | throw new WorkflowCreationError(error.message, workflow.name); 54 | } 55 | } 56 | 57 | static async createFromYaml( 58 | yamlString: string, 59 | options: WorkflowOptions, 60 | ): Promise { 61 | const workflow = WorkflowUtils.createFromYaml(yamlString); 62 | return this.create(workflow, options); 63 | } 64 | 65 | static async createFromFilePath( 66 | workflowPath: string, 67 | options: WorkflowOptions, 68 | ): Promise { 69 | const workflow = await WorkflowUtils.createWorkflowFromFilePath(workflowPath); 70 | return this.create(workflow, options); 71 | } 72 | 73 | private static async prepareBindings( 74 | workflowBindings: Binding[], 75 | options: WorkflowOptionsInternal, 76 | ): Promise> { 77 | return { 78 | ...libraryBindings, 79 | ...(await WorkflowUtils.extractWorkflowOptionsBindings(options)), 80 | ...(await WorkflowUtils.extractBindings(options, workflowBindings)), 81 | ...options.creationTimeBindings, 82 | }; 83 | } 84 | 85 | private static async createStepExecutors( 86 | steps: Step[], 87 | options: WorkflowOptionsInternal, 88 | ): Promise { 89 | return Promise.all(steps.map((step) => StepExecutorFactory.create(step, options))); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/workflow/index.ts: -------------------------------------------------------------------------------- 1 | export * from './default_executor'; 2 | export * from './engine'; 3 | export * from './factory'; 4 | export * from './utils'; 5 | -------------------------------------------------------------------------------- /src/workflow/output_validator.ts: -------------------------------------------------------------------------------- 1 | import { BatchStep, SimpleStep, Step, StepType, Workflow, WorkflowStep } from '../common'; 2 | import { WorkflowCreationError } from '../errors'; 3 | 4 | // Matches: $.outputs.stepName, 5 | // Result: [$.outputs.stepName, stepName, stepName]; 6 | // Matches: $.outputs.workflow.step, 7 | // Result: [$.outputs.workflow.step, workflow.step, workflow, .step, step]; 8 | const regexOutputReference = /\$\.?outputs\.((\w+)(\.(\w+))?)/g; 9 | 10 | export class WorkflowOutputsValidator { 11 | private readonly workflow: Workflow; 12 | 13 | private readonly seenSteps: Set; 14 | 15 | private readonly stepTypeMap: Map; 16 | 17 | constructor(workflow: Workflow) { 18 | this.workflow = workflow; 19 | this.seenSteps = new Set(); 20 | this.stepTypeMap = new Map(); 21 | } 22 | 23 | validateWorkflowOutputReference(match: RegExpMatchArray, stepName: string, parentName?: string) { 24 | const workflowName = match[2]; // The name of the workflow step 25 | // Access to the child step outputs is restricted to within the same parent workflow step; 26 | if (parentName !== workflowName) { 27 | throw new WorkflowCreationError( 28 | `Invalid output reference: ${match[0]}, step is not a child of ${parentName}`, 29 | this.workflow.name, 30 | parentName, 31 | stepName, 32 | ); 33 | } 34 | } 35 | 36 | validateExistanceOfOutputReference( 37 | match: RegExpMatchArray, 38 | stepName: string, 39 | parentName?: string, 40 | ) { 41 | const fullOutputName = match[1]; // For workflow step 42 | const outputStepName = match[2]; // For simple step 43 | // Check the referenced step output is already executed. 44 | if (!this.seenSteps.has(fullOutputName) && !this.seenSteps.has(outputStepName)) { 45 | throw new WorkflowCreationError( 46 | `Invalid output reference: ${match[0]}, step is not executed yet.`, 47 | this.workflow.name, 48 | parentName ?? stepName, 49 | parentName ? stepName : undefined, 50 | ); 51 | } 52 | } 53 | 54 | validateOutputReferences(stepName: string, template?: string, parentName?: string) { 55 | if (!template) { 56 | return; 57 | } 58 | const outputMatches = [...template.matchAll(regexOutputReference)]; 59 | 60 | // Multiple outputs may exist within the template so we need a loop. 61 | // In this case, we are looking for workflow step output references. 62 | // Format: $.outputs.workflowStepName.ChildStepName 63 | for (const match of outputMatches) { 64 | const primaryStepName = match[2]; // The name of the step 65 | const childStepName = match[4]; // The name of the child step 66 | if (this.stepTypeMap.get(primaryStepName) === StepType.Workflow && childStepName) { 67 | this.validateWorkflowOutputReference(match, stepName, parentName); 68 | } 69 | this.validateExistanceOfOutputReference(match, stepName, parentName); 70 | } 71 | } 72 | 73 | validateCommonStepParams(step: Step, parentName?: string) { 74 | this.validateOutputReferences(step.name, step.condition, parentName); 75 | this.validateOutputReferences(step.name, step.inputTemplate, parentName); 76 | this.validateOutputReferences(step.name, step.loopCondition, parentName); 77 | if (step.else) { 78 | this.validateSteps([step.else], parentName); 79 | } 80 | } 81 | 82 | validateBatchStep(step: BatchStep, parentName?: string) { 83 | if (step.batches) { 84 | for (const batch of step.batches) { 85 | this.validateOutputReferences(step.name, batch.filter, parentName); 86 | this.validateOutputReferences(step.name, batch.map, parentName); 87 | } 88 | } 89 | } 90 | 91 | validateSteps(steps: Step[], parentName?: string) { 92 | for (const step of steps) { 93 | const stepName = step.name; 94 | const stepType = step.type as StepType; 95 | this.stepTypeMap.set(stepName, stepType); 96 | let outputName = stepName; 97 | this.validateCommonStepParams(step, parentName); 98 | if (step.type === StepType.Workflow) { 99 | const workflowStep = step as WorkflowStep; 100 | if (workflowStep.steps) { 101 | this.validateSteps(workflowStep.steps, stepName); 102 | } 103 | } else if (step.type === StepType.Simple) { 104 | this.validateOutputReferences(stepName, (step as SimpleStep).template, parentName); 105 | } else if (step.type === StepType.Batch) { 106 | this.validateBatchStep(step as BatchStep, parentName); 107 | } 108 | if (parentName) { 109 | outputName = `${parentName}.${stepName}`; 110 | } 111 | this.seenSteps.add(outputName); 112 | } 113 | } 114 | 115 | validateOutputs() { 116 | const { steps: workflowSteps } = this.workflow; 117 | this.validateSteps(workflowSteps); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/workflow/utils.ts: -------------------------------------------------------------------------------- 1 | import { pick } from 'lodash'; 2 | import path from 'path'; 3 | import yaml from 'yaml'; 4 | import { toArray } from '../bindings/common'; 5 | import { 6 | Binding, 7 | CommonUtils, 8 | Workflow, 9 | WorkflowBindingProvider, 10 | WorkflowExecutor, 11 | WorkflowOptionsInternal, 12 | } from '../common'; 13 | import { BindingNotFoundError, WorkflowCreationError } from '../errors'; 14 | import { DefaultWorkflowExecutor } from './default_executor'; 15 | import { WorkflowOutputsValidator } from './output_validator'; 16 | 17 | export class WorkflowUtils { 18 | private static populateWorkflowName(workflow: Workflow, workflowPath: string) { 19 | if (!workflow.name) { 20 | const { name } = path.parse(workflowPath); 21 | // eslint-disable-next-line no-param-reassign 22 | workflow.name = name; 23 | } 24 | } 25 | 26 | static async createWorkflowFromFilePath(yamlPath: string): Promise { 27 | const workflow = (await this.createFromFilePath(yamlPath)) ?? {}; 28 | this.populateWorkflowName(workflow, yamlPath); 29 | return workflow; 30 | } 31 | 32 | static async createFromFilePath(yamlPath: string): Promise { 33 | const yamlString = await CommonUtils.readFile(yamlPath); 34 | return this.createFromYaml(yamlString); 35 | } 36 | 37 | static createFromYaml(yamlString: string): T { 38 | return yaml.parse(yamlString) as T; 39 | } 40 | 41 | static validateWorkflow(workflow: Workflow) { 42 | if (!workflow?.steps?.length) { 43 | throw new WorkflowCreationError('Workflow should contain at least one step', workflow.name); 44 | } 45 | } 46 | 47 | static validate(workflow: Workflow) { 48 | const validator = new WorkflowOutputsValidator(workflow); 49 | validator.validateOutputs(); 50 | } 51 | 52 | private static async getModuleExports(modulePath: string): Promise { 53 | let moduleExports; 54 | try { 55 | moduleExports = await import(modulePath); 56 | } catch (error: any) { 57 | if (error.code !== 'MODULE_NOT_FOUND') { 58 | throw error; 59 | } 60 | } 61 | return moduleExports; 62 | } 63 | 64 | private static async getModuleExportsFromProvider( 65 | modulePath: string, 66 | provider: WorkflowBindingProvider, 67 | ): Promise { 68 | let moduleExports; 69 | try { 70 | moduleExports = await provider.provide(modulePath); 71 | } catch (error: any) { 72 | // Ignore error 73 | } 74 | return moduleExports; 75 | } 76 | 77 | private static async getModuleExportsFromBindingsPath(bindingPath: string): Promise { 78 | return ( 79 | (await this.getModuleExports(bindingPath)) ?? 80 | (await this.getModuleExports(path.join(process.cwd(), bindingPath))) 81 | ); 82 | } 83 | 84 | private static async getModuleExportsFromAllPaths( 85 | bindingPath: string, 86 | options: WorkflowOptionsInternal, 87 | ): Promise { 88 | const binding = 89 | (await this.getModuleExports(path.join(options.rootPath, bindingPath))) ?? 90 | (options.bindingProvider 91 | ? await this.getModuleExportsFromProvider(bindingPath, options.bindingProvider) 92 | : await this.getModuleExportsFromBindingsPath(bindingPath)); 93 | if (!binding) { 94 | throw new BindingNotFoundError(bindingPath); 95 | } 96 | return binding; 97 | } 98 | 99 | static async extractWorkflowOptionsBindings( 100 | options: WorkflowOptionsInternal, 101 | ): Promise> { 102 | if (!options.bindingsPaths?.length) { 103 | return {}; 104 | } 105 | 106 | const bindings = await Promise.all( 107 | options.bindingsPaths.map(async (bindingPath) => 108 | this.getModuleExportsFromAllPaths(bindingPath, options), 109 | ), 110 | ); 111 | return Object.assign({}, ...bindings); 112 | } 113 | 114 | static async extractBinding( 115 | binding: Binding, 116 | options: WorkflowOptionsInternal, 117 | ): Promise> { 118 | const parentBindings = options.parentBindings ?? {}; 119 | const { name, value, path: bidningPath, fromParent, exportAll } = binding as any; 120 | if (fromParent) { 121 | return { [name]: parentBindings[name] }; 122 | } 123 | if (value) { 124 | return { [name]: value }; 125 | } 126 | const bindingSource = await this.getModuleExportsFromAllPaths( 127 | bidningPath ?? 'bindings', 128 | options, 129 | ); 130 | if (!name) { 131 | return bindingSource; 132 | } 133 | const names = toArray(name) as string[]; 134 | if (names.length === 1) { 135 | return { [name]: exportAll ? bindingSource : bindingSource[name] }; 136 | } 137 | return pick(bindingSource, names); 138 | } 139 | 140 | static async extractBindings( 141 | options: WorkflowOptionsInternal, 142 | bindings: Binding[] = [], 143 | ): Promise> { 144 | if (bindings.length === 0) { 145 | return {}; 146 | } 147 | const bindingsData = await Promise.all( 148 | bindings.map(async (binding) => this.extractBinding(binding, options)), 149 | ); 150 | return Object.assign({}, ...bindingsData); 151 | } 152 | 153 | static async getExecutor( 154 | workflow: Workflow, 155 | options: WorkflowOptionsInternal, 156 | ): Promise { 157 | if (workflow?.executor) { 158 | const executor = options.currentBindings[workflow.executor] as WorkflowExecutor; 159 | if (typeof executor?.execute !== 'function') { 160 | throw new WorkflowCreationError('Workflow executor not found', workflow.executor); 161 | } 162 | return executor; 163 | } 164 | return options.executor ?? DefaultWorkflowExecutor.INSTANCE; 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /stryker.conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@stryker-mutator/core/schema/stryker-schema.json", 3 | "_comment": "This config was generated using 'stryker init'. Please take a look at: https://stryker-mutator.io/docs/stryker-js/configuration/ for more information.", 4 | "packageManager": "npm", 5 | "reporters": ["html", "clear-text", "progress"], 6 | "testRunner": "command", 7 | "testRunner_comment": "Take a look at (missing 'homepage' URL in package.json) for information about the command plugin.", 8 | "coverageAnalysis": "all", 9 | "buildCommand": "npm run build", 10 | "mutate": [ 11 | "{src,lib}/**/!(*.+(s|S)pec|*.+(t|T)est).+(cjs|mjs|js|ts|jsx|tsx|html|vue)", 12 | "!{src,lib}/**/__tests__/**/*.+(cjs|mjs|js|ts|jsx|tsx|html|vue)" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /test/custom_scenarios/loop_over_input/wrapped_error/external_workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: sample 3 | debug: true 4 | template: | 5 | .error ? $.doThrow(.error.message, 500); 6 | .output ? .output : { "foo": "bar" } 7 | -------------------------------------------------------------------------------- /test/custom_scenarios/loop_over_input/wrapped_error/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: loopOverInput 3 | loopOverInput: true 4 | externalWorkflow: 5 | path: ./external_workflow.yaml 6 | -------------------------------------------------------------------------------- /test/e2e-custom.test.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'path'; 2 | import { WorkflowEngineFactory } from '../src'; 3 | 4 | const rootDirName = 'custom_scenarios'; 5 | 6 | describe('Custom Scenarios tests', () => { 7 | describe('loop over input', () => { 8 | it('should return original error when error wrapped', async () => { 9 | const scenarioDir = join(__dirname, rootDirName, 'loop_over_input', 'wrapped_error'); 10 | const workflowPath = join(scenarioDir, 'workflow.yaml'); 11 | const workflowEngine = await WorkflowEngineFactory.createFromFilePath(workflowPath, { 12 | rootPath: scenarioDir, 13 | }); 14 | const result = await workflowEngine.execute([ 15 | { 16 | error: { 17 | message: 'some error', 18 | }, 19 | }, 20 | ]); 21 | expect(result.output[0].error?.error.message).toEqual('some error'); 22 | expect(result.output[0].error?.originalError.message).toEqual('some error'); 23 | }); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /test/e2e.test.ts: -------------------------------------------------------------------------------- 1 | import { Command } from 'commander'; 2 | import { readdirSync } from 'fs'; 3 | import { join } from 'path'; 4 | import { logger } from '../src'; 5 | import { ScenarioUtils } from './utils'; 6 | import { CommonUtils } from './utils/common'; 7 | 8 | const rootDirName = 'scenarios'; 9 | const command = new Command(); 10 | command.allowUnknownOption().option('--scenarios ', 'Enter Scenario Names', 'all').parse(); 11 | 12 | const opts = command.opts(); 13 | let scenarios = opts.scenarios.split(/[, ]/); 14 | 15 | if (scenarios[0] === 'all') { 16 | scenarios = readdirSync(join(__dirname, rootDirName)); 17 | } 18 | 19 | describe('Scenarios tests', () => { 20 | scenarios.forEach((scenario) => { 21 | describe(`${scenario}`, () => { 22 | const scenarioDir = join(__dirname, rootDirName, scenario); 23 | const scenarios = ScenarioUtils.extractScenarios(scenarioDir); 24 | scenarios.forEach((scenario, index) => { 25 | it(`Scenario ${index}: ${scenario.workflowPath ?? 'workflow.yaml'}`, async () => { 26 | const previousLogLevel = logger.getLogLevel(); 27 | if (scenario.logLevel !== undefined) { 28 | logger.setLogLevel(scenario.logLevel); 29 | } 30 | try { 31 | const workflowEngine = await ScenarioUtils.createWorkflowEngine(scenarioDir, scenario); 32 | const result = await ScenarioUtils.executeScenario(workflowEngine, scenario); 33 | expect(result.output).toEqual(scenario.output); 34 | } catch (error: any) { 35 | CommonUtils.matchError(error, scenario.error); 36 | } finally { 37 | if (scenario.logLevel !== undefined) { 38 | logger.setLogLevel(previousLogLevel); 39 | } 40 | } 41 | }); 42 | }); 43 | }); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /test/scenario.test.ts: -------------------------------------------------------------------------------- 1 | import { Command } from 'commander'; 2 | import { join } from 'path'; 3 | import { WorkflowOutput } from '../src'; 4 | import { Scenario } from './types'; 5 | import { CommonUtils, ScenarioUtils } from './utils'; 6 | 7 | // Run: npm run test:scenario -- --scenario=batch_step --index=1 8 | const command = new Command(); 9 | command 10 | .allowUnknownOption() 11 | .option('-s, --scenario ', 'Enter Scenario Name') 12 | .option('-i, --index ', 'Enter Test case index') 13 | .parse(); 14 | 15 | const opts = command.opts(); 16 | const scenarioName = opts.scenario ?? 'none'; 17 | const index = +(opts.index ?? 0); 18 | 19 | describe(`${scenarioName}:`, () => { 20 | it(`Scenario ${index}`, async () => { 21 | if (scenarioName === 'none') { 22 | return; 23 | } 24 | const scenarioDir = join(__dirname, 'scenarios', scenarioName); 25 | const scenarios = ScenarioUtils.extractScenarios(scenarioDir); 26 | const scenario: Scenario = scenarios[index] ?? scenarios[0]; 27 | let result: WorkflowOutput = {}; 28 | try { 29 | console.log( 30 | `Executing scenario: ${scenarioName}, test: ${index}, workflow: ${ 31 | scenario.workflowPath ?? 'workflow.yaml' 32 | }`, 33 | ); 34 | const workflowEngine = await ScenarioUtils.createWorkflowEngine(scenarioDir, scenario); 35 | result = await ScenarioUtils.executeScenario(workflowEngine, scenario); 36 | expect(result.output).toEqual(scenario.output); 37 | } catch (error: any) { 38 | console.log('Actual result', JSON.stringify(result.output, null, 2)); 39 | console.log('Expected result', JSON.stringify(scenario.output, null, 2)); 40 | CommonUtils.matchError(error, scenario.error); 41 | } 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /test/scenarios/basic_workflow/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "a": 10, 5 | "b": 2, 6 | "op": "+" 7 | }, 8 | "output": 12 9 | }, 10 | { 11 | "input": { 12 | "a": 10, 13 | "b": 2, 14 | "op": "-" 15 | }, 16 | "output": 8 17 | }, 18 | { 19 | "input": { 20 | "a": 10, 21 | "b": 2, 22 | "op": "*" 23 | }, 24 | "output": 20 25 | }, 26 | { 27 | "input": { 28 | "a": 10, 29 | "b": 2, 30 | "op": "/" 31 | }, 32 | "output": 5 33 | }, 34 | { 35 | "input": { 36 | "a": 10, 37 | "b": 0, 38 | "op": "/" 39 | }, 40 | "error": { 41 | "message": "division by zero is not allowed" 42 | } 43 | }, 44 | { 45 | "input": { 46 | "a": 10, 47 | "b": 0, 48 | "op": "^" 49 | }, 50 | "error": { 51 | "message": "unsupported operation" 52 | } 53 | } 54 | ] 55 | -------------------------------------------------------------------------------- /test/scenarios/basic_workflow/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: unsupported 3 | condition: $not(op in ["+", "-", "*", "/"]) 4 | template: | 5 | $doThrow("unsupported operation") 6 | - name: add 7 | description: Do addition 8 | condition: op = "+" 9 | template: | 10 | ( a + b ) 11 | - name: subtract 12 | description: Do subtraction 13 | condition: op = "-" 14 | template: | 15 | ( a - b ) 16 | - name: multiply 17 | description: Do multiplication 18 | condition: op = "*" 19 | template: | 20 | ( a * b ) 21 | - name: divide 22 | description: Do division 23 | condition: op = "/" 24 | template: | 25 | ( 26 | $assert( b != 0, "division by zero is not allowed"); 27 | a / b 28 | ) 29 | -------------------------------------------------------------------------------- /test/scenarios/batch_step/bindings.ts: -------------------------------------------------------------------------------- 1 | import { SimpleBatchExecutor } from '../../../src/steps/base/batch/simple_batch_executor'; 2 | 3 | export const batchExecutor = new SimpleBatchExecutor({ options: { length: 2 }, key: 'one' }); 4 | -------------------------------------------------------------------------------- /test/scenarios/batch_step/using_executor.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: batchExecutor 3 | steps: 4 | - name: batchData 5 | type: batch 6 | executor: batchExecutor 7 | -------------------------------------------------------------------------------- /test/scenarios/batch_step/workflow.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: batchData 4 | type: batch 5 | batches: 6 | - key: heros 7 | options: 8 | length: 3 9 | filter: .type === "hero" 10 | map: .name 11 | - key: villains 12 | options: 13 | size: 100 14 | filter: .type === "villain" 15 | - key: gods 16 | disabled: true 17 | filter: .type === "god" 18 | map: .name 19 | -------------------------------------------------------------------------------- /test/scenarios/bindings/array_bindings.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | bindings: 3 | - name: 4 | - add 5 | - multiply 6 | path: ./test/scenarios/bindings_paths/functions.ts 7 | steps: 8 | - name: simple 9 | template: | 10 | $.add(1, $.multiply(2,5)) 11 | -------------------------------------------------------------------------------- /test/scenarios/bindings/bindings_using_current_dir.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | bindings: 3 | - name: add 4 | path: ./test/scenarios/bindings_paths/functions.ts 5 | steps: 6 | - name: simple 7 | template: | 8 | $.add(1, 2) 9 | -------------------------------------------------------------------------------- /test/scenarios/bindings/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "array_bindings.yaml", 4 | "output": 11 5 | }, 6 | { 7 | "workflowPath": "bindings_using_current_dir.yaml", 8 | "output": 3 9 | }, 10 | { 11 | "workflowPath": "execution_bindings.yaml", 12 | "executionBindings": { 13 | "a": 10, 14 | "b": 2 15 | }, 16 | "output": 12 17 | }, 18 | { 19 | "input": { 20 | "a": 10, 21 | "b": 2, 22 | "op": "+" 23 | }, 24 | "output": 12 25 | }, 26 | { 27 | "input": { 28 | "a": 10, 29 | "b": 2, 30 | "op": "-" 31 | }, 32 | "output": 8 33 | }, 34 | { 35 | "input": { 36 | "a": 10, 37 | "b": 2, 38 | "op": "*" 39 | }, 40 | "output": 20 41 | }, 42 | { 43 | "input": { 44 | "a": 10, 45 | "b": 2, 46 | "op": "/" 47 | }, 48 | "output": 5 49 | }, 50 | { 51 | "input": { 52 | "a": 10, 53 | "b": 0, 54 | "op": "/" 55 | }, 56 | "error": { 57 | "message": "division by zero is not allowed" 58 | } 59 | }, 60 | { 61 | "input": { 62 | "a": 10, 63 | "b": 0, 64 | "op": "^" 65 | }, 66 | "error": { 67 | "message": "unsupported operation" 68 | } 69 | }, 70 | { 71 | "workflowPath": "./external_library.yaml", 72 | "input": [[1], [2, 3]], 73 | "output": [1, 2, 3] 74 | }, 75 | { 76 | "workflowPath": "invalid_binding.yaml", 77 | "error": { 78 | "message": "Invalid binding" 79 | } 80 | }, 81 | { 82 | "workflowPath": "non_existant_binding.yaml", 83 | "error": { 84 | "message": "Binding not found" 85 | } 86 | } 87 | ] 88 | -------------------------------------------------------------------------------- /test/scenarios/bindings/execution_bindings.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: testExecutionBindings 4 | template: | 5 | $.a + $.b 6 | -------------------------------------------------------------------------------- /test/scenarios/bindings/external_library.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: flatten 3 | path: lodash 4 | 5 | steps: 6 | - name: flatten 7 | template: | 8 | $flatten($) 9 | -------------------------------------------------------------------------------- /test/scenarios/bindings/functions.ts: -------------------------------------------------------------------------------- 1 | export const add = (a, b) => a + b; 2 | export const subtract = (a, b) => a - b; 3 | export const multiply = (a, b) => a * b; 4 | export const divide = (a, b) => a / b; 5 | -------------------------------------------------------------------------------- /test/scenarios/bindings/invalid_binding.ts: -------------------------------------------------------------------------------- 1 | throw new Error('Invalid binding'); 2 | -------------------------------------------------------------------------------- /test/scenarios/bindings/invalid_binding.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: ./invalid_binding 3 | name: foo 4 | 5 | steps: 6 | - name: dummy 7 | template: | 8 | $ 9 | -------------------------------------------------------------------------------- /test/scenarios/bindings/non_existant_binding.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: ./non-existant 3 | name: foo 4 | 5 | steps: 6 | - name: dummy 7 | template: | 8 | $ 9 | -------------------------------------------------------------------------------- /test/scenarios/bindings/ops.ts: -------------------------------------------------------------------------------- 1 | export const plus = '+'; 2 | export const minus = '-'; 3 | export const multiply = '*'; 4 | export const divide = '/'; 5 | -------------------------------------------------------------------------------- /test/scenarios/bindings/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: plus 3 | path: ./ops 4 | - name: ops 5 | path: ./ops 6 | exportAll: true 7 | - path: ./functions 8 | steps: 9 | - name: unsupported 10 | condition: $not(op in ["+", "-", "*", "/"]) 11 | template: | 12 | $doThrow("unsupported operation") 13 | - name: add 14 | description: Do addition 15 | condition: op = $plus 16 | template: | 17 | $add(a, b) 18 | - name: subtract 19 | description: Do subtraction 20 | condition: op = $ops.minus 21 | template: | 22 | $subtract(a, b) 23 | - name: multiply 24 | description: Do multiplication 25 | condition: op = $ops.multiply 26 | template: | 27 | $multiply(a, b) 28 | - name: divide 29 | description: Do division 30 | condition: op = $ops.divide 31 | template: | 32 | ( 33 | $assert( b != 0, "division by zero is not allowed"); 34 | $divide(a, b) 35 | ) 36 | -------------------------------------------------------------------------------- /test/scenarios/bindings_paths/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "options": { 4 | "bindingsPaths": ["functions", "ops"] 5 | }, 6 | "input": { 7 | "a": 10, 8 | "b": 2, 9 | "op": "+" 10 | }, 11 | "output": 12 12 | }, 13 | { 14 | "options": { 15 | "bindingsPaths": ["functions", "ops"] 16 | }, 17 | "input": { 18 | "a": 10, 19 | "b": 2, 20 | "op": "-" 21 | }, 22 | "output": 8 23 | }, 24 | { 25 | "options": { 26 | "bindingsPaths": ["functions", "ops"] 27 | }, 28 | "input": { 29 | "a": 10, 30 | "b": 2, 31 | "op": "*" 32 | }, 33 | "output": 20 34 | }, 35 | { 36 | "options": { 37 | "bindingsPaths": ["functions", "ops"] 38 | }, 39 | "input": { 40 | "a": 10, 41 | "b": 2, 42 | "op": "/" 43 | }, 44 | "output": 5 45 | }, 46 | { 47 | "options": { 48 | "bindingsPaths": ["functions", "ops"] 49 | }, 50 | "input": { 51 | "a": 10, 52 | "b": 0, 53 | "op": "/" 54 | }, 55 | "error": { 56 | "message": "division by zero is not allowed" 57 | } 58 | }, 59 | { 60 | "options": { 61 | "bindingsPaths": ["functions", "ops"] 62 | }, 63 | "input": { 64 | "a": 10, 65 | "b": 0, 66 | "op": "^" 67 | }, 68 | "error": { 69 | "message": "unsupported operation" 70 | } 71 | }, 72 | { 73 | "options": { 74 | "bindingsPaths": ["invalid_path"] 75 | }, 76 | "input": { 77 | "a": 10, 78 | "b": 2, 79 | "op": "+" 80 | }, 81 | "error": { 82 | "message": "Binding not found" 83 | } 84 | } 85 | ] 86 | -------------------------------------------------------------------------------- /test/scenarios/bindings_paths/functions.ts: -------------------------------------------------------------------------------- 1 | export const add = (a, b) => a + b; 2 | export const subtract = (a, b) => a - b; 3 | export const multiply = (a, b) => a * b; 4 | export const divide = (a, b) => a / b; 5 | -------------------------------------------------------------------------------- /test/scenarios/bindings_paths/ops.ts: -------------------------------------------------------------------------------- 1 | export const ops = { 2 | plus: '+', 3 | minus: '-', 4 | multiply: '*', 5 | divide: '/', 6 | }; 7 | -------------------------------------------------------------------------------- /test/scenarios/bindings_paths/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: unsupported 3 | condition: $not(op in $values($ops)) 4 | template: | 5 | $doThrow("unsupported operation") 6 | - name: add 7 | description: Do addition 8 | condition: op = $ops.plus 9 | template: | 10 | $add(a, b) 11 | - name: subtract 12 | description: Do subtraction 13 | condition: op = $ops.minus 14 | template: | 15 | $subtract(a, b) 16 | - name: multiply 17 | description: Do multiplication 18 | condition: op = $ops.multiply 19 | template: | 20 | $multiply(a, b) 21 | - name: divide 22 | description: Do division 23 | condition: op = $ops.divide 24 | template: | 25 | ( 26 | $assert( b != 0, "division by zero is not allowed"); 27 | $divide(a, b) 28 | ) 29 | -------------------------------------------------------------------------------- /test/scenarios/bindings_provider/bad_binding_from_provider.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: badBinding 3 | 4 | steps: 5 | - name: getMessage 6 | template: | 7 | $message & $anotherMessage 8 | -------------------------------------------------------------------------------- /test/scenarios/bindings_provider/bindings.ts: -------------------------------------------------------------------------------- 1 | export const anotherMessage = 'Got binding from normal binding.'; 2 | -------------------------------------------------------------------------------- /test/scenarios/bindings_provider/data.ts: -------------------------------------------------------------------------------- 1 | import { BindingProvider } from './provider'; 2 | import { Scenario } from '../../types'; 3 | 4 | export const data = [ 5 | { 6 | output: 'Got binding from provider.Got binding from normal binding.', 7 | options: { 8 | bindingProvider: BindingProvider.INSTANCE, 9 | }, 10 | }, 11 | { 12 | workflowPath: 'bad_binding_from_provider.yaml', 13 | options: { 14 | bindingProvider: BindingProvider.INSTANCE, 15 | }, 16 | error: { 17 | message: 'Binding not found', 18 | }, 19 | }, 20 | ] as Scenario[]; 21 | -------------------------------------------------------------------------------- /test/scenarios/bindings_provider/provider.ts: -------------------------------------------------------------------------------- 1 | import { WorkflowBindingProvider } from '../../../src'; 2 | export class BindingProvider implements WorkflowBindingProvider { 3 | static readonly INSTANCE = new BindingProvider(); 4 | provide(name: string): Promise { 5 | if (name == 'message') { 6 | return Promise.resolve({ message: 'Got binding from provider.' }); 7 | } 8 | return Promise.reject(new Error('Binding not found')); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /test/scenarios/bindings_provider/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | # this will be resolved using custom binding provider 3 | - path: message 4 | - name: anotherMessage 5 | 6 | steps: 7 | - name: getMessage 8 | template: | 9 | $message & $anotherMessage 10 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/assert_throw_using_custom_error.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | bindings: 3 | - name: CustomError 4 | steps: 5 | - name: assertThrow 6 | template: | 7 | $.assertThrow(false, new $.CustomError("some error")) 8 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/assert_throw_using_string.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: assertThrow 4 | template: | 5 | $.assertThrow(false, "some error") 6 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/bindings.ts: -------------------------------------------------------------------------------- 1 | export const data = { 2 | a: { 3 | b: [ 4 | { 5 | d: 1, 6 | }, 7 | { 8 | e: 2, 9 | }, 10 | ], 11 | c: { 12 | f: 3, 13 | }, 14 | }, 15 | }; 16 | 17 | export class CustomError extends Error { 18 | constructor(message: string) { 19 | super(message); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "assert_throw_using_custom_error.yaml", 4 | "error": { 5 | "message": "some error", 6 | "class": "CustomError" 7 | } 8 | }, 9 | { 10 | "workflowPath": "assert_throw_using_string.yaml", 11 | "error": { 12 | "message": "some error", 13 | "class": "Error" 14 | } 15 | }, 16 | { 17 | "workflowPath": "get_one_by_paths.yaml", 18 | "input": ["a.b.0.d", "a.b[1].e", "a.c"], 19 | "output": 1 20 | }, 21 | { 22 | "workflowPath": "get_one_by_paths.yaml", 23 | "input": ["a.b.0.non_existing", "a.b[1].e", "a.c"], 24 | "output": 2 25 | }, 26 | { 27 | "workflowPath": "get_one_by_paths.yaml", 28 | "input": ["a.b.0.non_existing", "a.b[1].non_existing", "a.c"], 29 | "output": { "f": 3 } 30 | }, 31 | { 32 | "workflowPath": "get_one_by_paths.yaml", 33 | "description": "no input so no output" 34 | }, 35 | { 36 | "workflowPath": "get_by_paths.yaml", 37 | "input": ["a.b.0.d", "a.b[1].e", "a.c"], 38 | "output": [ 39 | 1, 40 | 2, 41 | { 42 | "f": 3 43 | } 44 | ] 45 | }, 46 | { 47 | "workflowPath": "get_by_paths.yaml", 48 | "input": ["a.b[0].d", "a.c.f"], 49 | "output": [1, 3] 50 | }, 51 | { 52 | "workflowPath": "get_by_paths.yaml", 53 | "input": ["a.c.f", "iDontExist", "a.c"], 54 | "output": [ 55 | 3, 56 | { 57 | "f": 3 58 | } 59 | ] 60 | }, 61 | { 62 | "workflowPath": "get_by_paths.yaml", 63 | "input": "a.b", 64 | "output": [ 65 | { 66 | "d": 1 67 | }, 68 | { 69 | "e": 2 70 | } 71 | ] 72 | }, 73 | { 74 | "workflowPath": "get_by_paths.yaml", 75 | "input": "a.c.f", 76 | "output": 3 77 | }, 78 | { 79 | "workflowPath": "get_by_paths.yaml", 80 | "description": "no input so no output" 81 | }, 82 | { 83 | "workflowPath": "logger.yaml", 84 | "logLevel": 0 85 | }, 86 | { 87 | "workflowPath": "sum.yaml", 88 | "input": [1, 2, 3, 4], 89 | "output": 10 90 | }, 91 | { 92 | "workflowPath": "sha256-json-template.yaml", 93 | "input": 1234, 94 | "output": "03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4" 95 | }, 96 | { 97 | "workflowPath": "sha256.yaml", 98 | "input": "hello world", 99 | "output": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" 100 | }, 101 | { 102 | "workflowPath": "sha256.yaml", 103 | "input": 1234, 104 | "output": "03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4" 105 | }, 106 | { 107 | "workflowPath": "to_milli_seconds.yaml", 108 | "output": 1650965724561 109 | }, 110 | { 111 | "workflowPath": "to_seconds.yaml", 112 | "output": 1650965724 113 | } 114 | ] 115 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/get_by_paths.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: data # by default picks from ./bindings.(ts|js) 3 | steps: 4 | - name: getByPaths 5 | template: $getByPaths($data, $) # $ refers to entire input 6 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/get_one_by_paths.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: data # by default picks from ./bindings.(ts|js) 3 | steps: 4 | - name: getOneByPaths 5 | template: $getOneByPaths($data, $) # $ refers to entire input 6 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/logger.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: debug 3 | template: $debug("some log") 4 | - name: info 5 | template: $info("some log") 6 | - name: warn 7 | template: $warn("some log") 8 | - name: error 9 | template: $error("some log") 10 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/sha256-json-template.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: sha256 4 | template: | 5 | $.SHA256(^) 6 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/sha256.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: sha256 3 | template: | 4 | $SHA256($) 5 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/sum.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: sum 3 | template: | 4 | $sum($) 5 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/to_milli_seconds.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: toMilliSeconds 3 | template: $toMilliseconds("2022-04-26T09:35:24.561Z") 4 | -------------------------------------------------------------------------------- /test/scenarios/common_bindings/to_seconds.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: toSeconds 3 | template: $toSeconds("2022-04-26T09:35:24.561Z") 4 | -------------------------------------------------------------------------------- /test/scenarios/compile_time_expressions/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "a": 10, 5 | "b": 2, 6 | "op": "+" 7 | }, 8 | "output": 12 9 | }, 10 | { 11 | "input": { 12 | "a": 10, 13 | "b": 2, 14 | "op": "-" 15 | }, 16 | "output": 8 17 | }, 18 | { 19 | "input": { 20 | "a": 10, 21 | "b": 2, 22 | "op": "*" 23 | }, 24 | "output": 20 25 | }, 26 | { 27 | "input": { 28 | "a": 10, 29 | "b": 2, 30 | "op": "/" 31 | }, 32 | "output": 5 33 | } 34 | ] 35 | -------------------------------------------------------------------------------- /test/scenarios/compile_time_expressions/functions.ts: -------------------------------------------------------------------------------- 1 | export const add = (a, b) => a + b; 2 | export const subtract = (a, b) => a - b; 3 | export const multiply = (a, b) => a * b; 4 | export const divide = (a, b) => a / b; 5 | -------------------------------------------------------------------------------- /test/scenarios/compile_time_expressions/ops.ts: -------------------------------------------------------------------------------- 1 | export const ops = { 2 | PLUS: '+', 3 | MINUS: '-', 4 | MULTIPLY: '*', 5 | DIVIDE: '/', 6 | }; 7 | -------------------------------------------------------------------------------- /test/scenarios/compile_time_expressions/workflow.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | bindings: 3 | - name: ops 4 | path: ./ops 5 | - path: ./functions 6 | steps: 7 | - name: add 8 | description: Do addition 9 | condition: .op === {{$.ops.PLUS}} 10 | template: | 11 | $.add(.a, .b) 12 | - name: subtract 13 | description: Do subtraction 14 | condition: .op === {{$.ops.MINUS}} 15 | template: | 16 | $.subtract(.a, .b) 17 | - name: multiply 18 | description: Do multiplication 19 | condition: .op === {{$.ops.MULTIPLY}} 20 | template: | 21 | $.multiply(.a, .b) 22 | - name: divide 23 | description: Do division 24 | condition: .op === {{$.ops.DIVIDE}} 25 | template: | 26 | $.divide(.a, .b) 27 | -------------------------------------------------------------------------------- /test/scenarios/conditions/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "else_step.yaml", 4 | "input": { 5 | "a": 1 6 | }, 7 | "output": { 8 | "aIsLessThan5": true 9 | } 10 | }, 11 | { 12 | "workflowPath": "else_step.yaml", 13 | "input": { 14 | "a": 6 15 | }, 16 | "output": { 17 | "aIsGreaterThan5": true 18 | } 19 | }, 20 | { 21 | "workflowPath": "else_step.yaml", 22 | "input": { 23 | "a": 5 24 | }, 25 | "output": { 26 | "aIs5": true 27 | } 28 | }, 29 | { 30 | "workflowPath": "using_context.yaml", 31 | "input": { 32 | "a": "1" 33 | }, 34 | "output": { 35 | "aIsLessThan5": true 36 | } 37 | }, 38 | { 39 | "workflowPath": "using_context.yaml", 40 | "input": { 41 | "a": "5" 42 | }, 43 | "output": { 44 | "aIs5": true 45 | } 46 | }, 47 | { 48 | "workflowPath": "using_context.yaml", 49 | "input": { 50 | "a": "10" 51 | }, 52 | "output": { 53 | "aIsGreaterThan5": true 54 | } 55 | }, 56 | { 57 | "workflowPath": "using_outputs.yaml", 58 | "output": { 59 | "notASimpleCondition": true 60 | } 61 | }, 62 | { 63 | "workflowPath": "using_outputs.yaml", 64 | "output": { 65 | "notASimpleCondition": true 66 | } 67 | } 68 | ] 69 | -------------------------------------------------------------------------------- /test/scenarios/conditions/else_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: ifelse 3 | condition: a < 5 4 | template: | 5 | {"aIsLessThan5": true} 6 | else: 7 | name: aIsGreaterThanOrEqualsTo5 8 | condition: a = 5 9 | template: | 10 | {"aIs5": true} 11 | else: 12 | name: aIsGreaterThan5 13 | template: | 14 | {"aIsGreaterThan5": true} 15 | -------------------------------------------------------------------------------- /test/scenarios/conditions/using_context.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: prepareContext 3 | template: | 4 | $setContext("a", $number(a)) 5 | - name: aIsLessThan5 6 | condition: $context.a < 5 7 | template: | 8 | {"aIsLessThan5": true} 9 | - name: aIsGreaterThan5 10 | condition: $context.a > 5 11 | template: | 12 | {"aIsGreaterThan5": true} 13 | - name: aIs5 14 | condition: $context.a = 5 15 | template: | 16 | {"aIs5": true} 17 | -------------------------------------------------------------------------------- /test/scenarios/conditions/using_outputs.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: simpleCondition 3 | condition: simple = true 4 | template: | 5 | {"simpleCondition": true} 6 | - name: notASimpleCondition 7 | condition: $not($exists($outputs.simpleCondition)) 8 | template: | 9 | {"notASimpleCondition": true} 10 | -------------------------------------------------------------------------------- /test/scenarios/context/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "jsontemplate.yaml", 4 | "input": { 5 | "foo": true 6 | }, 7 | "output": "foo" 8 | }, 9 | { 10 | "workflowPath": "jsontemplate.yaml", 11 | "input": { 12 | "bar": true 13 | }, 14 | "output": "bar" 15 | }, 16 | { 17 | "workflowPath": "jsontemplate.yaml", 18 | "output": "default" 19 | }, 20 | { 21 | "input": { 22 | "foo": true 23 | }, 24 | "output": "foo" 25 | }, 26 | { 27 | "input": { 28 | "bar": true 29 | }, 30 | "output": "bar" 31 | }, 32 | { 33 | "output": "default" 34 | } 35 | ] 36 | -------------------------------------------------------------------------------- /test/scenarios/context/jsontemplate.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: setDefaultContext 4 | template: | 5 | $.context.something = "default" 6 | - name: setContextForFoo 7 | condition: .foo 8 | template: | 9 | $.context.something = "foo" 10 | - name: setContextForB 11 | condition: .bar 12 | template: | 13 | $.context.something = "bar" 14 | - name: returnContext 15 | template: | 16 | $.context.something 17 | -------------------------------------------------------------------------------- /test/scenarios/context/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: setDefaultContext 3 | template: | 4 | $setContext("something", "default") 5 | - name: setContextForFoo 6 | condition: $exists(foo) 7 | template: | 8 | $setContext("something", "foo") 9 | - name: setContextForB 10 | condition: $exists(bar) 11 | template: | 12 | $setContext("something", "bar") 13 | - name: returnContext 14 | template: | 15 | $context.something 16 | -------------------------------------------------------------------------------- /test/scenarios/create/data.ts: -------------------------------------------------------------------------------- 1 | import { Scenario } from '../../types'; 2 | 3 | export const data: Scenario[] = [ 4 | { 5 | workflowYAML: ` 6 | name: simple 7 | steps: 8 | - name: step1 9 | template: | 10 | "hello world" 11 | `, 12 | output: 'hello world', 13 | }, 14 | ]; 15 | -------------------------------------------------------------------------------- /test/scenarios/custom_executor/bad_executor.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: ./custom_executors 3 | name: badWorkflowExecutor 4 | 5 | executor: badWorkflowExecutor 6 | steps: 7 | - name: dummy 8 | template: | 9 | "error will be thrown by executor" 10 | -------------------------------------------------------------------------------- /test/scenarios/custom_executor/chained_executor.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: produceA 3 | template: | 4 | {"a": a + 1} 5 | - name: cubeA 6 | template: | 7 | a * a * a 8 | -------------------------------------------------------------------------------- /test/scenarios/custom_executor/custom_executors.ts: -------------------------------------------------------------------------------- 1 | import { WorkflowEngine, WorkflowExecutor, WorkflowOutput } from '../../../src'; 2 | 3 | class CustomWorkflowExecutor implements WorkflowExecutor { 4 | async execute(_engine: WorkflowEngine, _input: any): Promise { 5 | return { 6 | output: 'custom executor output', 7 | }; 8 | } 9 | } 10 | 11 | class BadWorkflowExecutor implements WorkflowExecutor { 12 | async execute(_engine: WorkflowEngine, _input: any): Promise { 13 | throw new Error('I am bad executor'); 14 | } 15 | } 16 | 17 | export const customWorkflowExecutor = new CustomWorkflowExecutor(); 18 | export const badWorkflowExecutor = new BadWorkflowExecutor(); 19 | -------------------------------------------------------------------------------- /test/scenarios/custom_executor/data.ts: -------------------------------------------------------------------------------- 1 | import { chainExecutor } from '../../../src'; 2 | import { Scenario } from '../../types'; 3 | 4 | export const data = [ 5 | { 6 | workflowPath: './bad_executor.yaml', 7 | error: { 8 | message: 'I am bad executor', 9 | workflowName: 'bad_executor', 10 | }, 11 | }, 12 | { 13 | output: 'custom executor output', 14 | }, 15 | { 16 | input: { 17 | a: 1, 18 | }, 19 | output: 8, 20 | workflowPath: './chained_executor.yaml', 21 | options: { 22 | executor: chainExecutor, 23 | }, 24 | }, 25 | { 26 | workflowPath: './non_existing_executor.yaml', 27 | error: { 28 | message: 'Workflow executor not found', 29 | }, 30 | }, 31 | ] as Scenario[]; 32 | -------------------------------------------------------------------------------- /test/scenarios/custom_executor/non_existing_executor.yaml: -------------------------------------------------------------------------------- 1 | executor: invalidExecutor 2 | 3 | steps: 4 | - name: dummy 5 | template: | 6 | "this will be not executed" 7 | -------------------------------------------------------------------------------- /test/scenarios/custom_executor/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: ./custom_executors 3 | name: customWorkflowExecutor 4 | 5 | executor: customWorkflowExecutor 6 | steps: 7 | - name: dummy 8 | template: | 9 | "this will be ignored by custom executor" 10 | -------------------------------------------------------------------------------- /test/scenarios/custom_step/bindings.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CustomStep, 3 | CustomStepExecutor, 4 | CustomStepExecutorProvider, 5 | ExecutionBindings, 6 | Executor, 7 | } from '../../../src'; 8 | 9 | export class TestCustomStepExecutorProvider implements CustomStepExecutorProvider { 10 | async provide(_step: CustomStep): Promise { 11 | return new TestCustomStepExecutor(); 12 | } 13 | } 14 | 15 | export class TestCustomStepExecutor implements CustomStepExecutor { 16 | execute( 17 | input: any, 18 | _bindings: ExecutionBindings, 19 | params?: Record | undefined, 20 | ): Promise { 21 | return Promise.resolve({ ...input, ...params }); 22 | } 23 | } 24 | 25 | export const testCustomStepExecutorProvider = new TestCustomStepExecutorProvider(); 26 | export const testCustomStepExecutor = new TestCustomStepExecutor(); 27 | -------------------------------------------------------------------------------- /test/scenarios/custom_step/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "provider.yaml", 4 | "input": { 5 | "a": 1, 6 | "b": 2 7 | }, 8 | "output": { 9 | "a": 1, 10 | "b": 2, 11 | "c": 3, 12 | "d": 4 13 | } 14 | }, 15 | { 16 | "workflowPath": "executor.yaml", 17 | "input": { 18 | "a": 1, 19 | "b": 2 20 | }, 21 | "output": { 22 | "a": 1, 23 | "b": 2, 24 | "c": 3, 25 | "d": 4 26 | } 27 | } 28 | ] 29 | -------------------------------------------------------------------------------- /test/scenarios/custom_step/executor.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: testCustomStepExecutor 3 | 4 | steps: 5 | - name: customStep 6 | type: custom 7 | executor: testCustomStepExecutor 8 | params: 9 | c: 3 10 | d: 4 11 | -------------------------------------------------------------------------------- /test/scenarios/custom_step/provider.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: testCustomStepExecutorProvider 3 | 4 | steps: 5 | - name: customStep 6 | type: custom 7 | provider: testCustomStepExecutorProvider 8 | params: 9 | c: 3 10 | d: 4 11 | -------------------------------------------------------------------------------- /test/scenarios/debug_step/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "output": "I will be logged" 4 | } 5 | ] 6 | -------------------------------------------------------------------------------- /test/scenarios/debug_step/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: debugStep 3 | debug: true 4 | template: | 5 | "I will be logged" 6 | -------------------------------------------------------------------------------- /test/scenarios/execute_steps/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "stepName": "add", 4 | "input": { 5 | "a": 10, 6 | "b": 2 7 | }, 8 | "output": 12 9 | }, 10 | { 11 | "stepName": "subtract", 12 | "input": { 13 | "a": 10, 14 | "b": 2 15 | }, 16 | "output": 8 17 | }, 18 | { 19 | "stepName": "invalid_step", 20 | "error": { 21 | "message": "invalid_step was not found" 22 | } 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /test/scenarios/execute_steps/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: add 3 | template: (a + b) 4 | - name: subtract 5 | template: (a - b) 6 | - name: moreOps 7 | steps: 8 | - name: multiply 9 | template: | 10 | (a * b) 11 | - name: divide 12 | template: | 13 | (a / b) 14 | -------------------------------------------------------------------------------- /test/scenarios/exit_actions/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "output": { 5 | "foo": "bar" 6 | } 7 | }, 8 | "output": { 9 | "foo": "bar" 10 | } 11 | }, 12 | { 13 | "input": {}, 14 | "output": { 15 | "hello": "world" 16 | } 17 | }, 18 | { 19 | "input": { 20 | "bad_data": "I am very very bad" 21 | }, 22 | "error": { 23 | "message": "bad data found" 24 | } 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /test/scenarios/exit_actions/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: checkIfProcessed 3 | condition: $exists(output) 4 | template: | 5 | output 6 | onComplete: return 7 | - name: validateInput 8 | template: | 9 | $assert($not($exists(bad_data)), "bad data found") 10 | - name: ignoreErrors 11 | template: | 12 | $doThrow("I simply fail") 13 | onError: continue 14 | - name: process 15 | template: | 16 | { "hello": "world" } 17 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/bad_workflow.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemaplte 2 | steps: 3 | - name: throwError 4 | template: | 5 | $.doThrow("some error") 6 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/bindings.ts: -------------------------------------------------------------------------------- 1 | export const foo = 'overridden by parent'; 2 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "external_workflow_parent_binding.yaml", 4 | "output": "overridden by parent" 5 | }, 6 | { 7 | "workflowPath": "external_workflow_value_binding.yaml", 8 | "output": "overridden by value" 9 | }, 10 | { 11 | "workflowPath": "external_workflow_with_errors.yaml", 12 | "error": { 13 | "message": "some error", 14 | "error": { 15 | "stepName": "throwError", 16 | "workflowName": "bad_workflow", 17 | "class": "StatusError" 18 | } 19 | } 20 | }, 21 | { 22 | "workflowPath": "external_workflow.yaml", 23 | "output": "original" 24 | } 25 | ] 26 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/external_workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: useExternalWorkflow 3 | externalWorkflow: 4 | path: ./workflow.yaml 5 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/external_workflow_parent_binding.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: foo 3 | steps: 4 | - name: useExternalWorkflow 5 | externalWorkflow: 6 | path: ./workflow.yaml 7 | bindings: 8 | - name: foo 9 | fromParent: true 10 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/external_workflow_value_binding.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: useExternalWorkflow 3 | externalWorkflow: 4 | path: ./workflow.yaml 5 | bindings: 6 | - name: foo 7 | value: 'overridden by value' 8 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/external_workflow_with_errors.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: useExternalWorkflow 3 | externalWorkflow: 4 | path: ./bad_workflow.yaml 5 | -------------------------------------------------------------------------------- /test/scenarios/external_workflows/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: foo 3 | value: 'original' 4 | steps: 5 | - name: returnFoo 6 | template: $foo 7 | -------------------------------------------------------------------------------- /test/scenarios/input_template/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "jsontemplate.yaml", 4 | "input": { 5 | "a": "1", 6 | "b": "2" 7 | }, 8 | "output": { 9 | "a": "1", 10 | "b": "2", 11 | "sum": 3 12 | } 13 | }, 14 | { 15 | "workflowPath": "jsontemplate.yaml", 16 | "input": { 17 | "a": 1, 18 | "b": "2" 19 | }, 20 | "output": { 21 | "a": 1, 22 | "b": "2", 23 | "sum": 3 24 | } 25 | }, 26 | { 27 | "workflowPath": "jsontemplate.yaml", 28 | 29 | "input": { 30 | "a": "1", 31 | "b": 2 32 | }, 33 | "output": { 34 | "a": "1", 35 | "b": 2, 36 | "sum": 3 37 | } 38 | }, 39 | { 40 | "workflowPath": "jsontemplate.yaml", 41 | "input": { 42 | "a": 1, 43 | "b": 2 44 | }, 45 | "output": { 46 | "a": 1, 47 | "b": 2, 48 | "sum": 3 49 | } 50 | }, 51 | { 52 | "input": { 53 | "a": "1", 54 | "b": "2" 55 | }, 56 | "output": { 57 | "a": "1", 58 | "b": "2", 59 | "sum": 3 60 | } 61 | }, 62 | { 63 | "input": { 64 | "a": 1, 65 | "b": "2" 66 | }, 67 | "output": { 68 | "a": 1, 69 | "b": "2", 70 | "sum": 3 71 | } 72 | }, 73 | { 74 | "input": { 75 | "a": "1", 76 | "b": 2 77 | }, 78 | "output": { 79 | "a": "1", 80 | "b": 2, 81 | "sum": 3 82 | } 83 | }, 84 | { 85 | "input": { 86 | "a": 1, 87 | "b": 2 88 | }, 89 | "output": { 90 | "a": 1, 91 | "b": 2, 92 | "sum": 3 93 | } 94 | } 95 | ] 96 | -------------------------------------------------------------------------------- /test/scenarios/input_template/jsontemplate.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: sum 4 | # here input is customized 5 | inputTemplate: | 6 | {"a": Number(.a), "b": Number(.b)} 7 | template: | 8 | (.a + .b) 9 | - name: combineResults 10 | # here input is original 11 | template: | 12 | { "a": .a, "b": .b, "sum": $.outputs.sum } 13 | -------------------------------------------------------------------------------- /test/scenarios/input_template/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: sum 3 | # here input is customized 4 | inputTemplate: | 5 | {"a": $number(a), "b": $number(b)} 6 | template: | 7 | (a + b) 8 | - name: combineResults 9 | # here input is original 10 | template: | 11 | { "a": a, "b": b, "sum": $outputs.sum } 12 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/blank_externalworkflow_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidExternalWorkflowStep 3 | externalWorkflow: 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/blank_function_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidFunctionStep 3 | functionName: 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/blank_template_path_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidTemplateStep 3 | templatePath: 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/blank_template_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidTemplateStep 3 | template: 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/blank_workflow_step_path.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidWorkflowStep 3 | workflowStepPath: 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/duplicate_step_names.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: duplicateName1 3 | template: | 4 | $info("something 1") 5 | - name: duplicateName1 6 | template: | 7 | $info("something otherthing 1") 8 | - name: duplicateName2 9 | template: | 10 | $info("something 2") 11 | - name: duplicateName2 12 | template: | 13 | $info("something otherthing 2") 14 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/duplicate_step_names_in_workflowstep.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: step1 3 | template: | 4 | $info("something 1") 5 | - name: workflowStep 6 | steps: 7 | - name: duplicateName2 8 | template: | 9 | $info("something 2") 10 | - name: duplicateName2 11 | template: | 12 | $info("something otherthing 2") 13 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/empty_steps.yaml: -------------------------------------------------------------------------------- 1 | steps: [] 2 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/empty_workflow.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rudderlabs/rudder-workflow-engine/9f30a62897324aeb939978c0b98037277cf654e2/test/scenarios/invalid_workflows/empty_workflow.yaml -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_batch_step_executor.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: batch 3 | type: batch 4 | executor: invalidExecutor 5 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_batch_step_executor_and_batches.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: batch 3 | type: batch 4 | executor: someExecutor 5 | batches: 6 | - key: someBatch 7 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_batch_step_filter_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidOutput 3 | type: batch 4 | batches: 5 | - key: foo 6 | filter: $.outputs.foo 7 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_batch_step_map_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidOutput 3 | type: batch 4 | batches: 5 | - key: foo 6 | map: $.outputs.foo 7 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_batch_step_no_executor_or_batches.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: batch 3 | type: batch 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_batch_step_with_loop.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: batch 3 | loopOverInput: true 4 | type: batch 5 | batches: 6 | - key: one 7 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_binding.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: iDontExist 3 | steps: 4 | - name: someStep 5 | template: | 6 | () 7 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_custom_step_executor.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: customStep 3 | type: custom 4 | executor: invalid 5 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_custom_step_executor_and_provider.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: customStep 3 | type: custom 4 | executor: someExecutor 5 | provider: someProvider 6 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_custom_step_no_executor_or_provider.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: customStep 3 | type: custom 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_custom_step_provider.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: customStep 3 | type: custom 4 | provider: invalid 5 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_else_simple_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: foo 3 | condition: hello 4 | template: 'foo' 5 | else: 6 | name: invalidOutput 7 | template: $outputs.bar 8 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_else_workflow_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: foo 3 | condition: hello 4 | template: 'foo' 5 | else: 6 | name: workflow 7 | steps: 8 | - name: invalidOutput 9 | template: $outputs.bar 10 | 11 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_function_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidFunctionStep 3 | functionName: iDontExist 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_input_template_output_references.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidOutput 3 | inputTemplate: $.outputs.foo 4 | template: | 5 | "hello" 6 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_loop_condition_output_references.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidOutput 3 | loopOverInput: true 4 | loopCondition: $.outputs.foo 5 | template: | 6 | "hello" 7 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_loop_condition_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidLoopConditionStep 3 | loopCondition: check 4 | template: | 5 | "hello" 6 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_nested_workflow_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: workflow 3 | steps: 4 | - name: foo 5 | template: 'foo' 6 | - name: invalidChildWorkflow 7 | steps: 8 | - name: bar 9 | template: 'bar' 10 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_simple_mutliple_output_references.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: foo 3 | template: 'foo' 4 | - name: invalidOutput 5 | template: $outputs.foo + $outputs.bar 6 | - name: bar 7 | template: 'bar' 8 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_simple_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: foo 3 | template: 'foo' 4 | - name: invalidOutput 5 | template: $outputs.bar 6 | - name: bar 7 | template: 'bar' 8 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_step_in_workflowstep.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidWorkflowStep 3 | steps: 4 | - name: invalidSimpleStep 5 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_step_name_with_spaces.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: workflowStep 3 | steps: 4 | - name: I have spaces in my name 5 | template: | 6 | $info("something 1") 7 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_step_name_with_special_chars.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: Step$ 3 | template: | 4 | $info("something 1") 5 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_step_nobody.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidStep 3 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_step_noname.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 3 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_step_noname_with_workflow_name.yaml: -------------------------------------------------------------------------------- 1 | name: Invalid Step without a name 2 | steps: 3 | - name: 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_template.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidTemplate 3 | template: | 4 | ({) 5 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_usage_else_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: InvalidElseStep 3 | description: else step should be used only in a conditional step 4 | template: | 5 | { "foo": "bar" } 6 | else: 7 | name: someOtherStep 8 | template: | 9 | { "hello": "bar" } 10 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_usage_nested_else_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: InvalidElseStep 3 | description: else step should be used only in a conditional step 4 | condition: a = 2 5 | template: | 6 | { "foo": "bar" } 7 | else: 8 | name: someOtherStep 9 | template: | 10 | { "hello": "bar" } 11 | else: 12 | name: someNestedElseStep 13 | template: | 14 | { "world": "bar" } 15 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_usage_oncomplete.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: InvalidOnComplete 3 | description: onComplete = return should be used only in a conditional step 4 | template: | 5 | { "foo": "bar" } 6 | onComplete: return 7 | - name: someOtherStep 8 | template: | 9 | { "hello": "bar" } 10 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_workflow_inner_simple_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: workflow 3 | steps: 4 | - name: foo 5 | template: 'foo' 6 | - name: bar 7 | template: 'bar' 8 | - name: invalidOutput 9 | template: $outputs.foo 10 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_workflow_inner_workflow_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: workflow 3 | steps: 4 | - name: foo 5 | template: 'foo' 6 | - name: invalidOutput 7 | template: $outputs.workflow.bar 8 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_workflow_outer_workflow_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: workflow1 3 | steps: 4 | - name: foo 5 | template: 'foo' 6 | - name: workflow2 7 | steps: 8 | - name: invalidOutput 9 | template: $outputs.workflow1.foo 10 | 11 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_workflow_self_output_reference.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: workflow 3 | steps: 4 | - name: foo 5 | template: 'foo' 6 | - name: invalidOutput 7 | template: $outputs.workflow 8 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_workflowstep.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidWorkflowStep 3 | steps: [] 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/invalid_workflowstep_from_path.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidWorkflowStep 3 | workflowStepPath: ./empty_steps.yaml 4 | -------------------------------------------------------------------------------- /test/scenarios/invalid_workflows/workflow_step_in_workflow_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: invalidWorkflowStep 3 | steps: 4 | - name: invalidChildStep 5 | steps: 6 | - name: someStep 7 | template: | 8 | () 9 | -------------------------------------------------------------------------------- /test/scenarios/jsontemplate/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "a": [ 5 | { 6 | "b": { 7 | "c": 1 8 | } 9 | } 10 | ] 11 | }, 12 | "output": 1 13 | }, 14 | { 15 | "input": { 16 | "a": { 17 | "b": [ 18 | { 19 | "c": 1 20 | } 21 | ] 22 | } 23 | }, 24 | "output": 1 25 | }, 26 | { 27 | "input": [ 28 | { 29 | "a": { 30 | "b": [ 31 | { 32 | "c": 1 33 | } 34 | ] 35 | } 36 | } 37 | ], 38 | "output": 1 39 | } 40 | ] 41 | -------------------------------------------------------------------------------- /test/scenarios/jsontemplate/workflow.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: sample 4 | template: | 5 | ~r .a.b.c 6 | -------------------------------------------------------------------------------- /test/scenarios/loop_over_input/data.ts: -------------------------------------------------------------------------------- 1 | export const data = [ 2 | { 3 | input: { 4 | description: 'I am not array so I will fail', 5 | }, 6 | error: { 7 | message: 'loopOverInput requires array input', 8 | }, 9 | }, 10 | { 11 | input: { 12 | elements: [ 13 | { 14 | description: 'I will return error', 15 | error: { 16 | message: 'some error', 17 | }, 18 | skip: true, 19 | }, 20 | { 21 | description: 'I will return the same output', 22 | output: { 23 | hello: 'world', 24 | }, 25 | }, 26 | { 27 | description: 'I will return custom output', 28 | }, 29 | ], 30 | }, 31 | output: [ 32 | { 33 | error: { 34 | message: 'some error', 35 | status: 500, 36 | error: expect.any(Error), 37 | originalError: expect.any(Error), 38 | }, 39 | }, 40 | { 41 | output: { 42 | hello: 'world', 43 | }, 44 | }, 45 | { 46 | output: { 47 | foo: 'bar', 48 | }, 49 | }, 50 | ], 51 | }, 52 | { 53 | workflowPath: 'loop_condition.yaml', 54 | input: { 55 | elements: [ 56 | { 57 | description: 'I will return error', 58 | error: { 59 | message: 'some error', 60 | }, 61 | }, 62 | { 63 | description: 'I will return the same output', 64 | output: { 65 | hello: 'world', 66 | }, 67 | execute: true, 68 | }, 69 | { 70 | description: 'I will return custom output', 71 | }, 72 | ], 73 | }, 74 | output: [ 75 | { 76 | skipped: true, 77 | }, 78 | { 79 | output: { 80 | hello: 'world', 81 | }, 82 | }, 83 | { 84 | skipped: true, 85 | }, 86 | ], 87 | }, 88 | ]; 89 | -------------------------------------------------------------------------------- /test/scenarios/loop_over_input/loop_condition.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: loopOverInput 3 | loopOverInput: true 4 | inputTemplate: elements 5 | loopCondition: 'execute' 6 | template: | 7 | ( 8 | $exists(error) ? $doThrow(error.message, 500); 9 | $exists(output) ? output : { "foo": "bar" } 10 | ) 11 | -------------------------------------------------------------------------------- /test/scenarios/loop_over_input/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: loopOverInput 3 | loopOverInput: true 4 | inputTemplate: elements 5 | template: | 6 | ( 7 | $exists(error) ? $doThrow(error.message, 500); 8 | $exists(output) ? output : { "foo": "bar" } 9 | ) 10 | -------------------------------------------------------------------------------- /test/scenarios/mappings/data.ts: -------------------------------------------------------------------------------- 1 | import { Scenario } from '../../types'; 2 | 3 | export const data: Scenario[] = [ 4 | { 5 | input: { 6 | a: 1, 7 | }, 8 | output: { 9 | bar: 1, 10 | }, 11 | }, 12 | ]; 13 | -------------------------------------------------------------------------------- /test/scenarios/mappings/workflow.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: mappings 4 | mappings: true 5 | description: | 6 | InputTemplate will be parsed as normal json template/ 7 | template will be parsed as mappings. 8 | inputTemplate: | 9 | { 10 | foo: ^.a 11 | } 12 | template: | 13 | [ 14 | { 15 | "input": "$.foo", 16 | "output": "$.bar" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /test/scenarios/multiplexing/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "foo": true 5 | }, 6 | "output": [ 7 | { 8 | "foo": "hello" 9 | } 10 | ] 11 | }, 12 | { 13 | "input": { 14 | "bar": true 15 | }, 16 | "output": [ 17 | { 18 | "bar": "world" 19 | } 20 | ] 21 | }, 22 | { 23 | "input": { 24 | "foo": true, 25 | "bar": true 26 | }, 27 | "output": [ 28 | { 29 | "foo": "hello" 30 | }, 31 | { 32 | "bar": "world" 33 | } 34 | ] 35 | } 36 | ] 37 | -------------------------------------------------------------------------------- /test/scenarios/multiplexing/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: responseFoo 3 | condition: foo = true 4 | template: | 5 | { "foo": "hello" } 6 | - name: responseBar 7 | condition: bar = true 8 | template: | 9 | { "bar": "world" } 10 | - name: multiplex 11 | template: | 12 | [$outputs.responseFoo, $outputs.responseBar] 13 | -------------------------------------------------------------------------------- /test/scenarios/outputs/complex_output_reference.yaml: -------------------------------------------------------------------------------- 1 | templateType: jsontemplate 2 | steps: 3 | - name: complexOutput 4 | template: | 5 | { 6 | "foo": "foo", 7 | "bar": "bar" 8 | } 9 | - name: workflow 10 | steps: 11 | - name: foo 12 | template: | 13 | { output: $.outputs.complexOutput.foo } 14 | - name: bar 15 | template: | 16 | { output: $.outputs.complexOutput.bar } 17 | - name: foobar 18 | template: | 19 | $.outputs.workflow.foo.output + " " + $.outputs.workflow.bar.output 20 | 21 | - name: copyWorkflowOutput 22 | template: $.outputs.workflow 23 | -------------------------------------------------------------------------------- /test/scenarios/outputs/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "workflowPath": "complex_output_reference.yaml", 4 | "output": "foo bar" 5 | }, 6 | { 7 | "input": { 8 | "description": "a + b + c", 9 | "a": "10", 10 | "b": "2", 11 | "c": "1", 12 | "op": "+" 13 | }, 14 | "output": 13 15 | }, 16 | { 17 | "input": { 18 | "description": "a - b + c", 19 | "a": "10", 20 | "b": "2", 21 | "c": "1", 22 | "op": "-" 23 | }, 24 | "output": 9 25 | }, 26 | { 27 | "input": { 28 | "description": "a * b + c", 29 | "a": "10", 30 | "b": "2", 31 | "c": "1", 32 | "op": "*" 33 | }, 34 | "output": 21 35 | }, 36 | { 37 | "input": { 38 | "description": "a / b + c", 39 | "a": "10", 40 | "b": "2", 41 | "c": "1", 42 | "op": "/" 43 | }, 44 | "output": 6 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /test/scenarios/outputs/workflow.yaml: -------------------------------------------------------------------------------- 1 | # a op b + c 2 | steps: 3 | - name: a 4 | description: Converts to a number 5 | template: | 6 | ($number(a)) 7 | - name: b 8 | description: Converts to a number 9 | template: | 10 | ($number(b)) 11 | - name: operation 12 | description: a op b + c 13 | steps: 14 | - name: c 15 | description: Converts to a number 16 | template: | 17 | ($number(c)) 18 | - name: add 19 | description: Do addition 20 | condition: op = "+" 21 | template: | 22 | ( $outputs.a + $outputs.b + $outputs.operation.c) 23 | - name: subtract 24 | description: Do subtraction 25 | condition: op = "-" 26 | template: | 27 | ( $outputs.a - $outputs.b + $outputs.operation.c) 28 | - name: multiply 29 | description: Do multiplication 30 | condition: op = "*" 31 | template: | 32 | ( ($outputs.a * $outputs.b) + $outputs.operation.c) 33 | - name: divide 34 | description: Do division 35 | condition: op = "/" 36 | template: | 37 | ( ($outputs.a / $outputs.b) + $outputs.operation.c) 38 | -------------------------------------------------------------------------------- /test/scenarios/override_bindings/bindings.ts: -------------------------------------------------------------------------------- 1 | export const MAX_BATCH_SIZE = 1000; 2 | export const MAX_NUM_REQUESTS = 500; 3 | -------------------------------------------------------------------------------- /test/scenarios/override_bindings/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "Should get overriden bindings", 4 | "options": { 5 | "creationTimeBindings": { 6 | "MAX_BATCH_SIZE": 10, 7 | "MAX_NUM_REQUESTS": 5 8 | } 9 | }, 10 | "output": { 11 | "maxBatchSize": 10, 12 | "maxNumRequests": 5 13 | } 14 | }, 15 | { 16 | "description": "Should get overriden bindings", 17 | "workflowPath": "workflow_step.yaml", 18 | "options": { 19 | "creationTimeBindings": { 20 | "MAX_BATCH_SIZE": 10, 21 | "MAX_NUM_REQUESTS": 5 22 | } 23 | }, 24 | "output": { 25 | "maxBatchSize": 10, 26 | "maxNumRequests": 5 27 | } 28 | }, 29 | { 30 | "description": "Should get overriden bindings", 31 | "workflowPath": "external_workflow.yaml", 32 | "options": { 33 | "creationTimeBindings": { 34 | "MAX_BATCH_SIZE": 10, 35 | "MAX_NUM_REQUESTS": 5 36 | } 37 | }, 38 | "output": { 39 | "maxBatchSize": 10, 40 | "maxNumRequests": 5 41 | } 42 | }, 43 | { 44 | "description": "Should get partial bindings", 45 | "options": { 46 | "creationTimeBindings": { 47 | "MAX_BATCH_SIZE": 10 48 | } 49 | }, 50 | "output": { 51 | "maxBatchSize": 10, 52 | "maxNumRequests": 500 53 | } 54 | }, 55 | { 56 | "description": "Should get default bindings", 57 | "output": { 58 | "maxBatchSize": 1000, 59 | "maxNumRequests": 500 60 | } 61 | } 62 | ] 63 | -------------------------------------------------------------------------------- /test/scenarios/override_bindings/external_workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: returnBindings 3 | externalWorkflow: 4 | path: workflow.yaml 5 | -------------------------------------------------------------------------------- /test/scenarios/override_bindings/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: bindings 3 | steps: 4 | - name: returnBindings 5 | template: | 6 | { 7 | "maxBatchSize": $MAX_BATCH_SIZE, 8 | "maxNumRequests": $MAX_NUM_REQUESTS 9 | } 10 | -------------------------------------------------------------------------------- /test/scenarios/override_bindings/workflow_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: workflowStep 3 | bindings: 4 | - path: bindings 5 | steps: 6 | - name: returnBindings 7 | template: | 8 | { 9 | "maxBatchSize": $MAX_BATCH_SIZE, 10 | "maxNumRequests": $MAX_NUM_REQUESTS 11 | } 12 | -------------------------------------------------------------------------------- /test/scenarios/return_within_template/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "returnCustomOutput": true, 5 | "customOutput": { "foo": "bar" } 6 | }, 7 | "output": { "foo": "bar" } 8 | }, 9 | { 10 | "input": { 11 | "defaultOutput": { "hello": "world" } 12 | }, 13 | "output": { "hello": "world" } 14 | } 15 | ] 16 | -------------------------------------------------------------------------------- /test/scenarios/return_within_template/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: returnNothing 3 | 4 | template: | 5 | ( 6 | /* this is same as returning nothing */ 7 | $doReturn() 8 | ) 9 | - name: returnEarly 10 | template: ( 11 | returnCustomOutput ? $doReturn(customOutput); 12 | defaultOutput 13 | ) 14 | -------------------------------------------------------------------------------- /test/scenarios/simple_steps/bindings.ts: -------------------------------------------------------------------------------- 1 | import { StepOutput } from '../../../src'; 2 | 3 | export const add = (input: { a: number; b: number }, bindings: Record): StepOutput => { 4 | const { a, b } = input; 5 | return { 6 | output: a + b, 7 | }; 8 | }; 9 | -------------------------------------------------------------------------------- /test/scenarios/simple_steps/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "a": 10, 5 | "b": 2, 6 | "op": "+" 7 | }, 8 | "output": 12 9 | }, 10 | { 11 | "input": { 12 | "a": 10, 13 | "b": 2, 14 | "op": "-" 15 | }, 16 | "output": 8 17 | }, 18 | { 19 | "input": { 20 | "a": 10, 21 | "b": 2, 22 | "op": "*" 23 | }, 24 | "output": 20 25 | }, 26 | { 27 | "input": { 28 | "a": 10, 29 | "b": 2, 30 | "op": "/" 31 | }, 32 | "output": 5 33 | }, 34 | { 35 | "input": { 36 | "a": 10, 37 | "b": 0, 38 | "op": "/" 39 | }, 40 | "error": { 41 | "message": "division by zero is not allowed" 42 | } 43 | } 44 | ] 45 | -------------------------------------------------------------------------------- /test/scenarios/simple_steps/divide_workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: validateInput 3 | template: $assert( b != 0, "division by zero is not allowed") 4 | - name: division 5 | template: | 6 | ( a / b ) 7 | -------------------------------------------------------------------------------- /test/scenarios/simple_steps/multiply.jsonata: -------------------------------------------------------------------------------- 1 | ( a * b ) -------------------------------------------------------------------------------- /test/scenarios/simple_steps/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: add 3 | path: bindings 4 | steps: 5 | - name: functionStepToAdd 6 | condition: op = "+" 7 | functionName: add 8 | - name: templateStepToSubtract 9 | condition: op = "-" 10 | template: | 11 | ( a - b ) 12 | - name: templatePathToMultiply 13 | condition: op = "*" 14 | templatePath: ./multiply.jsonata 15 | - name: externalWorkflowToDivide 16 | condition: op = "/" 17 | externalWorkflow: 18 | path: ./divide_workflow.yaml 19 | -------------------------------------------------------------------------------- /test/scenarios/throw_error/bad_workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: throwError 3 | template: | 4 | $doThrow(error, status) 5 | -------------------------------------------------------------------------------- /test/scenarios/throw_error/bindings.ts: -------------------------------------------------------------------------------- 1 | export function badFunction(input: any) { 2 | throw new BadFunctionError(input.error, input.status); 3 | } 4 | 5 | class BadFunctionError extends Error { 6 | response: { status?: number }; 7 | constructor(message: string, status?: number) { 8 | super(message); 9 | this.response = { status }; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test/scenarios/throw_error/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "throwAssertionError": true, 5 | "assertionPassed": false 6 | }, 7 | "error": { 8 | "message": "assertion failed", 9 | "status": 400, 10 | "stepName": "assertError" 11 | }, 12 | "class": "StatusError" 13 | }, 14 | { 15 | "input": { 16 | "throwTemplateError": true, 17 | "error": "some 4xx error", 18 | "status": 400 19 | }, 20 | "error": { 21 | "message": "some 4xx error", 22 | "status": 400, 23 | "workflowName": "workflow", 24 | "stepName": "workflowErrors", 25 | "childStepName": "templateError" 26 | }, 27 | "class": "StatusError" 28 | }, 29 | { 30 | "input": { 31 | "throwFunctionError": true, 32 | "error": "some 4xx error", 33 | "status": 400 34 | }, 35 | "error": { 36 | "message": "some 4xx error", 37 | "status": 400, 38 | "workflowName": "workflow", 39 | "stepName": "workflowErrors", 40 | "childStepName": "functionError" 41 | }, 42 | "class": "BadFunctionError" 43 | }, 44 | { 45 | "input": { 46 | "throwFunctionError": true, 47 | "error": "some unknown error" 48 | }, 49 | "error": { 50 | "message": "some unknown error", 51 | "status": 500, 52 | "workflowName": "workflow", 53 | "stepName": "workflowErrors", 54 | "childStepName": "functionError" 55 | }, 56 | "class": "BadFunctionError" 57 | }, 58 | { 59 | "input": { 60 | "throwExternalWorkflowError": true, 61 | "error": "some unknown error" 62 | }, 63 | "error": { 64 | "message": "some unknown error", 65 | "status": 500, 66 | "workflowName": "workflow", 67 | "stepName": "workflowErrors", 68 | "childStepName": "externalWorkflowError" 69 | }, 70 | "class": "WorkflowExecutionError" 71 | } 72 | ] 73 | -------------------------------------------------------------------------------- /test/scenarios/throw_error/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - name: badFunction 3 | steps: 4 | - name: assertError 5 | debug: true 6 | condition: throwAssertionError 7 | template: | 8 | $assert(assertionPassed, "assertion failed") 9 | - name: workflowErrors 10 | steps: 11 | - name: templateError 12 | condition: throwTemplateError 13 | template: | 14 | $doThrow(error, status) 15 | - name: functionError 16 | condition: throwFunctionError 17 | functionName: badFunction 18 | - name: externalWorkflowError 19 | condition: throwExternalWorkflowError 20 | externalWorkflow: 21 | path: bad_workflow.yaml 22 | -------------------------------------------------------------------------------- /test/scenarios/to_array/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "a": "foo", 5 | "b": ["foo"], 6 | "c": ["foo", "bar"] 7 | }, 8 | "output": { 9 | "a": ["foo"], 10 | "b": ["foo"], 11 | "c": ["foo", "bar"] 12 | } 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /test/scenarios/to_array/workflow.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: returnArray 3 | template: | 4 | { 5 | "a": $toArray(a), 6 | "b": $toArray(b), 7 | "c": $toArray(c), 8 | "d": $toArray(d) 9 | } 10 | -------------------------------------------------------------------------------- /test/scenarios/workflow_steps/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input": { 4 | "a": 10, 5 | "b": 2, 6 | "op": "+" 7 | }, 8 | "output": 12 9 | }, 10 | { 11 | "input": { 12 | "a": 10, 13 | "b": 2, 14 | "op": "-" 15 | }, 16 | "output": 8 17 | }, 18 | { 19 | "input": { 20 | "a": 10, 21 | "b": 2, 22 | "op": "*" 23 | }, 24 | "output": 20 25 | }, 26 | { 27 | "input": { 28 | "a": 10, 29 | "b": 2, 30 | "op": "/" 31 | }, 32 | "output": 5 33 | }, 34 | { 35 | "input": { 36 | "a": 10, 37 | "b": 0, 38 | "op": "/" 39 | }, 40 | "error": { 41 | "message": "division by zero is not allowed" 42 | } 43 | }, 44 | { 45 | "input": { 46 | "a": 10, 47 | "b": 0, 48 | "op": "^" 49 | }, 50 | "error": { 51 | "message": "unsupported operation" 52 | } 53 | } 54 | ] 55 | -------------------------------------------------------------------------------- /test/scenarios/workflow_steps/functions.ts: -------------------------------------------------------------------------------- 1 | export const add = (a, b) => a + b; 2 | export const subtract = (a, b) => a - b; 3 | export const multiply = (a, b) => a * b; 4 | export const divide = (a, b) => a / b; 5 | -------------------------------------------------------------------------------- /test/scenarios/workflow_steps/validation_workflow_step.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: validateOp 3 | template: | 4 | $assert(op in ["+", "-", "*", "/"], "unsupported operation") 5 | - name: validateForDivide 6 | condition: op = "/" 7 | template: | 8 | $assert( b != 0, "division by zero is not allowed") 9 | -------------------------------------------------------------------------------- /test/scenarios/workflow_steps/workflow.yaml: -------------------------------------------------------------------------------- 1 | bindings: 2 | - path: functions 3 | steps: 4 | - name: validateInput 5 | workflowStepPath: ./validation_workflow_step.yaml 6 | - name: operations 7 | description: Do operations using workflow step 8 | steps: 9 | - name: add 10 | description: Do addition 11 | condition: op = "+" 12 | template: | 13 | $add(a, b) 14 | onComplete: return 15 | - name: subtract 16 | description: Do subtraction 17 | condition: op = "-" 18 | template: | 19 | $subtract(a, b) 20 | onComplete: return 21 | - name: multiply 22 | description: Do multiplication 23 | condition: op = "*" 24 | template: | 25 | $multiply(a, b) 26 | onComplete: return 27 | - name: divide 28 | description: Do division 29 | condition: op = "/" 30 | template: | 31 | $divide(a, b) 32 | -------------------------------------------------------------------------------- /test/types.ts: -------------------------------------------------------------------------------- 1 | import { ExecutionBindings, LogLevel, WorkflowOptions } from '../src'; 2 | 3 | export type ScenarioError = { 4 | message?: string; 5 | status?: string; 6 | stepName?: string; 7 | childStepName?: string; 8 | workflowName?: string; 9 | class?: string; 10 | error?: ScenarioError; 11 | }; 12 | 13 | export type Scenario = { 14 | description?: string; 15 | input?: any; 16 | workflowPath?: string; 17 | workflowYAML?: string; 18 | options?: WorkflowOptions; 19 | stepName?: string; 20 | output?: any; 21 | error?: ScenarioError; 22 | logLevel?: LogLevel; 23 | executionBindings?: ExecutionBindings; 24 | }; 25 | -------------------------------------------------------------------------------- /test/utils/common.ts: -------------------------------------------------------------------------------- 1 | import { WorkflowExecutionError } from '../../src'; 2 | import { ScenarioError } from '../types'; 3 | 4 | export class CommonUtils { 5 | static matchError(actual: WorkflowExecutionError, expected?: ScenarioError) { 6 | if (expected === undefined) { 7 | throw actual; 8 | } 9 | 10 | this.matchErrorMessage(actual, expected); 11 | this.matchStatus(actual, expected); 12 | this.matchStepName(actual, expected); 13 | this.matchChildStepName(actual, expected); 14 | this.matchWorkflowName(actual, expected); 15 | this.matchErrorClass(actual, expected); 16 | this.matchNestedError(actual, expected); 17 | } 18 | 19 | private static matchErrorMessage(actual: WorkflowExecutionError, expected: ScenarioError) { 20 | if (expected.message) { 21 | expect(actual.message).toEqual(expect.stringContaining(expected.message)); 22 | } 23 | } 24 | 25 | private static matchStatus(actual: WorkflowExecutionError, expected: ScenarioError) { 26 | if (expected.status) { 27 | expect(actual.status).toEqual(expected.status); 28 | } 29 | } 30 | 31 | private static matchStepName(actual: WorkflowExecutionError, expected: ScenarioError) { 32 | if (expected.stepName) { 33 | expect(actual.stepName).toEqual(expected.stepName); 34 | } 35 | } 36 | 37 | private static matchChildStepName(actual: WorkflowExecutionError, expected: ScenarioError) { 38 | if (expected.childStepName) { 39 | expect(actual.childStepName).toEqual(expected.childStepName); 40 | } 41 | } 42 | 43 | private static matchWorkflowName(actual: WorkflowExecutionError, expected: ScenarioError) { 44 | if (expected.workflowName) { 45 | expect(actual.workflowName).toEqual(expected.workflowName); 46 | } 47 | } 48 | 49 | private static matchErrorClass(actual: WorkflowExecutionError, expected: ScenarioError) { 50 | if (expected.class) { 51 | expect(actual.originalError.constructor.name).toEqual(expected.class); 52 | } 53 | } 54 | 55 | private static matchNestedError(actual: WorkflowExecutionError, expected: ScenarioError) { 56 | if (expected.error) { 57 | this.matchError(actual.error as WorkflowExecutionError, expected.error); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /test/utils/index.ts: -------------------------------------------------------------------------------- 1 | export * from './common'; 2 | export * from './scenario'; 3 | -------------------------------------------------------------------------------- /test/utils/scenario.ts: -------------------------------------------------------------------------------- 1 | import { existsSync, readFileSync } from 'fs'; 2 | import { join } from 'path'; 3 | import { 4 | Executor, 5 | TemplateType, 6 | WorkflowEngine, 7 | WorkflowEngineFactory, 8 | WorkflowOptions, 9 | } from '../../src'; 10 | import { Scenario } from '../types'; 11 | 12 | export class ScenarioUtils { 13 | static createWorkflowEngine(scenarioDir: string, scenario: Scenario): Promise { 14 | const defaultOptions: WorkflowOptions = { 15 | rootPath: scenarioDir, 16 | templateType: TemplateType.JSONATA, 17 | }; 18 | scenario.options = { ...defaultOptions, ...scenario.options }; 19 | if (scenario.workflowYAML) { 20 | return WorkflowEngineFactory.createFromYaml(scenario.workflowYAML, scenario.options); 21 | } 22 | const workflowPath = join(scenarioDir, scenario.workflowPath ?? 'workflow.yaml'); 23 | return WorkflowEngineFactory.createFromFilePath(workflowPath, scenario.options); 24 | } 25 | 26 | private static async execute(executor: Executor, scenario: Scenario): Promise { 27 | let result = await executor.execute(scenario.input, scenario.executionBindings); 28 | return { output: result.output }; 29 | } 30 | 31 | static executeScenario(workflowEngine: WorkflowEngine, scenario: Scenario) { 32 | let executor: Executor = workflowEngine; 33 | if (scenario.stepName) { 34 | executor = workflowEngine.getStepExecutor(scenario.stepName); 35 | } 36 | return this.execute(executor, scenario); 37 | } 38 | 39 | static extractScenariosJSON(scenarioDir: string): Scenario[] { 40 | try { 41 | const scenariosJSON = readFileSync(join(scenarioDir, 'data.json'), { encoding: 'utf-8' }); 42 | return JSON.parse(scenariosJSON) as Scenario[]; 43 | } catch (e) { 44 | console.error(scenarioDir, e); 45 | throw e; 46 | } 47 | } 48 | 49 | static extractScenarios(scenarioDir: string): Scenario[] { 50 | if (existsSync(join(scenarioDir, 'data.json'))) { 51 | return this.extractScenariosJSON(scenarioDir); 52 | } 53 | const { data } = require(join(scenarioDir, 'data.ts')); 54 | return data as Scenario[]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts"] 4 | } 5 | --------------------------------------------------------------------------------