├── .codecov.yml ├── .github ├── CODEOWNERS ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── CI.yml │ ├── add-backport-label.yml.disabled │ ├── add-untriaged.yml │ ├── backport.yml │ ├── changelog_verifier.yml │ ├── publish-snapshots.yml │ ├── test-api-consistency.yml │ ├── test_bwc.yml │ ├── test_security.yml │ └── wrapper.yml ├── .gitignore ├── .whitesource ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVELOPER_GUIDE.md ├── LICENSE ├── MAINTAINERS.md ├── NOTICE ├── README.md ├── SECURITY.md ├── build.gradle ├── config └── checkstyle │ ├── checkstyle.xml │ └── checkstyle_suppressions.xml ├── formatter ├── formatterConfig.xml ├── formatting.gradle └── license-header.txt ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── release-notes ├── opensearch-flow-framework.release-notes-2.12.0.0.md ├── opensearch-flow-framework.release-notes-2.13.0.0.md ├── opensearch-flow-framework.release-notes-2.14.0.0.md ├── opensearch-flow-framework.release-notes-2.15.0.0.md ├── opensearch-flow-framework.release-notes-2.16.0.0.md ├── opensearch-flow-framework.release-notes-2.17.0.0.md ├── opensearch-flow-framework.release-notes-2.17.1.0.md ├── opensearch-flow-framework.release-notes-2.18.0.0.md ├── opensearch-flow-framework.release-notes-2.19.0.0.md ├── opensearch-flow-framework.release-notes-2.19.2.0.md ├── opensearch-flow-framework.release-notes-3.0.0.0-alpha1.md ├── opensearch-flow-framework.release-notes-3.0.0.0-beta1.md └── opensearch-flow-framework.release-notes-3.0.0.0.md ├── sample-templates ├── README.md ├── alert-summary-agent-claude-tested.json ├── alert-summary-agent-claude-tested.yml ├── alert-summary-log-pattern-agent.json ├── alert-summary-log-pattern-agent.yml ├── anomaly-detector-suggestion-agent-claude.json ├── anomaly-detector-suggestion-agent-claude.yml ├── create-knowledge-base-alert-agent.json ├── create-knowledge-base-alert-agent.yml ├── deploy-bedrock-claude-model.json ├── deploy-bedrock-claude-model.yml ├── deploy-openai-model.json ├── deploy-openai-model.yml ├── deploy-sagemaker-mistral-model.json ├── deploy-sagemaker-mistral-model.yml ├── observability-chat-agent-openai-untested.json ├── observability-chat-agent-openai-untested.yml ├── query-assist-agent-claude-tested.json ├── query-assist-agent-claude-tested.yml ├── query-assist-data-summary-agent-claude-tested.json ├── query-assist-data-summary-agent-claude-tested.yml ├── query-assist-data-summary-with-log-pattern-agent-claude-tested.json ├── query-assist-data-summary-with-log-pattern-agent-claude-tested.yml ├── text-to-visualization-claude.json └── text-to-visualization-claude.yml ├── settings.gradle └── src ├── main ├── java │ └── org │ │ └── opensearch │ │ └── flowframework │ │ ├── FlowFrameworkPlugin.java │ │ ├── common │ │ ├── CommonValue.java │ │ ├── DefaultUseCases.java │ │ ├── FlowFrameworkSettings.java │ │ ├── ThrowingSupplier.java │ │ ├── ThrowingSupplierWrapper.java │ │ └── WorkflowResources.java │ │ ├── exception │ │ ├── ApiSpecParseException.java │ │ ├── FlowFrameworkException.java │ │ └── WorkflowStepException.java │ │ ├── indices │ │ ├── FlowFrameworkIndex.java │ │ └── FlowFrameworkIndicesHandler.java │ │ ├── model │ │ ├── Config.java │ │ ├── PipelineProcessor.java │ │ ├── ProvisioningProgress.java │ │ ├── ResourceCreated.java │ │ ├── State.java │ │ ├── Template.java │ │ ├── Workflow.java │ │ ├── WorkflowEdge.java │ │ ├── WorkflowNode.java │ │ ├── WorkflowState.java │ │ ├── WorkflowStepValidator.java │ │ └── WorkflowValidator.java │ │ ├── rest │ │ ├── AbstractSearchWorkflowAction.java │ │ ├── RestCreateWorkflowAction.java │ │ ├── RestDeleteWorkflowAction.java │ │ ├── RestDeprovisionWorkflowAction.java │ │ ├── RestGetWorkflowAction.java │ │ ├── RestGetWorkflowStateAction.java │ │ ├── RestGetWorkflowStepAction.java │ │ ├── RestProvisionWorkflowAction.java │ │ ├── RestSearchWorkflowAction.java │ │ └── RestSearchWorkflowStateAction.java │ │ ├── transport │ │ ├── CreateWorkflowAction.java │ │ ├── CreateWorkflowTransportAction.java │ │ ├── DeleteWorkflowAction.java │ │ ├── DeleteWorkflowTransportAction.java │ │ ├── DeprovisionWorkflowAction.java │ │ ├── DeprovisionWorkflowTransportAction.java │ │ ├── GetWorkflowAction.java │ │ ├── GetWorkflowResponse.java │ │ ├── GetWorkflowStateAction.java │ │ ├── GetWorkflowStateRequest.java │ │ ├── GetWorkflowStateResponse.java │ │ ├── GetWorkflowStateTransportAction.java │ │ ├── GetWorkflowStepAction.java │ │ ├── GetWorkflowStepResponse.java │ │ ├── GetWorkflowStepTransportAction.java │ │ ├── GetWorkflowTransportAction.java │ │ ├── ProvisionWorkflowAction.java │ │ ├── ProvisionWorkflowTransportAction.java │ │ ├── ReprovisionWorkflowAction.java │ │ ├── ReprovisionWorkflowRequest.java │ │ ├── ReprovisionWorkflowTransportAction.java │ │ ├── SearchWorkflowAction.java │ │ ├── SearchWorkflowStateAction.java │ │ ├── SearchWorkflowStateTransportAction.java │ │ ├── SearchWorkflowTransportAction.java │ │ ├── WorkflowRequest.java │ │ ├── WorkflowResponse.java │ │ └── handler │ │ │ └── SearchHandler.java │ │ ├── util │ │ ├── ApiSpecFetcher.java │ │ ├── EncryptorUtils.java │ │ ├── ParseUtils.java │ │ ├── RestHandlerUtils.java │ │ ├── TenantAwareHelper.java │ │ └── WorkflowTimeoutUtility.java │ │ └── workflow │ │ ├── AbstractCreatePipelineStep.java │ │ ├── AbstractRegisterLocalModelStep.java │ │ ├── AbstractRetryableWorkflowStep.java │ │ ├── AbstractUpdatePipelineStep.java │ │ ├── CreateConnectorStep.java │ │ ├── CreateIndexStep.java │ │ ├── CreateIngestPipelineStep.java │ │ ├── CreateSearchPipelineStep.java │ │ ├── DeleteAgentStep.java │ │ ├── DeleteConnectorStep.java │ │ ├── DeleteIndexStep.java │ │ ├── DeleteIngestPipelineStep.java │ │ ├── DeleteModelStep.java │ │ ├── DeleteSearchPipelineStep.java │ │ ├── DeployModelStep.java │ │ ├── NoOpStep.java │ │ ├── ProcessNode.java │ │ ├── RegisterAgentStep.java │ │ ├── RegisterLocalCustomModelStep.java │ │ ├── RegisterLocalPretrainedModelStep.java │ │ ├── RegisterLocalSparseEncodingModelStep.java │ │ ├── RegisterModelGroupStep.java │ │ ├── RegisterRemoteModelStep.java │ │ ├── ReindexStep.java │ │ ├── ToolStep.java │ │ ├── UndeployModelStep.java │ │ ├── UpdateIndexStep.java │ │ ├── UpdateIngestPipelineStep.java │ │ ├── UpdateSearchPipelineStep.java │ │ ├── WorkflowData.java │ │ ├── WorkflowDataStep.java │ │ ├── WorkflowProcessSorter.java │ │ ├── WorkflowStep.java │ │ └── WorkflowStepFactory.java ├── plugin-metadata │ └── plugin-security.policy └── resources │ ├── defaults │ ├── bedrock-titan-embedding-defaults.json │ ├── bedrock-titan-multimodal-defaults.json │ ├── cohere-chat-defaults.json │ ├── cohere-embedding-defaults.json │ ├── cohere-embedding-semantic-search-defaults.json │ ├── cohere-embedding-semantic-search-with-query-enricher-defaults.json │ ├── conversational-search-defaults.json │ ├── conversational-search-rag-tool-defaults.json │ ├── hybrid-search-defaults.json │ ├── hybrid-search-with-local-model-defaults.json │ ├── local-sparse-search-biencoder-defaults.json │ ├── multi-modal-search-defaults.json │ ├── multimodal-search-bedrock-titan-defaults.json │ ├── openai-chat-defaults.json │ ├── openai-embedding-defaults.json │ ├── semantic-search-defaults.json │ ├── semantic-search-query-enricher-defaults.json │ ├── semantic-search-with-local-model-defaults.json │ └── semantic-search-with-reindex-defaults.json │ ├── log4j2.xml │ ├── mappings │ ├── config.json │ ├── global-context.json │ └── workflow-state.json │ └── substitutionTemplates │ ├── conversational-search-with-bedrock-rag-tool-template.json │ ├── conversational-search-with-cohere-model-template.json │ ├── deploy-remote-bedrock-model-template.json │ ├── deploy-remote-model-chat-template.json │ ├── deploy-remote-model-extra-params-template.json │ ├── deploy-remote-model-template.json │ ├── hybrid-search-template.json │ ├── hybrid-search-with-local-model-template.json │ ├── multi-modal-search-template.json │ ├── multi-modal-search-with-bedrock-titan-template.json │ ├── neural-sparse-local-biencoder-template.json │ ├── semantic-search-template.json │ ├── semantic-search-with-local-model-template.json │ ├── semantic-search-with-model-and-query-enricher-template.json │ ├── semantic-search-with-model-template.json │ ├── semantic-search-with-query-enricher-template.json │ └── semantic-search-with-reindex-template.json ├── test ├── java │ └── org │ │ └── opensearch │ │ └── flowframework │ │ ├── FlowFrameworkPluginIT.java │ │ ├── FlowFrameworkPluginTests.java │ │ ├── FlowFrameworkRestTestCase.java │ │ ├── FlowFrameworkTenantAwareRestTestCase.java │ │ ├── TestHelpers.java │ │ ├── bwc │ │ └── FlowFrameworkBackwardsCompatibilityIT.java │ │ ├── common │ │ ├── DefaultUseCasesTests.java │ │ └── FlowFrameworkSettingsTests.java │ │ ├── exception │ │ ├── ApiSpecParseExceptionTests.java │ │ └── FlowFrameworkExceptionTests.java │ │ ├── indices │ │ └── FlowFrameworkIndicesHandlerTests.java │ │ ├── model │ │ ├── ConfigTests.java │ │ ├── PipelineProcessorTests.java │ │ ├── ResourceCreatedTests.java │ │ ├── TemplateTestJsonUtil.java │ │ ├── TemplateTests.java │ │ ├── WorkflowEdgeTests.java │ │ ├── WorkflowNodeTests.java │ │ ├── WorkflowStateTests.java │ │ ├── WorkflowStepValidatorTests.java │ │ ├── WorkflowTests.java │ │ └── WorkflowValidatorTests.java │ │ ├── rest │ │ ├── FlowFrameworkRestApiIT.java │ │ ├── FlowFrameworkSecureRestApiIT.java │ │ ├── RestCreateWorkflowActionTests.java │ │ ├── RestDeleteWorkflowActionTests.java │ │ ├── RestDeprovisionWorkflowActionTests.java │ │ ├── RestGetWorkflowActionTests.java │ │ ├── RestGetWorkflowStateActionTests.java │ │ ├── RestGetWorkflowStepActionTests.java │ │ ├── RestProvisionWorkflowActionTests.java │ │ ├── RestSearchWorkflowActionTests.java │ │ ├── RestSearchWorkflowStateActionTests.java │ │ ├── RestWorkflowProvisionTenantAwareIT.java │ │ ├── RestWorkflowStateTenantAwareIT.java │ │ └── RestWorkflowTenantAwareIT.java │ │ ├── transport │ │ ├── CreateWorkflowTransportActionTests.java │ │ ├── DeleteWorkflowTransportActionTests.java │ │ ├── DeprovisionWorkflowTransportActionTests.java │ │ ├── GetWorkflowStateTransportActionTests.java │ │ ├── GetWorkflowStepTransportActionTests.java │ │ ├── GetWorkflowTransportActionTests.java │ │ ├── ProvisionWorkflowTransportActionTests.java │ │ ├── ReprovisionWorkflowRequestTests.java │ │ ├── ReprovisionWorkflowTransportActionTests.java │ │ ├── SearchWorkflowStateTransportActionTests.java │ │ ├── SearchWorkflowTransportActionTests.java │ │ ├── WorkflowRequestResponseTests.java │ │ └── handler │ │ │ └── SearchHandlerTests.java │ │ ├── util │ │ ├── ApiSpecFetcherTests.java │ │ ├── EncryptorUtilsTests.java │ │ ├── ParseUtilsTests.java │ │ ├── RestHandlerUtilsTests.java │ │ ├── TenantAwareHelperTests.java │ │ └── WorkflowTimeoutUtilityTests.java │ │ └── workflow │ │ ├── CreateConnectorStepTests.java │ │ ├── CreateIndexStepTests.java │ │ ├── CreateIngestPipelineStepTests.java │ │ ├── CreateSearchPipelineStepTests.java │ │ ├── DeleteAgentStepTests.java │ │ ├── DeleteConnectorStepTests.java │ │ ├── DeleteIndexStepTests.java │ │ ├── DeleteIngestPipelineStepTests.java │ │ ├── DeleteModelStepTests.java │ │ ├── DeleteSearchPipelineStepTests.java │ │ ├── DeployModelStepTests.java │ │ ├── NoOpStepTests.java │ │ ├── ProcessNodeTests.java │ │ ├── RegisterAgentTests.java │ │ ├── RegisterLocalCustomModelStepTests.java │ │ ├── RegisterLocalPretrainedModelStepTests.java │ │ ├── RegisterLocalSparseEncodingModelStepTests.java │ │ ├── RegisterModelGroupStepTests.java │ │ ├── RegisterRemoteModelStepTests.java │ │ ├── ReindexStepTests.java │ │ ├── ToolStepTests.java │ │ ├── UndeployModelStepTests.java │ │ ├── UpdateIndexStepTests.java │ │ ├── UpdateIngestPipelineStepTests.java │ │ ├── UpdateSearchPipelineStepTests.java │ │ ├── WorkflowDataStepTests.java │ │ ├── WorkflowDataTests.java │ │ └── WorkflowProcessSorterTests.java └── resources │ └── template │ ├── agent-framework.json │ ├── createconnector-createconnectortool-createflowagent.json │ ├── createconnector-registerremotemodel-deploymodel.json │ ├── finaltemplate.json │ ├── ingest-search-pipeline-template.json │ ├── noop.json │ ├── register-deploylocalsparseencodingmodel.json │ ├── registerremotemodel-createindex.json │ ├── registerremotemodel-ingestpipeline-createindex.json │ ├── registerremotemodel-ingestpipeline-updateindex.json │ ├── registerremotemodel-ingestpipeline.json │ └── registerremotemodel.json └── yamlRestTest ├── java └── org │ └── opensearch │ └── flowframework │ └── FlowFrameworkPluginYamlTestSuiteIT.java └── resources └── rest-api-spec └── test └── 10_basic.yml /.codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: true 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "70...100" 8 | status: 9 | project: 10 | default: 11 | target: auto 12 | threshold: 2% # project coverage can drop 13 | patch: 14 | default: 15 | target: 70% # required diff coverage value 16 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @dbwiddis @owaiskazi19 @joshpalis @ohltyler @amitgalitz @jackiehanyang @junweid62 -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Description 2 | [Describe what this change achieves] 3 | 4 | ### Related Issues 5 | Resolves #[Issue number to be closed when this PR is merged] 6 | 7 | 8 | ### Check List 9 | - [ ] New functionality includes testing. 10 | - [ ] New functionality has been documented. 11 | - [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md). 12 | - [ ] Commits are signed per the DCO using `--signoff`. 13 | - [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose). 14 | 15 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 16 | For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/flow-framework/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). 17 | -------------------------------------------------------------------------------- /.github/workflows/add-backport-label.yml.disabled: -------------------------------------------------------------------------------- 1 | name: Add Backport Label 2 | on: 3 | pull_request_target: 4 | branches: main 5 | types: opened 6 | 7 | jobs: 8 | add_labels: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: actions-ecosystem/action-add-labels@v1 13 | with: 14 | labels: backport 2.x 15 | -------------------------------------------------------------------------------- /.github/workflows/add-untriaged.yml: -------------------------------------------------------------------------------- 1 | name: Apply 'untriaged' label during issue lifecycle 2 | 3 | on: 4 | issues: 5 | types: [opened, reopened, transferred] 6 | 7 | jobs: 8 | apply-label: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/github-script@v7 12 | with: 13 | script: | 14 | github.rest.issues.addLabels({ 15 | issue_number: context.issue.number, 16 | owner: context.repo.owner, 17 | repo: context.repo.repo, 18 | labels: ['untriaged'] 19 | }) 20 | -------------------------------------------------------------------------------- /.github/workflows/backport.yml: -------------------------------------------------------------------------------- 1 | name: Backport 2 | on: 3 | pull_request_target: 4 | types: 5 | - closed 6 | - labeled 7 | 8 | jobs: 9 | backport: 10 | name: Backport 11 | runs-on: ubuntu-latest 12 | # Only react to merged PRs for security reasons. 13 | # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. 14 | if: > 15 | github.event.pull_request.merged 16 | && ( 17 | github.event.action == 'closed' 18 | || ( 19 | github.event.action == 'labeled' 20 | && contains(github.event.label.name, 'backport') 21 | ) 22 | ) 23 | permissions: 24 | contents: write 25 | pull-requests: write 26 | steps: 27 | - name: GitHub App token 28 | id: github_app_token 29 | uses: tibdex/github-app-token@v2 30 | with: 31 | app_id: ${{ secrets.APP_ID }} 32 | private_key: ${{ secrets.APP_PRIVATE_KEY }} 33 | installation_id: 22958780 34 | 35 | - name: Backport 36 | uses: VachaShah/backport@v2.2.0 37 | with: 38 | github_token: ${{ steps.github_app_token.outputs.token }} 39 | head_template: backport/backport-<%= number %>-to-<%= base %> 40 | failure_labels: backport-failed 41 | -------------------------------------------------------------------------------- /.github/workflows/changelog_verifier.yml: -------------------------------------------------------------------------------- 1 | name: "Changelog Verifier" 2 | on: 3 | push: 4 | branches-ignore: 5 | - 'whitesource-remediate/**' 6 | - 'backport/**' 7 | pull_request: 8 | branches: main 9 | types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] 10 | 11 | jobs: 12 | # Enforces the update of a changelog file on every pull request 13 | verify-changelog: 14 | if: github.repository == 'opensearch-project/flow-framework' 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | with: 19 | token: ${{ secrets.GITHUB_TOKEN }} 20 | ref: ${{ github.event.pull_request.head.sha }} 21 | 22 | - uses: dangoslen/changelog-enforcer@v3 23 | with: 24 | skipLabels: "autocut, skip-changelog" 25 | -------------------------------------------------------------------------------- /.github/workflows/publish-snapshots.yml: -------------------------------------------------------------------------------- 1 | name: Publish snapshots to maven 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - 'main' 8 | - '[0-9]+.[0-9]+' 9 | - '[0-9]+.x' 10 | 11 | jobs: 12 | build-and-publish-snapshots: 13 | strategy: 14 | fail-fast: false 15 | if: github.repository == 'opensearch-project/flow-framework' 16 | runs-on: ubuntu-latest 17 | 18 | permissions: 19 | id-token: write 20 | contents: write 21 | 22 | steps: 23 | - uses: actions/setup-java@v4 24 | with: 25 | distribution: temurin # Temurin is a distribution of adoptium 26 | java-version: 21 27 | - uses: actions/checkout@v4 28 | - uses: aws-actions/configure-aws-credentials@v4.2.1 29 | with: 30 | role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} 31 | aws-region: us-east-1 32 | - name: publish snapshots to maven 33 | run: | 34 | export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) 35 | export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) 36 | echo "::add-mask::$SONATYPE_USERNAME" 37 | echo "::add-mask::$SONATYPE_PASSWORD" 38 | ./gradlew publishPluginZipPublicationToSnapshotsRepository 39 | -------------------------------------------------------------------------------- /.github/workflows/test-api-consistency.yml: -------------------------------------------------------------------------------- 1 | name: Daily API Consistency Test 2 | 3 | on: 4 | schedule: 5 | - cron: '0 8 * * *' # Runs daily at 8 AM UTC 6 | workflow_dispatch: 7 | 8 | jobs: 9 | API-consistency-test: 10 | if: github.repository == 'opensearch-project/flow-framework' 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | java: [21] 15 | 16 | steps: 17 | - name: Checkout Flow Framework 18 | uses: actions/checkout@v3 19 | 20 | - name: Setup Java ${{ matrix.java }} 21 | uses: actions/setup-java@v3 22 | with: 23 | distribution: 'temurin' 24 | java-version: ${{ matrix.java }} 25 | 26 | - name: Run API Consistency Tests 27 | run: ./gradlew test --tests "org.opensearch.flowframework.workflow.*" 28 | -------------------------------------------------------------------------------- /.github/workflows/test_bwc.yml: -------------------------------------------------------------------------------- 1 | name: BWC 2 | on: 3 | workflow_dispatch: 4 | push: 5 | branches-ignore: 6 | - 'whitesource-remediate/**' 7 | - 'backport/**' 8 | - 'create-pull-request/**' 9 | pull_request: 10 | types: [opened, synchronize, reopened] 11 | 12 | jobs: 13 | Build-ff-linux: 14 | strategy: 15 | matrix: 16 | java: [21] 17 | fail-fast: false 18 | 19 | name: Test Flow Framework BWC 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - name: Setup Java ${{ matrix.java }} 24 | uses: actions/setup-java@v4 25 | with: 26 | distribution: 'temurin' 27 | java-version: ${{ matrix.java }} 28 | 29 | - name: Checkout Flow Framework 30 | uses: actions/checkout@v4 31 | 32 | - name: Assemble Flow Framework 33 | run: | 34 | plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` 35 | echo plugin_version $plugin_version 36 | ./gradlew assemble 37 | echo "Creating ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version ..." 38 | mkdir -p ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version 39 | echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version ..." 40 | ls ./build/distributions/ 41 | cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version 42 | echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version ..." 43 | ls ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version 44 | - name: Run Flow Framework Backwards Compatibility Tests 45 | run: | 46 | echo "Running backwards compatibility tests ..." 47 | ./gradlew bwcTestSuite -Dtests.security.manager=false 48 | -------------------------------------------------------------------------------- /.github/workflows/test_security.yml: -------------------------------------------------------------------------------- 1 | name: Security test workflow for Flow Framework 2 | on: 3 | workflow_dispatch: 4 | push: 5 | branches-ignore: 6 | - 'whitesource-remediate/**' 7 | - 'backport/**' 8 | - 'create-pull-request/**' 9 | pull_request: 10 | types: [opened, synchronize, reopened] 11 | 12 | 13 | jobs: 14 | Get-CI-Image-Tag: 15 | uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main 16 | with: 17 | product: opensearch 18 | 19 | integ-test-with-security-linux: 20 | strategy: 21 | matrix: 22 | java: [21] 23 | 24 | name: Run Security Integration Tests on Linux 25 | runs-on: ubuntu-latest 26 | needs: Get-CI-Image-Tag 27 | container: 28 | # using the same image which is used by opensearch-build team to build the OpenSearch Distribution 29 | # this image tag is subject to change as more dependencies and updates will arrive over time 30 | image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} 31 | # need to switch to root so that github actions can install runner binary on container without permission issues. 32 | options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} 33 | 34 | steps: 35 | - name: Run start commands 36 | run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} 37 | - name: Checkout Flow Framework 38 | uses: actions/checkout@v4 39 | - name: Setup Java ${{ matrix.java }} 40 | uses: actions/setup-java@v4 41 | with: 42 | distribution: 'temurin' 43 | java-version: ${{ matrix.java }} 44 | - name: Run tests 45 | # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. 46 | run: | 47 | chown -R 1000:1000 `pwd` 48 | su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" 49 | -------------------------------------------------------------------------------- /.github/workflows/wrapper.yml: -------------------------------------------------------------------------------- 1 | name: Validate Gradle Wrapper 2 | on: 3 | push: 4 | branches-ignore: 5 | - 'whitesource-remediate/**' 6 | - 'backport/**' 7 | pull_request: 8 | types: [opened, synchronize, reopened] 9 | 10 | jobs: 11 | validate: 12 | name: Validate 13 | if: github.repository == 'opensearch-project/flow-framework' 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: gradle/actions/wrapper-validation@v4 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # intellij files 2 | .idea/ 3 | *.iml 4 | *.ipr 5 | *.iws 6 | *.log 7 | build-idea/ 8 | out/ 9 | 10 | # eclipse files 11 | .classpath 12 | .project 13 | .settings 14 | 15 | # gradle stuff 16 | .gradle/ 17 | build/ 18 | bin/ 19 | 20 | # vscode stuff 21 | .vscode/ 22 | 23 | # osx stuff 24 | .DS_Store 25 | -------------------------------------------------------------------------------- /.whitesource: -------------------------------------------------------------------------------- 1 | { 2 | "scanSettings": { 3 | "configMode": "AUTO", 4 | "configExternalURL": "", 5 | "projectToken": "", 6 | "baseBranches": [] 7 | }, 8 | "scanSettingsSAST": { 9 | "enableScan": false, 10 | "scanPullRequests": false, 11 | "incrementalScan": true, 12 | "baseBranches": [], 13 | "snippetSize": 10 14 | }, 15 | "checkRunSettings": { 16 | "vulnerableCheckRunConclusionLevel": "failure", 17 | "displayMode": "diff", 18 | "useMendCheckNames": true 19 | }, 20 | "checkRunSettingsSAST": { 21 | "checkRunConclusionLevel": "failure", 22 | "severityThreshold": "high" 23 | }, 24 | "issueSettings": { 25 | "minSeverityLevel": "LOW", 26 | "issueType": "DEPENDENCY" 27 | }, 28 | "remediateSettings": { 29 | "enableRenovate": true, 30 | "extends": [ 31 | "config:base", 32 | ":gitSignOff", 33 | "github>whitesource/merge-confidence:beta" 34 | ], 35 | "addLabels": ["skip-changelog"], 36 | "workflowRules": { 37 | "enabled": true 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | All notable changes to this project are documented in this file. 3 | 4 | Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) 5 | 6 | ## [Unreleased 3.1](https://github.com/opensearch-project/flow-framework/compare/3.0...HEAD) 7 | ### Features 8 | ### Enhancements 9 | - Make thread pool sizes configurable ([#1139](https://github.com/opensearch-project/flow-framework/issues/1139)) 10 | 11 | ### Bug Fixes 12 | - Fixing llm field processing in RegisterAgentStep ([#1151](https://github.com/opensearch-project/flow-framework/pull/1151)) 13 | - Include exception type in WorkflowState error field even if no cause ([#1154](https://github.com/opensearch-project/flow-framework/pull/1154)) 14 | - Pass llm spec params to builder ([#1155](https://github.com/opensearch-project/flow-framework/pull/1155)) 15 | 16 | ### Infrastructure 17 | - Conditionally include ddb-client dependency only if env variable set ([#1141](https://github.com/opensearch-project/flow-framework/issues/1141)) 18 | 19 | ### Documentation 20 | - Feat: add data summary with log pattern agent template ([#1137](https://github.com/opensearch-project/flow-framework/pull/1137)) 21 | ### Maintenance 22 | ### Refactoring 23 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | 6 | This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media and any other communication channels used by the project. 7 | 8 | **Our open source communities endeavor to:** 9 | 10 | * Be Inclusive: We are committed to being a community where everyone can join and contribute. This means using inclusive and welcoming language. 11 | * Be Welcoming: We are committed to maintaining a safe space for everyone to be able to contribute. 12 | * Be Respectful: We are committed to encouraging differing viewpoints, accepting constructive criticism and work collaboratively towards decisions that help the project grow. Disrespectful and unacceptable behavior will not be tolerated. 13 | * Be Collaborative: We are committed to supporting what is best for our community and users. When we build anything for the benefit of the project, we should document the work we do and communicate to others on how this affects their work. 14 | 15 | **Our Responsibility. As contributors, members, or bystanders we each individually have the responsibility to behave professionally and respectfully at all times. Disrespectful and unacceptable behaviors include, but are not limited to:** 16 | 17 | * The use of violent threats, abusive, discriminatory, or derogatory language; 18 | * Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, race, political or religious affiliation; 19 | * Posting of sexually explicit or violent content; 20 | * The use of sexualized language and unwelcome sexual attention or advances; 21 | * Public or private harassment of any kind; 22 | * Publishing private information, such as physical or electronic address, without permission; 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting; 24 | * Advocating for or encouraging any of the above behaviors. 25 | 26 | **Enforcement and Reporting Code of Conduct Issues:** 27 | 28 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:opensource-codeofconduct@amazon.com). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. 29 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | 3 | This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). 4 | 5 | ## Current Maintainers 6 | 7 | | Maintainer | GitHub ID | Affiliation | 8 | | ----------------- | ------------------------------------------------------- | ----------- | 9 | | Junwei Dai | [junweid62](https://github.com/junweid62) | Amazon | 10 | | Amit Galitzky | [amitgalitz](https://github.com/amitgalitz) | Amazon | 11 | | Jackie Han | [jackiehanyang](https://github.com/jackiehanyang) | Amazon | 12 | | Owais Kazi | [owaiskazi19](https://github.com/owaiskazi19) | Amazon | 13 | | Tyler Ohlsen | [ohltyler](https://github.com/ohltyler) | Amazon | 14 | | Josh Palis | [joshpalis](https://github.com/joshpalis) | Amazon | 15 | | Dan Widdis | [dbwiddis](https://github.com/dbwiddis) | Amazon | 16 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | OpenSearch (https://opensearch.org/) 2 | Copyright OpenSearch Contributors 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## OpenSearch Flow Framework 2 | 3 | This project is an OpenSearch plugin that enables builders to innovate AI applications on OpenSearch. 4 | 5 | The current process of using ML offerings in OpenSearch, such as Semantic Search, requires users to handle complex setup and pre-processing tasks, and send verbose user queries, both of which can be time-consuming and error-prone. 6 | 7 | We want to introduce our customers to a new no-code/low-code builder experience ([Backend RFC](https://github.com/opensearch-project/OpenSearch/issues/9213) and [Frontend RFC](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/4755)) that empowers users to compose AI-augmented query and ingestion flows, integrate ML models supported by ML-Commons, and streamline the OpenSearch app development experience through a drag-and-drop designer. The front end will help users create use case templates, which provide a compact description of configuration steps for automated workflows such as Retrieval Augment Generation (RAG), AI connectors and other components that prime OpenSearch as a backend to leverage generative models. Once primed, builders can query OpenSearch directly without building middleware logic to stitch together data flows and ML models. 8 | 9 | While the initial development has focused on ML use cases, the framework is generic and can be adapted to other use cases. 10 | 11 | See the [Development Plan](https://github.com/opensearch-project/flow-framework/issues/475) to view or comment on current incremental development priorities. 12 | 13 | ## Security 14 | 15 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 16 | 17 | ## License 18 | 19 | This project is licensed under the Apache-2.0 License. 20 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Reporting a Vulnerability 2 | 3 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue. 4 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 13 | 14 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle_suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /formatter/formatting.gradle: -------------------------------------------------------------------------------- 1 | allprojects { 2 | spotless { 3 | java { 4 | // Normally this isn't necessary, but we have Java sources in 5 | // non-standard places 6 | target '**/*.java' 7 | 8 | removeUnusedImports() 9 | importOrder('de.thetaphi', 10 | 'com.carrotsearch', 11 | 'com.fasterxml', 12 | 'com.avast', 13 | 'com.sun', 14 | 'com.maxmind|com.github|com.networknt|groovy|nebula', 15 | 'org.antlr', 16 | 'software.amazon', 17 | 'com.azure|com.microsoft|com.ibm|com.google|joptsimple|org.apache|org.bouncycastle|org.codehaus|org.opensearch|org.objectweb|org.joda|org.hamcrest|org.openjdk|org.gradle|org.junit', 18 | 'javax', 19 | 'java', 20 | '', 21 | '\\#java|\\#org.opensearch|\\#org.hamcrest|\\#') 22 | eclipse().configFile rootProject.file('formatter/formatterConfig.xml') 23 | trimTrailingWhitespace() 24 | endWithNewline() 25 | 26 | // See DEVELOPER_GUIDE.md for details of when to enable this. 27 | if (System.getProperty('spotless.paddedcell') != null) { 28 | paddedCell() 29 | } 30 | } 31 | format 'misc', { 32 | target '*.md', '*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg' 33 | 34 | trimTrailingWhitespace() 35 | endWithNewline() 36 | leadingTabsToSpaces() 37 | } 38 | format("license", { 39 | licenseHeaderFile("${rootProject.file("formatter/license-header.txt")}", "package ") 40 | target("src/*/java/**/*.java") 41 | }) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /formatter/license-header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/flow-framework/deb02e7bf3354a845e80cd87292346ea6a261cf0/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionSha256Sum=61ad310d3c7d3e5da131b76bbf22b5a4c0786e9d892dae8c1658d4b484de3caa 4 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip 5 | networkTimeout=10000 6 | validateDistributionUrl=true 7 | zipStoreBase=GRADLE_USER_HOME 8 | zipStorePath=wrapper/dists 9 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.12.0.0.md: -------------------------------------------------------------------------------- 1 | ## 2024-02-20 Version 2.12.0.0 2 | 3 | Compatible with OpenSearch 2.12.0 4 | 5 | ### Features 6 | * Initial release of Flow Framework -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.13.0.0.md: -------------------------------------------------------------------------------- 1 | ## 2024-03-18 Version 2.13.0.0 2 | 3 | Compatible with OpenSearch 2.13.0 4 | 5 | ### Features 6 | * Added create ingest pipeline step ([#558](https://github.com/opensearch-project/flow-framework/pull/558)) 7 | * Added create search pipeline step ([#569](https://github.com/opensearch-project/flow-framework/pull/569)) 8 | * Added create index step ([#574](https://github.com/opensearch-project/flow-framework/pull/574)) 9 | * Added default use cases ([#583](https://github.com/opensearch-project/flow-framework/pull/583)) 10 | 11 | ### Enhancements 12 | * Substitute REST path or body parameters in Workflow Steps ([#525](https://github.com/opensearch-project/flow-framework/pull/525)) 13 | * Added an optional workflow_step param to the get workflow steps API ([#538](https://github.com/opensearch-project/flow-framework/pull/538)) 14 | * Add created, updated, and provisioned timestamps to saved template ([#551](https://github.com/opensearch-project/flow-framework/pull/551)) 15 | * Enable Flow Framework by default ([#553](https://github.com/opensearch-project/flow-framework/pull/553)) 16 | * Adding new exception type for workflow step failures ([#577](https://github.com/opensearch-project/flow-framework/pull/577)) 17 | 18 | ### Refactoring 19 | * Moved workflow-steps.json to Enum ([#523](https://github.com/opensearch-project/flow-framework/pull/523)) 20 | * Refactored logging for consistency ([#524](https://github.com/opensearch-project/flow-framework/pull/524)) 21 | 22 | ### Bug Fixes 23 | * Fixing create index and use case input parsing bugs ([#600](https://github.com/opensearch-project/flow-framework/pull/600)) 24 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.14.0.0.md: -------------------------------------------------------------------------------- 1 | ## 2024-04-30 Version 2.14.0.0 2 | 3 | Compatible with OpenSearch 2.14.0 4 | 5 | ### Enhancements 6 | - Add guardrails to default use case params ([#658](https://github.com/opensearch-project/flow-framework/pull/658)) 7 | - Allow strings for boolean workflow step parameters ([#671](https://github.com/opensearch-project/flow-framework/pull/671)) 8 | - Add optional delay parameter to no-op step ([#674](https://github.com/opensearch-project/flow-framework/pull/674)) 9 | - Add model interface support for remote and local custom models ([#701](https://github.com/opensearch-project/flow-framework/pull/701)) 10 | 11 | ### Bug Fixes 12 | - Reset workflow state to initial state after successful deprovision ([#635](https://github.com/opensearch-project/flow-framework/pull/635)) 13 | - Silently ignore content on APIs that don't require it ([#639](https://github.com/opensearch-project/flow-framework/pull/639)) 14 | - Hide user and credential field from search response ([#680](https://github.com/opensearch-project/flow-framework/pull/680)) 15 | - Throw the correct error message in status API for WorkflowSteps ([#676](https://github.com/opensearch-project/flow-framework/pull/676)) 16 | - Delete workflow state when template is deleted and no resources exist ([#689](https://github.com/opensearch-project/flow-framework/pull/689)) 17 | - Fixing model group parsing and restoring context ([#695] (https://github.com/opensearch-project/flow-framework/pull/695)) 18 | 19 | 20 | ### Infrastructure 21 | - Switch macOS runner to macos-13 from macos-latest since macos-latest is now arm64 ([#686](https://github.com/opensearch-project/flow-framework/pull/686)) 22 | 23 | ### Refactoring 24 | - Improve error messages for workflow states other than NOT_STARTED ([#642](https://github.com/opensearch-project/flow-framework/pull/642)) 25 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.15.0.0.md: -------------------------------------------------------------------------------- 1 | ## Version 2.15.0.0 2 | 3 | Compatible with OpenSearch 2.15.0 4 | 5 | ### Enhancements 6 | - Add Workflow Step for Reindex from source index to destination ([#718](https://github.com/opensearch-project/flow-framework/pull/718)) 7 | - Add param to delete workflow API to clear status even if resources exist ([#719](https://github.com/opensearch-project/flow-framework/pull/719)) 8 | - Add additional default use cases ([#731](https://github.com/opensearch-project/flow-framework/pull/731)) 9 | - Add conversation search default use case with RAG tool ([#732](https://github.com/opensearch-project/flow-framework/pull/732)) 10 | 11 | ### Bug Fixes 12 | - Add user mapping to Workflow State index ([#705](https://github.com/opensearch-project/flow-framework/pull/705)) 13 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.16.0.0.md: -------------------------------------------------------------------------------- 1 | ## Version 2.16.0.0 2 | 3 | Compatible with OpenSearch 2.16.0 4 | 5 | ### Enhancements 6 | - Register system index descriptors through SystemIndexPlugin.getSystemIndexDescriptors ([#750](https://github.com/opensearch-project/flow-framework/pull/750)) 7 | - Support editing of certain workflow fields on a provisioned workflow ([#757](https://github.com/opensearch-project/flow-framework/pull/757)) 8 | - Add allow_delete parameter to Deprovision API ([#763](https://github.com/opensearch-project/flow-framework/pull/763)) 9 | - Improve Template and WorkflowState builders ([#778](https://github.com/opensearch-project/flow-framework/pull/778)) 10 | 11 | ### Bug Fixes 12 | - Handle Not Found deprovision exceptions as successful deletions ([#805](https://github.com/opensearch-project/flow-framework/pull/805)) 13 | - Wrap CreateIndexRequest mappings in _doc key as required ([#809](https://github.com/opensearch-project/flow-framework/pull/809)) 14 | - Have FlowFrameworkException status recognized by ExceptionsHelper ([#811](https://github.com/opensearch-project/flow-framework/pull/811)) 15 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.17.0.0.md: -------------------------------------------------------------------------------- 1 | ## Version 2.17.0.0 2 | 3 | Compatible with OpenSearch 2.17.0 4 | 5 | ### Features 6 | - Adds reprovision API to support updating search pipelines, ingest pipelines index settings ([#804](https://github.com/opensearch-project/flow-framework/pull/804)) 7 | - Adds user level access control based on backend roles ([#838](https://github.com/opensearch-project/flow-framework/pull/838)) 8 | - Support parsing connector_id when creating tools ([#846](https://github.com/opensearch-project/flow-framework/pull/846)) 9 | 10 | ### Refactoring 11 | - Refactor workflow step resource updates to eliminate duplication ([#796](https://github.com/opensearch-project/flow-framework/pull/796)) 12 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.17.1.0.md: -------------------------------------------------------------------------------- 1 | ## Version 2.17.1.0 2 | 3 | Compatible with OpenSearch 2.17.1 4 | 5 | ### Maintenance 6 | - Fix flaky integ test reprovisioning before template update ([#880](https://github.com/opensearch-project/flow-framework/pull/880)) 7 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.18.0.0.md: -------------------------------------------------------------------------------- 1 | ## Version 2.18.0.0 2 | 3 | Compatible with OpenSearch 2.18.0 4 | 5 | ### Features 6 | - Add ApiSpecFetcher for Fetching and Comparing API Specifications ([#651](https://github.com/opensearch-project/flow-framework/issues/651)) 7 | - Add optional config field to tool step ([#899](https://github.com/opensearch-project/flow-framework/pull/899)) 8 | 9 | ### Enhancements 10 | - Incrementally remove resources from workflow state during deprovisioning ([#898](https://github.com/opensearch-project/flow-framework/pull/898)) 11 | 12 | ### Bug Fixes 13 | - Fixed Template Update Location and Improved Logger Statements in ReprovisionWorkflowTransportAction ([#918](https://github.com/opensearch-project/flow-framework/pull/918)) 14 | 15 | ### Documentation 16 | - Add query assist data summary agent into sample templates ([#875](https://github.com/opensearch-project/flow-framework/pull/875)) 17 | 18 | ### Refactoring 19 | - Update workflow state without using painless script ([#894](https://github.com/opensearch-project/flow-framework/pull/894)) 20 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.19.0.0.md: -------------------------------------------------------------------------------- 1 | ## Version 2.19.0.0 2 | 3 | Compatible with OpenSearch 2.19.0 4 | 5 | ### Features 6 | - Add multitenant remote metadata client ([#980](https://github.com/opensearch-project/flow-framework/pull/980)) 7 | - Add synchronous execution option to workflow provisioning ([#990](https://github.com/opensearch-project/flow-framework/pull/990)) 8 | 9 | ### Bug Fixes 10 | - Remove useCase and defaultParams field in WorkflowRequest ([#758](https://github.com/opensearch-project/flow-framework/pull/758)) 11 | - Fix RBAC fetching from workflow state when template is not present ([#998](https://github.com/opensearch-project/flow-framework/pull/998)) 12 | 13 | ### Refactoring 14 | - Replace String concatenation with Log4j ParameterizedMessage for readability ([#943](https://github.com/opensearch-project/flow-framework/pull/943)) 15 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-2.19.2.0.md: -------------------------------------------------------------------------------- 1 | ## Version 2.19.2.0 2 | 3 | Compatible with OpenSearch 2.19.2 4 | 5 | ### Bug Fixes 6 | - Fix Config parser does not handle tenant_id field ([#1096](https://github.com/opensearch-project/flow-framework/pull/1096)) 7 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-3.0.0.0-alpha1.md: -------------------------------------------------------------------------------- 1 | ## Version 3.0.0.0-alpha1 Release Notes 2 | 3 | Compatible with OpenSearch 3.0.0-alpha1 4 | 5 | ### Infrastructure 6 | - Set Java target compatibility to JDK 21 ([#730](https://github.com/opensearch-project/flow-framework/pull/730)) 7 | 8 | ### Documentation 9 | - Add text to visualization agent template ([#936](https://github.com/opensearch-project/flow-framework/pull/936)) 10 | 11 | ### Maintenance 12 | - Fix breaking changes for 3.0.0 release ([#1026](https://github.com/opensearch-project/flow-framework/pull/1026)) -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-3.0.0.0-beta1.md: -------------------------------------------------------------------------------- 1 | ## Version 3.0.0.0-beta1 Release Notes 2 | 3 | Compatible with OpenSearch 3.0.0-beta1 4 | 5 | ### Features 6 | - Add per-tenant provisioning throttling ([#1074](https://github.com/opensearch-project/flow-framework/pull/1074)) 7 | 8 | ### Bug Fixes 9 | - Change REST status codes for RBAC and provisioning ([#1083](https://github.com/opensearch-project/flow-framework/pull/1083)) 10 | - Fix Config parser does not handle tenant_id field ([#1096](https://github.com/opensearch-project/flow-framework/pull/1096)) 11 | - Complete action listener on failed synchronous workflow provisioning ([#1098](https://github.com/opensearch-project/opensearch-remote-metadata-sdk/pull/1098)) 12 | 13 | ### Maintenance 14 | - Migrate from BC to BCFIPS libraries ([#1087](https://github.com/opensearch-project/flow-framework/pull/1087)) 15 | -------------------------------------------------------------------------------- /release-notes/opensearch-flow-framework.release-notes-3.0.0.0.md: -------------------------------------------------------------------------------- 1 | ## Version 3.0.0.0 Release Notes 2 | 3 | Compatible with OpenSearch 3.0.0 4 | 5 | ### Features 6 | - Add per-tenant provisioning throttling ([#1074](https://github.com/opensearch-project/flow-framework/pull/1074)) 7 | 8 | ### Bug Fixes 9 | - Change REST status codes for RBAC and provisioning ([#1083](https://github.com/opensearch-project/flow-framework/pull/1083)) 10 | - Fix Config parser does not handle tenant_id field ([#1096](https://github.com/opensearch-project/flow-framework/pull/1096)) 11 | - Complete action listener on failed synchronous workflow provisioning ([#1098](https://github.com/opensearch-project/opensearch-remote-metadata-sdk/pull/1098)) 12 | - Add new attributes field to ToolStep ([#1113](https://github.com/opensearch-project/flow-framework/pull/1113)) 13 | - Fix bug handleReprovision missing wait_for_completion_timeout response ([#1107](https://github.com/opensearch-project/flow-framework/pull/1107)) 14 | 15 | ### Maintenance 16 | - Fix breaking changes for 3.0.0 release ([#1026](https://github.com/opensearch-project/flow-framework/pull/1026)) 17 | - Migrate from BC to BCFIPS libraries ([#1087](https://github.com/opensearch-project/flow-framework/pull/1087)) 18 | 19 | ### Infrastructure 20 | - Set Java target compatibility to JDK 21 ([#730](https://github.com/opensearch-project/flow-framework/pull/730)) 21 | - Use java-agent Gradle plugin to support phasing off SecurityManager usage in favor of Java Agent ([#1108](https://github.com/opensearch-project/flow-framework/pull/1108)) 22 | 23 | 24 | ### Documentation 25 | - Add text to visualization agent template ([#936](https://github.com/opensearch-project/flow-framework/pull/936)) 26 | -------------------------------------------------------------------------------- /sample-templates/README.md: -------------------------------------------------------------------------------- 1 | ## Flow Framework Sample Templates 2 | 3 | This folder contains sample workflow templates that can be used with Flow Framework. 4 | 5 | Each template is provided in both YAML and JSON format with identical functionality. 6 | The YAML templates include comments which give more insight into the template's usage. 7 | Use the corresponding `Content-Type` (`application/yaml` or `application/json`) when providing them as the body of a REST request. 8 | 9 | **Note:** Several of the templates use both the single quote (`'`) and double quote (`"`) characters which may create issues using the templates with `curl` on the command line and the template in `--data`. Escaping the single quotes or reading the template from a file is needed to work around this. 10 | 11 | You will need to update the `credentials` field with appropriate API keys. 12 | 13 | To create a workflow and provision the resources: 14 | 15 | ``` 16 | POST /_plugins/_flow_framework/workflow?provision=true 17 | { template as body } 18 | ``` 19 | 20 | This will return a `workflow_id`. To get the IDs of created resources, call the workflow status API. 21 | 22 | ``` 23 | GET /_plugins/_flow_framework/workflow//_status 24 | ``` 25 | 26 | For the Query Assist Agent API, the `agent_id` of the `root_agent` can be used to query it. 27 | 28 | ``` 29 | POST /_plugins/_ml/agents/_/_execute 30 | { 31 | "parameters": { 32 | "question": "How many 5xx logs do I have?" 33 | } 34 | } 35 | ``` -------------------------------------------------------------------------------- /sample-templates/alert-summary-agent-claude-tested.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Alert Summary Agent 3 | description: Create Alert Summary Agent using Claude on BedRock 4 | use_case: REGISTER_AGENT 5 | version: 6 | template: 1.0.0 7 | compatibility: 8 | - 2.17.0 9 | - 3.0.0 10 | workflows: 11 | provision: 12 | user_params: {} 13 | nodes: 14 | - id: create_claude_connector 15 | type: create_connector 16 | previous_node_inputs: {} 17 | user_inputs: 18 | version: '1' 19 | name: Claude instant runtime Connector 20 | protocol: aws_sigv4 21 | description: The connector to BedRock service for Claude model 22 | actions: 23 | - headers: 24 | x-amz-content-sha256: required 25 | content-type: application/json 26 | method: POST 27 | request_body: '{"prompt":"\n\nHuman: ${parameters.prompt}\n\nAssistant:", "max_tokens_to_sample":${parameters.max_tokens_to_sample}, 28 | "temperature":${parameters.temperature}, "anthropic_version":"${parameters.anthropic_version}" 29 | }' 30 | action_type: predict 31 | url: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-instant-v1/invoke 32 | credential: 33 | access_key: "" 34 | secret_key: "" 35 | session_token: "" 36 | parameters: 37 | region: us-west-2 38 | endpoint: bedrock-runtime.us-west-2.amazonaws.com 39 | content_type: application/json 40 | auth: Sig_V4 41 | max_tokens_to_sample: '8000' 42 | service_name: bedrock 43 | temperature: '0.0001' 44 | response_filter: "$.completion" 45 | anthropic_version: bedrock-2023-05-31 46 | - id: register_claude_model 47 | type: register_remote_model 48 | previous_node_inputs: 49 | create_claude_connector: connector_id 50 | user_inputs: 51 | description: Claude model 52 | deploy: true 53 | name: claude-instant 54 | - id: create_alert_summary_ml_model_tool 55 | type: create_tool 56 | previous_node_inputs: 57 | register_claude_model: model_id 58 | user_inputs: 59 | parameters: 60 | prompt: "You are an OpenSearch Alert Assistant to help summarize the alerts.\n Here is the detail of alert: ${parameters.context};\n The question is: ${parameters.question}." 61 | name: MLModelTool 62 | type: MLModelTool 63 | - id: create_alert_summary_agent 64 | type: register_agent 65 | previous_node_inputs: 66 | create_alert_summary_ml_model_tool: tools 67 | user_inputs: 68 | parameters: {} 69 | type: flow 70 | name: Alert Summary Agent 71 | description: this is an alert summary agent 72 | -------------------------------------------------------------------------------- /sample-templates/deploy-bedrock-claude-model.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Deploy Claude Model", 3 | "description": "Deploy a model using a connector to Claude", 4 | "use_case": "PROVISION", 5 | "version": { 6 | "template": "1.0.0", 7 | "compatibility": [ 8 | "2.12.0", 9 | "3.0.0" 10 | ] 11 | }, 12 | "workflows": { 13 | "provision": { 14 | "nodes": [ 15 | { 16 | "id": "create_claude_connector", 17 | "type": "create_connector", 18 | "user_inputs": { 19 | "name": "Claude Instant Runtime Connector", 20 | "version": "1", 21 | "protocol": "aws_sigv4", 22 | "description": "The connector to BedRock service for Claude model", 23 | "actions": [ 24 | { 25 | "headers": { 26 | "x-amz-content-sha256": "required", 27 | "content-type": "application/json" 28 | }, 29 | "method": "POST", 30 | "request_body": "{ \"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature}, \"anthropic_version\":\"${parameters.anthropic_version}\" }", 31 | "action_type": "predict", 32 | "url": "https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-instant-v1/invoke" 33 | } 34 | ], 35 | "credential": { 36 | "access_key": "PUT_YOUR_ACCESS_KEY_HERE", 37 | "secret_key": "PUT_YOUR_SECRET_KEY_HERE" 38 | }, 39 | "parameters": { 40 | "endpoint": "bedrock-runtime.us-west-2.amazonaws.com", 41 | "content_type": "application/json", 42 | "auth": "Sig_V4", 43 | "max_tokens_to_sample": "8000", 44 | "service_name": "bedrock", 45 | "temperature": "0.0001", 46 | "response_filter": "$.completion", 47 | "region": "us-west-2", 48 | "anthropic_version": "bedrock-2023-05-31" 49 | } 50 | } 51 | }, 52 | { 53 | "id": "register_claude_model", 54 | "type": "register_remote_model", 55 | "previous_node_inputs": { 56 | "create_claude_connector": "connector_id" 57 | }, 58 | "user_inputs": { 59 | "name": "claude-instant" 60 | } 61 | }, 62 | { 63 | "id": "deploy_claude_model", 64 | "type": "deploy_model", 65 | "previous_node_inputs": { 66 | "register_claude_model": "model_id" 67 | } 68 | } 69 | ] 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /sample-templates/deploy-bedrock-claude-model.yml: -------------------------------------------------------------------------------- 1 | # This template creates a connector to the BedRock service for Claude model 2 | # It then registers a model using that connector and deploys it. 3 | # 4 | # To use: 5 | # - update the "credential" fields under the create_claude_connector node. 6 | # - if needed, update region 7 | # 8 | # After provisioning: 9 | # - returns a workflow ID 10 | # - use the status API to get the deployed model ID 11 | --- 12 | name: Deploy Claude Model 13 | description: Deploy a model using a connector to Claude 14 | use_case: PROVISION 15 | version: 16 | template: 1.0.0 17 | compatibility: 18 | - 2.12.0 19 | - 3.0.0 20 | # This section defines the provision workflow. Nodes are connected in a graph. 21 | # Either previous_node_inputs or explicit edges can be used to enforce ordering. 22 | workflows: 23 | provision: 24 | # Each id field in a workflow must be unique. 25 | nodes: 26 | - id: create_claude_connector 27 | type: create_connector 28 | user_inputs: 29 | name: Claude Instant Runtime Connector 30 | version: '1' 31 | protocol: aws_sigv4 32 | description: The connector to BedRock service for Claude model 33 | actions: 34 | - headers: 35 | x-amz-content-sha256: required 36 | content-type: application/json 37 | method: POST 38 | request_body: '{ 39 | "prompt":"${parameters.prompt}", 40 | "max_tokens_to_sample":${parameters.max_tokens_to_sample}, 41 | "temperature":${parameters.temperature}, "anthropic_version":"${parameters.anthropic_version}" 42 | }' 43 | action_type: predict 44 | url: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-instant-v1/invoke 45 | credential: 46 | access_key: 'PUT_YOUR_ACCESS_KEY_HERE' 47 | secret_key: 'PUT_YOUR_SECRET_KEY_HERE' 48 | parameters: 49 | endpoint: bedrock-runtime.us-west-2.amazonaws.com 50 | content_type: application/json 51 | auth: Sig_V4 52 | max_tokens_to_sample: '8000' 53 | service_name: bedrock 54 | temperature: '0.0001' 55 | response_filter: "$.completion" 56 | region: us-west-2 57 | anthropic_version: bedrock-2023-05-31 58 | - id: register_claude_model 59 | type: register_remote_model 60 | previous_node_inputs: 61 | create_claude_connector: connector_id 62 | user_inputs: 63 | name: claude-instant 64 | # Using deploy: true here would both registers and deploy the model 65 | # and the deploy model step below could be deleted 66 | # deploy: true 67 | - id: deploy_claude_model 68 | type: deploy_model 69 | previous_node_inputs: 70 | register_claude_model: model_id 71 | # Because the above nodes use previous_node_inputs, these edges are automatically generated. 72 | # edges: 73 | # - source: create_claude_connector 74 | # dest: register_claude_model 75 | # - source: register_claude_model 76 | # dest: deploy_claude_model 77 | -------------------------------------------------------------------------------- /sample-templates/deploy-openai-model.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Deploy OpenAI Model", 3 | "description": "Deploy a model using a connector to OpenAI", 4 | "use_case": "PROVISION", 5 | "version": { 6 | "template": "1.0.0", 7 | "compatibility": [ 8 | "2.12.0", 9 | "3.0.0" 10 | ] 11 | }, 12 | "workflows": { 13 | "provision": { 14 | "nodes": [ 15 | { 16 | "id": "create_openai_connector", 17 | "type": "create_connector", 18 | "user_inputs": { 19 | "name": "OpenAI Chat Connector", 20 | "description": "The connector to public OpenAI model service for GPT 3.5", 21 | "version": "1", 22 | "protocol": "http", 23 | "parameters": { 24 | "endpoint": "api.openai.com", 25 | "model": "gpt-3.5-turbo", 26 | "response_filter": "$.choices[0].message.content" 27 | }, 28 | "credential": { 29 | "openAI_key": "PUT_YOUR_API_KEY_HERE" 30 | }, 31 | "actions": [ 32 | { 33 | "action_type": "predict", 34 | "method": "POST", 35 | "url": "https://${parameters.endpoint}/v1/chat/completions" 36 | } 37 | ] 38 | } 39 | }, 40 | { 41 | "id": "register_openai_model", 42 | "type": "register_remote_model", 43 | "previous_node_inputs": { 44 | "create_openai_connector": "connector_id" 45 | }, 46 | "user_inputs": { 47 | "name": "openAI-gpt-3.5-turbo" 48 | } 49 | }, 50 | { 51 | "id": "deploy_openai_model", 52 | "type": "deploy_model", 53 | "previous_node_inputs": { 54 | "register_openai_model": "model_id" 55 | } 56 | } 57 | ] 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /sample-templates/deploy-openai-model.yml: -------------------------------------------------------------------------------- 1 | # This template creates a connector to the public OpenAI model service for GPT 3.5 2 | # It then registers a model using that connector and deploys it. 3 | # 4 | # To use: 5 | # - update the "credential" field under the create_openai_connector node. 6 | # 7 | # After provisioning: 8 | # - returns a workflow ID 9 | # - use the status API to get the deployed model ID 10 | --- 11 | name: Deploy OpenAI Model 12 | description: Deploy a model using a connector to OpenAI 13 | use_case: PROVISION 14 | version: 15 | template: 1.0.0 16 | compatibility: 17 | - 2.12.0 18 | - 3.0.0 19 | # This section defines the provision workflow. Nodes are connected in a graph. 20 | # Either previous_node_inputs or explicit edges can be used to enforce ordering. 21 | workflows: 22 | provision: 23 | # Each id field in a workflow must be unique. 24 | nodes: 25 | - id: create_openai_connector 26 | type: create_connector 27 | user_inputs: 28 | name: OpenAI Chat Connector 29 | description: The connector to public OpenAI model service for GPT 3.5 30 | version: '1' 31 | protocol: http 32 | parameters: 33 | endpoint: api.openai.com 34 | model: gpt-3.5-turbo 35 | response_filter: '$.choices[0].message.content' 36 | credential: 37 | openAI_key: 'PUT_YOUR_API_KEY_HERE' 38 | actions: 39 | - action_type: predict 40 | method: POST 41 | url: https://${parameters.endpoint}/v1/chat/completions 42 | - id: register_openai_model 43 | type: register_remote_model 44 | previous_node_inputs: 45 | create_openai_connector: connector_id 46 | user_inputs: 47 | name: openAI-gpt-3.5-turbo 48 | # Using deploy: true here would both registers and deploy the model 49 | # and the deploy model step below could be deleted 50 | # deploy: true 51 | - id: deploy_openai_model 52 | type: deploy_model 53 | previous_node_inputs: 54 | register_openai_model: model_id 55 | # Because the above nodes use previous_node_inputs, these edges are automatically generated. 56 | # edges: 57 | # - source: create_openai_connector 58 | # dest: register_openai_model 59 | # - source: register_openai_model 60 | # dest: deploy_openai_model 61 | -------------------------------------------------------------------------------- /sample-templates/deploy-sagemaker-mistral-model.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Deploy Mistral Model", 3 | "description": "Deploy a model using a connector to SageMaker Mistral model", 4 | "use_case": "PROVISION", 5 | "version": { 6 | "template": "1.0.0", 7 | "compatibility": [ 8 | "2.12.0", 9 | "3.0.0" 10 | ] 11 | }, 12 | "workflows": { 13 | "provision": { 14 | "nodes": [ 15 | { 16 | "id": "create_mistral_connector", 17 | "type": "create_connector", 18 | "user_inputs": { 19 | "name": "sagemaker: mistral", 20 | "description": "Test connector for Sagemaker mistral model", 21 | "version": "1", 22 | "protocol": "aws_sigv4", 23 | "credential": { 24 | "access_key": "PUT_YOUR_ACCESS_KEY_HERE", 25 | "secret_key": "PUT_YOUR_SECRET_KEY_HERE" 26 | }, 27 | "parameters": { 28 | "region": "us-east-1", 29 | "service_name": "sagemaker" 30 | }, 31 | "actions": [ 32 | { 33 | "action_type": "predict", 34 | "method": "POST", 35 | "headers": { 36 | "content-type": "application/json" 37 | }, 38 | "url": "https://PUT_YOUR_CUSTOM_SAGEMAKER_ENDPOINT_HERE", 39 | "request_body": "{\"prompt\":\"${parameters.prompt}\"}" 40 | } 41 | ] 42 | } 43 | }, 44 | { 45 | "id": "register_mistral_model", 46 | "type": "register_remote_model", 47 | "previous_node_inputs": { 48 | "create_mistral_connector": "connector_id" 49 | }, 50 | "user_inputs": { 51 | "name": "mistral fine-tuned model" 52 | } 53 | }, 54 | { 55 | "id": "deploy_mistral_model", 56 | "type": "deploy_model", 57 | "previous_node_inputs": { 58 | "register_mistral_model": "model_id" 59 | } 60 | } 61 | ] 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /sample-templates/deploy-sagemaker-mistral-model.yml: -------------------------------------------------------------------------------- 1 | # This template creates a connector to the SageMaker service for a Mistral model 2 | # It then registers a model using that connector and deploys it. 3 | # 4 | # To use: 5 | # - update the "credential" fields under the create_mistral_connector node. 6 | # - update the sagemaker endpoint 7 | # - if needed, update region 8 | # 9 | # After provisioning: 10 | # - returns a workflow ID 11 | # - use the status API to get the deployed model ID 12 | --- 13 | name: Deploy Mistral Model 14 | description: Deploy a model using a connector to SageMaker Mistral model 15 | use_case: PROVISION 16 | version: 17 | template: 1.0.0 18 | compatibility: 19 | - 2.12.0 20 | - 3.0.0 21 | # This section defines the provision workflow. Nodes are connected in a graph. 22 | # Either previous_node_inputs or explicit edges can be used to enforce ordering. 23 | workflows: 24 | provision: 25 | # Each id field in a workflow must be unique. 26 | nodes: 27 | - id: create_mistral_connector 28 | type: create_connector 29 | user_inputs: 30 | name: 'sagemaker: mistral' 31 | description: Test connector for Sagemaker mistral model 32 | version: '1' 33 | protocol: aws_sigv4 34 | credential: 35 | access_key: 'PUT_YOUR_ACCESS_KEY_HERE' 36 | secret_key: 'PUT_YOUR_SECRET_KEY_HERE' 37 | parameters: 38 | region: us-east-1 39 | service_name: sagemaker 40 | actions: 41 | - action_type: predict 42 | method: POST 43 | headers: 44 | content-type: application/json 45 | url: 'https://PUT_YOUR_CUSTOM_SAGEMAKER_ENDPOINT_HERE' 46 | request_body: '{"prompt":"${parameters.prompt}"}' 47 | - id: register_mistral_model 48 | type: register_remote_model 49 | previous_node_inputs: 50 | create_mistral_connector: connector_id 51 | user_inputs: 52 | name: mistral fine-tuned model 53 | # Using deploy: true here would both registers and deploy the model 54 | # and the deploy model step below could be deleted 55 | # deploy: true 56 | - id: deploy_mistral_model 57 | type: deploy_model 58 | previous_node_inputs: 59 | register_mistral_model: model_id 60 | # Because the above nodes use previous_node_inputs, these edges are automatically generated. 61 | # edges: 62 | # - source: create_mistral_connector 63 | # dest: register_mistral_model 64 | # - source: register_mistral_model 65 | # dest: deploy_mistral_model 66 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'opensearch-flow-framework' 2 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/common/ThrowingSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.common; 10 | 11 | /** 12 | * A supplier that can throw checked exception 13 | * 14 | * @param method parameter type 15 | * @param Exception type 16 | */ 17 | @FunctionalInterface 18 | public interface ThrowingSupplier { 19 | /** 20 | * Gets a result or throws an exception if unable to produce a result. 21 | * 22 | * @return the result 23 | * @throws E if unable to produce a result 24 | */ 25 | T get() throws E; 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/common/ThrowingSupplierWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.common; 10 | 11 | import java.util.function.Supplier; 12 | 13 | /** 14 | * Wrapper for throwing checked exception inside places that does not allow to do so 15 | */ 16 | public class ThrowingSupplierWrapper { 17 | 18 | private ThrowingSupplierWrapper() {} 19 | 20 | /** 21 | * Utility method to use a method throwing checked exception inside a place 22 | * that does not allow throwing the corresponding checked exception (e.g., 23 | * enum initialization). 24 | * Convert the checked exception thrown by throwingConsumer to a RuntimeException 25 | * so that the compiler won't complain. 26 | * @param the method's return type 27 | * @param throwingSupplier the method reference that can throw checked exception 28 | * @return converted method reference 29 | */ 30 | public static Supplier throwingSupplierWrapper(ThrowingSupplier throwingSupplier) { 31 | 32 | return () -> { 33 | try { 34 | return throwingSupplier.get(); 35 | } catch (Exception ex) { 36 | throw new RuntimeException(ex); 37 | } 38 | }; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/exception/ApiSpecParseException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.exception; 10 | 11 | import org.opensearch.OpenSearchException; 12 | 13 | import java.util.List; 14 | 15 | /** 16 | * Custom exception to be thrown when an error occurs during the parsing of an API specification. 17 | */ 18 | public class ApiSpecParseException extends OpenSearchException { 19 | 20 | /** 21 | * Constructor with message. 22 | * 23 | * @param message The detail message. 24 | */ 25 | public ApiSpecParseException(String message) { 26 | super(message); 27 | } 28 | 29 | /** 30 | * Constructor with message and cause. 31 | * 32 | * @param message The detail message. 33 | * @param cause The cause of the exception. 34 | */ 35 | public ApiSpecParseException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | /** 40 | * Constructor with message and list of detailed errors. 41 | * 42 | * @param message The detail message. 43 | * @param details The list of errors encountered during the parsing process. 44 | */ 45 | public ApiSpecParseException(String message, List details) { 46 | super(message + ": " + String.join(", ", details)); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/exception/WorkflowStepException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.exception; 10 | 11 | import org.opensearch.OpenSearchException; 12 | import org.opensearch.OpenSearchParseException; 13 | import org.opensearch.OpenSearchStatusException; 14 | import org.opensearch.core.rest.RestStatus; 15 | import org.opensearch.core.xcontent.ToXContentObject; 16 | import org.opensearch.core.xcontent.XContentBuilder; 17 | 18 | import java.io.IOException; 19 | 20 | /** 21 | * Representation of an exception that is caused by a workflow step failing outside of our plugin 22 | * This is caught by an external client (e.g. ml-client) returning the failure 23 | */ 24 | public class WorkflowStepException extends FlowFrameworkException implements ToXContentObject { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | /** 29 | * Constructor with error message. 30 | * 31 | * @param message message of the exception 32 | * @param restStatus HTTP status code of the response 33 | */ 34 | public WorkflowStepException(String message, RestStatus restStatus) { 35 | super(message, restStatus); 36 | } 37 | 38 | /** 39 | * Constructor with specified cause. 40 | * @param cause exception cause 41 | * @param restStatus HTTP status code of the response 42 | */ 43 | public WorkflowStepException(Throwable cause, RestStatus restStatus) { 44 | super(cause, restStatus); 45 | } 46 | 47 | /** 48 | * Constructor with specified error message adn cause. 49 | * @param message error message 50 | * @param cause exception cause 51 | * @param restStatus HTTP status code of the response 52 | */ 53 | public WorkflowStepException(String message, Throwable cause, RestStatus restStatus) { 54 | super(message, cause, restStatus); 55 | } 56 | 57 | /** 58 | * Getter for restStatus. 59 | * 60 | * @return the HTTP status code associated with the exception 61 | */ 62 | public RestStatus getRestStatus() { 63 | return restStatus; 64 | } 65 | 66 | @Override 67 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 68 | return builder.startObject().field("error", this.getMessage()).endObject(); 69 | } 70 | 71 | /** 72 | * Getter for safe exceptions 73 | * @param ex exception 74 | * @return exception if safe 75 | */ 76 | public static Exception getSafeException(Exception ex) { 77 | if (ex instanceof IllegalArgumentException 78 | || ex instanceof OpenSearchStatusException 79 | || ex instanceof OpenSearchParseException 80 | || (ex instanceof OpenSearchException && ex.getCause() instanceof OpenSearchParseException)) { 81 | return ex; 82 | } 83 | return null; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/indices/FlowFrameworkIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.indices; 10 | 11 | import org.opensearch.flowframework.common.ThrowingSupplierWrapper; 12 | 13 | import java.util.function.Supplier; 14 | 15 | import static org.opensearch.flowframework.common.CommonValue.CONFIG_INDEX; 16 | import static org.opensearch.flowframework.common.CommonValue.CONFIG_INDEX_VERSION; 17 | import static org.opensearch.flowframework.common.CommonValue.GLOBAL_CONTEXT_INDEX; 18 | import static org.opensearch.flowframework.common.CommonValue.GLOBAL_CONTEXT_INDEX_VERSION; 19 | import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_STATE_INDEX; 20 | import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_STATE_INDEX_VERSION; 21 | 22 | /** 23 | * An enumeration of Flow Framework indices 24 | */ 25 | public enum FlowFrameworkIndex { 26 | /** 27 | * Global Context Index 28 | */ 29 | GLOBAL_CONTEXT( 30 | GLOBAL_CONTEXT_INDEX, 31 | ThrowingSupplierWrapper.throwingSupplierWrapper(FlowFrameworkIndicesHandler::getGlobalContextMappings), 32 | GLOBAL_CONTEXT_INDEX_VERSION 33 | ), 34 | /** 35 | * Workflow State Index 36 | */ 37 | WORKFLOW_STATE( 38 | WORKFLOW_STATE_INDEX, 39 | ThrowingSupplierWrapper.throwingSupplierWrapper(FlowFrameworkIndicesHandler::getWorkflowStateMappings), 40 | WORKFLOW_STATE_INDEX_VERSION 41 | ), 42 | /** 43 | * Config Index 44 | */ 45 | CONFIG( 46 | CONFIG_INDEX, 47 | ThrowingSupplierWrapper.throwingSupplierWrapper(FlowFrameworkIndicesHandler::getConfigIndexMappings), 48 | CONFIG_INDEX_VERSION 49 | ); 50 | 51 | private final String indexName; 52 | private final String mapping; 53 | private final Integer version; 54 | 55 | FlowFrameworkIndex(String name, Supplier mappingSupplier, Integer version) { 56 | this.indexName = name; 57 | this.mapping = mappingSupplier.get(); 58 | this.version = version; 59 | } 60 | 61 | /** 62 | * Retrieves the index name 63 | * @return the index name 64 | */ 65 | public String getIndexName() { 66 | return indexName; 67 | } 68 | 69 | /** 70 | * Retrieves the index mapping 71 | * @return the index mapping 72 | */ 73 | public String getMapping() { 74 | return mapping; 75 | } 76 | 77 | /** 78 | * Retrieves the index version 79 | * @return the index version 80 | */ 81 | public Integer getVersion() { 82 | return version; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/model/ProvisioningProgress.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.model; 10 | 11 | /** 12 | * Enum relating to the provisioning progress 13 | */ 14 | // TODO: transfer this to more detailed array for each step 15 | public enum ProvisioningProgress { 16 | /** Not Started State */ 17 | NOT_STARTED, 18 | /** In Progress State */ 19 | IN_PROGRESS, 20 | /** Done State */ 21 | DONE, 22 | /** Failed State */ 23 | FAILED 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/model/State.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.model; 10 | 11 | /** 12 | * Enum relating to the state of a workflow 13 | */ 14 | public enum State { 15 | /** Not Started state */ 16 | NOT_STARTED, 17 | /** Provisioning state */ 18 | PROVISIONING, 19 | /** Failed state */ 20 | FAILED, 21 | /** Completed state */ 22 | COMPLETED 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/model/WorkflowValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.model; 10 | 11 | import org.opensearch.common.xcontent.json.JsonXContent; 12 | import org.opensearch.core.xcontent.ToXContentObject; 13 | import org.opensearch.core.xcontent.XContentBuilder; 14 | 15 | import java.io.IOException; 16 | import java.util.Map; 17 | 18 | /** 19 | * This represents the workflow steps json which maps each step to expected inputs and outputs 20 | */ 21 | public class WorkflowValidator implements ToXContentObject { 22 | 23 | private Map workflowStepValidators; 24 | 25 | /** 26 | * Intantiate the object representing a Workflow validator 27 | * @param workflowStepValidators a map of {@link WorkflowStepValidator} 28 | */ 29 | public WorkflowValidator(Map workflowStepValidators) { 30 | this.workflowStepValidators = workflowStepValidators; 31 | } 32 | 33 | /** 34 | * Output this object in a compact JSON string. 35 | * 36 | * @return a JSON representation of the template. 37 | */ 38 | public String toJson() { 39 | try { 40 | XContentBuilder builder = JsonXContent.contentBuilder(); 41 | return this.toXContent(builder, EMPTY_PARAMS).toString(); 42 | } catch (IOException e) { 43 | return "{\"error\": \"couldn't create JSON from XContent\"}"; 44 | } 45 | } 46 | 47 | /** 48 | * Get the map of WorkflowStepValidators 49 | * @return the map of WorkflowStepValidators 50 | */ 51 | public Map getWorkflowStepValidators() { 52 | return Map.copyOf(this.workflowStepValidators); 53 | } 54 | 55 | @Override 56 | public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { 57 | return xContentBuilder.map(workflowStepValidators); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/flowframework/rest/RestSearchWorkflowAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | package org.opensearch.flowframework.rest; 10 | 11 | import org.opensearch.flowframework.common.FlowFrameworkSettings; 12 | import org.opensearch.flowframework.model.Template; 13 | import org.opensearch.flowframework.transport.SearchWorkflowAction; 14 | 15 | import java.util.List; 16 | 17 | import static org.opensearch.flowframework.common.CommonValue.GLOBAL_CONTEXT_INDEX; 18 | import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_URI; 19 | 20 | /** 21 | * Rest Action to facilitate requests to search workflows 22 | */ 23 | public class RestSearchWorkflowAction extends AbstractSearchWorkflowAction