├── .github └── workflows │ ├── build_on_pr.yaml │ ├── ci.yaml │ └── release.yaml ├── .gitignore ├── CONFIGURATION.md ├── LICENSE ├── README.md ├── SECURITY.md ├── archive ├── README.md ├── build.gradle └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── orkes │ │ │ └── conductor │ │ │ ├── dao │ │ │ ├── archive │ │ │ │ ├── ArchiveDAO.java │ │ │ │ ├── ArchivedExecutionDAO.java │ │ │ │ ├── ArchivedIndexDAO.java │ │ │ │ ├── DocumentStoreDAO.java │ │ │ │ └── ScrollableSearchResult.java │ │ │ ├── indexer │ │ │ │ ├── IndexValueExtractor.java │ │ │ │ ├── IndexWorker.java │ │ │ │ ├── IndexWorkerProperties.java │ │ │ │ ├── TaskIndex.java │ │ │ │ └── WorkflowIndex.java │ │ │ └── postgres │ │ │ │ └── archive │ │ │ │ ├── PostgresArchiveDAO.java │ │ │ │ ├── PostgresArchiveDAOConfiguration.java │ │ │ │ └── SearchQuery.java │ │ │ ├── id │ │ │ └── TimeBasedUUIDGenerator.java │ │ │ └── metrics │ │ │ └── MetricsCollector.java │ └── resources │ │ └── db │ │ └── migration_archive_postgres │ │ └── V99__initial_schema.sql │ └── test │ ├── java │ └── io │ │ └── orkes │ │ └── conductor │ │ └── dao │ │ ├── archive │ │ └── TestScheduler.java │ │ ├── indexer │ │ └── TestIndexValueExtractor.java │ │ └── postgres │ │ ├── PostgresArchiveDAOTest.java │ │ ├── PostgresArchivePerformanceTest.java │ │ ├── PostgresDAOTestUtil.java │ │ └── UuidUtil.java │ └── resources │ ├── application-testazurearchive.properties │ ├── application-tests3archive.properties │ ├── drop_all.sql │ └── logback.xml ├── build.gradle ├── contributing.md ├── docker ├── DockerfileServer ├── DockerfileStandalone ├── build-ui.sh ├── config │ ├── config.properties │ ├── nginx.conf │ ├── redis.conf │ ├── start_all.sh │ └── startup.sh └── docker-compose.yaml ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── licenseheader.txt ├── persistence ├── build.gradle └── src │ ├── main │ └── java │ │ └── com │ │ └── netflix │ │ ├── conductor │ │ └── redis │ │ │ ├── config │ │ │ ├── AnyRedisCondition.java │ │ │ ├── InMemoryRedisConfiguration.java │ │ │ ├── RedisClusterConfiguration.java │ │ │ ├── RedisProperties.java │ │ │ ├── RedisSentinelConfiguration.java │ │ │ └── RedisStandaloneConfiguration.java │ │ │ ├── dao │ │ │ ├── BaseDynoDAO.java │ │ │ ├── OrkesMetadataDAO.java │ │ │ ├── RedisEventHandlerDAO.java │ │ │ ├── RedisExecutionDAO.java │ │ │ ├── RedisMetadataDAO.java │ │ │ ├── RedisPollDataDAO.java │ │ │ └── RedisRateLimitingDAO.java │ │ │ ├── dynoqueue │ │ │ ├── ConfigurationHostSupplier.java │ │ │ └── LocalhostHostSupplier.java │ │ │ └── jedis │ │ │ ├── JedisCluster.java │ │ │ ├── JedisMock.java │ │ │ ├── JedisSentinel.java │ │ │ ├── JedisStandalone.java │ │ │ ├── OrkesJedisCommands.java │ │ │ └── OrkesJedisProxy.java │ │ └── dyno │ │ └── connectionpool │ │ ├── Host.java │ │ ├── HostBuilder.java │ │ └── HostSupplier.java │ └── test │ ├── java │ └── com │ │ └── netflix │ │ └── conductor │ │ ├── common │ │ └── config │ │ │ └── TestObjectMapperConfiguration.java │ │ ├── dao │ │ ├── ExecutionDAOTest.java │ │ └── PollDataDAOTest.java │ │ └── redis │ │ ├── dao │ │ ├── BaseDynoDAOTest.java │ │ ├── RedisEventHandlerDAOTest.java │ │ ├── RedisExecutionDAOTest.java │ │ ├── RedisMetadataDAOTest.java │ │ ├── RedisPollDataDAOTest.java │ │ └── RedisRateLimitDAOTest.java │ │ └── jedis │ │ ├── ConfigurationHostSupplierTest.java │ │ ├── JedisClusterTest.java │ │ └── JedisSentinelTest.java │ └── resources │ ├── wf.json │ └── wf2.json ├── run_tests.sh ├── scripts └── run_local.sh ├── server ├── build.gradle └── src │ ├── main │ ├── java │ │ ├── com │ │ │ └── netflix │ │ │ │ └── conductor │ │ │ │ └── core │ │ │ │ └── execution │ │ │ │ └── OrkesWorkflowExecutor.java │ │ └── io │ │ │ └── orkes │ │ │ └── conductor │ │ │ ├── OrkesConductorApplication.java │ │ │ ├── config │ │ │ └── SecurityDisabledConfig.java │ │ │ ├── execution │ │ │ └── tasks │ │ │ │ ├── HttpSync.java │ │ │ │ ├── OrkesRestTemplateProvider.java │ │ │ │ ├── SubWorkflowSync.java │ │ │ │ └── mapper │ │ │ │ └── OrkesForkJoinDynamicTaskMapper.java │ │ │ ├── rest │ │ │ ├── VersionResource.java │ │ │ └── WorkflowResourceSync.java │ │ │ ├── server │ │ │ └── service │ │ │ │ ├── OrkesSweeperProperties.java │ │ │ │ ├── OrkesWorkflowSweepWorker.java │ │ │ │ └── OrkesWorkflowSweeper.java │ │ │ └── ui │ │ │ └── UIContextGenerator.java │ └── resources │ │ ├── application.properties │ │ ├── banner.txt │ │ ├── logback-spring.xml │ │ ├── tasks.json │ │ └── workflows.json │ └── test │ └── resources │ └── mockito-extensions │ └── org.mockito.plugins.MockMaker ├── settings.gradle └── test-harness ├── build.gradle └── src └── test ├── java └── io │ └── orkes │ └── conductor │ └── client │ └── e2e │ ├── AbstractConductorTest.java │ ├── BackoffTests.java │ ├── ConductorStartupTest.java │ ├── DynamicForkOptionalTests.java │ ├── FailureWorkflowTests.java │ ├── GraaljsTests.java │ ├── HttpWorkerTests.java │ ├── JSONJQTests.java │ ├── JavaSDKTests.java │ ├── SDKWorkers.java │ ├── SubWorkflowInlineTests.java │ ├── SubWorkflowTimeoutRetryTests.java │ ├── SubWorkflowVersionTests.java │ ├── WaitTaskTest.java │ ├── WorkflowRetryTests.java │ └── util │ ├── Commons.java │ ├── RegistrationUtil.java │ ├── SimpleWorker.java │ └── WorkflowUtil.java └── resources ├── logback-test.xml ├── metadata ├── fail.json ├── popminmax.json ├── rerun.json ├── sub_workflow_tests.json ├── sync_workflows.json └── workflows.json ├── sample_tasks.json └── sample_workflow.json /.github/workflows/build_on_pr.yaml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | 6 | name: Main branch PR Build 7 | on: 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | name: Gradle Build 16 | steps: 17 | - name: Checkout repo 18 | uses: actions/checkout@v3 19 | - name: Set up Zulu JDK 17 20 | uses: actions/setup-java@v3 21 | with: 22 | distribution: 'zulu' 23 | java-version: '17' 24 | - name: Build 25 | run: ./gradlew clean build 26 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [ push, pull_request ] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | name: Gradle Build 9 | steps: 10 | - name: Login to Docker Hub Container Registry 11 | uses: docker/login-action@v1 12 | with: 13 | username: ${{ secrets.DOCKERHUB_USERNAME }} 14 | password: ${{ secrets.DOCKERHUB_TOKEN }} 15 | 16 | - name: Checkout repo 17 | uses: actions/checkout@v3 18 | - name: Set up Zulu JDK 17 19 | uses: actions/setup-java@v3 20 | with: 21 | distribution: 'zulu' 22 | java-version: '17' 23 | - name: Build 24 | run: ./run_tests.sh -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | 6 | name: Build and Publish Docker Builds 7 | on: 8 | release: 9 | types: 10 | - released 11 | - prereleased 12 | push: 13 | branches: [ documentation ] 14 | 15 | jobs: 16 | publish: 17 | runs-on: ubuntu-latest 18 | environment: prod 19 | name: Gradle Build and Publish 20 | steps: 21 | - name: Checkout repo 22 | uses: actions/checkout@v3 23 | - name: Set up Zulu JDK 17 24 | uses: actions/setup-java@v3 25 | with: 26 | distribution: 'zulu' 27 | java-version: '17' 28 | - name: Build 29 | run: | 30 | export VERSION=${{github.ref_name}} 31 | export REL_VER=`echo ${VERSION:1}` 32 | echo "Release version is $REL_VER" 33 | echo "RELEASE_VERSION=$REL_VER" >> $GITHUB_ENV 34 | ./gradlew build -Pversion=$REL_VER 35 | - name: Set up Docker Buildx 36 | uses: docker/setup-buildx-action@v1 37 | 38 | - name: Login to Docker Hub Container Registry 39 | uses: docker/login-action@v1 40 | with: 41 | username: ${{ secrets.DOCKERHUB_USERNAME }} 42 | password: ${{ secrets.DOCKERHUB_TOKEN }} 43 | 44 | - name: Set up QEMU 45 | uses: docker/setup-qemu-action@v2 46 | 47 | - name: Set up Docker Buildx 48 | uses: docker/setup-buildx-action@v2 49 | 50 | - name: Build and push Server 51 | uses: docker/build-push-action@v3 52 | with: 53 | context: . 54 | file: docker/DockerfileServer 55 | push: true 56 | platforms: linux/arm64,linux/amd64 57 | tags: | 58 | orkesio/orkes-conductor-community:latest 59 | orkesio/orkes-conductor-community:${{ env.RELEASE_VERSION }} 60 | 61 | - name: Build and push Standalone 62 | uses: docker/build-push-action@v3 63 | with: 64 | context: . 65 | file: docker/DockerfileStandalone 66 | push: true 67 | platforms: linux/arm64,linux/amd64 68 | tags: | 69 | orkesio/orkes-conductor-community-standalone:latest 70 | orkesio/orkes-conductor-community-standalone:${{ env.RELEASE_VERSION }} 71 | env: 72 | ORG_GRADLE_PROJECT_signingKeyId: ${{ secrets.SIGNING_KEY_ID }} 73 | ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SIGNING_KEY }} 74 | ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_PASSWORD }} 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.war 15 | *.nar 16 | *.ear 17 | *.zip 18 | *.tar.gz 19 | *.rar 20 | 21 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 22 | hs_err_pid* 23 | .idea/ 24 | .gradle/ 25 | build/ 26 | 27 | .DS_Store 28 | tmp/ 29 | out/ 30 | build/ 31 | -------------------------------------------------------------------------------- /CONFIGURATION.md: -------------------------------------------------------------------------------- 1 | # Configuration Properties for Orkes-Conductor 2 | 3 | ## Default configuration 4 | The default configuration is set in the [application.properties](server/src/main/resources/application.properties) that is good enough for most use cases. 5 | 6 | ### Properties that can be overridden for production use case. 7 | Please note, the server comes with sensible defaults that works for most part, 8 | ```properties 9 | 10 | ########################################## 11 | ##### Redis database connectivity ##### 12 | ########################################## 13 | conductor.redis-lock.serverAddress=redis://HOSTNAME:PORT 14 | # us-east-1c is used by backward compatibility. Change this to the existing value used 15 | conductor.redis.hosts=HOSTNAME:PORT:us-east-1c 16 | 17 | ########################################## 18 | ##### Postgres database connectivity ##### 19 | ########################################## 20 | spring.datasource.url=jdbc:postgresql://PG_HOST:PORT/db_name 21 | 22 | #set the user and password accordingly 23 | spring.datasource.username= 24 | spring.datasource.password= 25 | 26 | 27 | ########################################## 28 | ##### Misc conductor server properties 29 | ########################################## 30 | 31 | # Enable/Disable ownerEmail field in the metadata being required 32 | # true | false 33 | conductor.app.owner-email-mandatory= 34 | 35 | #################################################################################### 36 | ##### Advanced performance settings -- The value shown is the default value ######## 37 | #################################################################################### 38 | # No. of threads allocated to the sweeper. Recommended to keep 2x-3x of the total CPU count on the host 39 | conductor.app.sweeperThreadCount=10 40 | 41 | # Batch size for the system task worker 42 | conductor.app.systemTaskMaxPollCount=10 43 | # No of threads configured for the system task worker. Recommended to be same as systemTaskMaxPollCount 44 | conductor.app.systemTaskWorkerThreadCount=10 45 | 46 | # Keep metadata longer in the cache to avoid excessive lookups from Redis. Recommended value 60 47 | # Values are in second 48 | conductor.redis.taskDefCacheRefreshInterval=1 49 | 50 | # Postgres connection pool size -- typically no need to change, unless you are sure 51 | # number. default: 8 52 | spring.datasource.hikari.maximum-pool-size=8 53 | 54 | ``` -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Orkes Conductor 2 | Orkes Conductor is a fully compatible version of Netflix Conductor with **Orkes certified stack**. 3 | #### What is the community edition - OR how is it different from Netflix Conductor? 4 | At Orkes we continue to maintain the Netflix Conductor, and while running Conductor at scale across multiple clouds (including on-prem) we have put together a stack that we leverage for our own cloud offering that is easy to maintain and operate. 5 | 6 | As a way to contribute back to community, we are open sourcing the core stack we run our cloud offering to the community that makes it easy to run Netflix Conductor. This is a "certified" build of Netflix Conductor that is officially supported by Orkes. 7 | 8 | 9 | [![CI](https://github.com/orkes-io/orkes-conductor-community/actions/workflows/ci.yaml/badge.svg)](https://github.com/orkes-io/orkes-conductor-community/actions/workflows/ci.yaml) 10 | [![CI](https://img.shields.io/badge/license-orkes%20community%20license-green)](https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt) 11 | 12 | 13 | ``` 14 | ______ .______ __ ___ _______ _______. 15 | / __ \ | _ \ | |/ / | ____| / | 16 | | | | | | |_) | | ' / | |__ | (----` 17 | | | | | | / | < | __| \ \ 18 | | `--' | | |\ \----.| . \ | |____.----) | 19 | \______/ | _| `._____||__|\__\ |_______|_______/ 20 | 21 | ______ ______ .__ __. _______ __ __ ______ .___________. ______ .______ 22 | / | / __ \ | \ | | | \ | | | | / || | / __ \ | _ \ 23 | | ,----'| | | | | \| | | .--. || | | | | ,----'`---| |----`| | | | | |_) | 24 | | | | | | | | . ` | | | | || | | | | | | | | | | | | / 25 | | `----.| `--' | | |\ | | '--' || `--' | | `----. | | | `--' | | |\ \----. 26 | \______| \______/ |__| \__| |_______/ \______/ \______| |__| \______/ | _| `._____| 27 | ``` 28 | 29 | ## Stack 30 | 1. **Redis** is the primary store for running workflows. 31 | 2. **Postgres** for storing completed workflows and indexing enabling full text search. 32 | 3. **[Orkes-Queues](https://github.com/orkes-io/orkes-queues)** - Redis-based queues that improve upon dyno-queues and providers higher performance and are built from the ground up to support Redis standalone and cluster mode 33 | ### Dependency Versions 34 | 35 | | Dependency | Supported Version | 36 | |-----------------------------------------|-------------------| 37 | | Redis (Standalone, Cluster or Sentinel) | 6.2+ | 38 | | Postgres | 14+ | 39 | 40 | ## Getting Started 41 | ### Docker 42 | Docker is the easiest way to run Conductor. Each release is published as `orkesio/orkes-conductor-community` docker images. 43 | 44 | #### Fully self-contained standalone server with all the dependencies 45 | Container images are useful for local development and testing. 46 | >**Note**: Self-contained docker image shouldn't be used in the production environment. 47 | 48 | #### Simple self-contained script to launch the docker image 49 | ```shell 50 | curl https://raw.githubusercontent.com/orkes-io/orkes-conductor-community/main/scripts/run_local.sh | sh 51 | ``` 52 | #### Using `docker run` manually (Provides more control) 53 | ```shell 54 | 55 | # Create volumes for persistent stores 56 | # Used to create a persistent volume that will preserve the 57 | docker volume create postgres 58 | docker volume create redis 59 | 60 | docker run --init -p 8080:8080 -p 1234:5000 --mount source=redis,target=/redis \ 61 | --mount source=postgres,target=/pgdata orkesio/orkes-conductor-community-standalone:latest 62 | ``` 63 | Navigate to http://localhost:1234 once the container starts to launch UI. 64 | 65 | #### Server + UI Docker 66 | ```shell 67 | docker pull orkesio/orkes-conductor-community:latest 68 | ``` 69 | >**Note**: To use a specific version of Conductor, replace `latest` with the release version. 70 | > e.g. 71 | > 72 | > ```docker pull orkesio/orkes-conductor-community:latest``` 73 | 74 | ### Published Artifacts 75 | 76 | The docker files are published at the following: 77 | #### Server build suitable for the production deployment 78 | https://hub.docker.com/r/orkesio/orkes-conductor-community 79 | 80 | #### Local build useful for local build and development 81 | https://hub.docker.com/r/orkesio/orkes-conductor-community-standalone 82 | 83 | #### Production Configuration Recommendations 84 | The container and server jar published come with sensible defaults that work for most use cases. 85 | 86 | ### Contributions 87 | We welcome community contributions and PRs to this repository. 88 | 89 | ### Get Support 90 | Use GitHub issue tracking for filing issues and Discussion Forum for any other questions, ideas or support requests. 91 | [Orkes](http://orkes.io) development team creates and maintains the Orkes-Conductor releases. 92 | 93 | ## License 94 | Copyright 2023 Orkes, Inc 95 | 96 | Licensed under Orkes Community License. You may obtain a copy of the License at: 97 | ``` 98 | https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 99 | ``` 100 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 101 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | Please open an issue to report vulnerability related to source code in this repository. 5 | -------------------------------------------------------------------------------- /archive/README.md: -------------------------------------------------------------------------------- 1 | # Workflow archival module 2 | This modules uses Postgres as an archival and indexing system 3 | 4 | See [schema folder](src/main/resources/db/migration_archive_postgres/) for the details on the schema 5 | 6 | ## Behavior 7 | When enabled, all the completed workflows (COMPLETED, TERMINATED, FAILED) are moved to the postgres archival and removed from primary datasource 8 | 9 | *Please do not use elasticsearch for indexing when using this - it will be redundant* 10 | 11 | ## Configuration properties 12 | ### Enable archival 13 | ```properties 14 | conductor.archive.db.enabled=true 15 | conductor.archive.db.type=postgres 16 | conductor.archive.db.indexer.threadCount=1 17 | conductor.archive.db.indexer.pollingInterval=1 18 | ``` 19 | 20 | ### Database configuration 21 | Below is the example for connecting to the local postgres 22 | ```properties 23 | spring.datasource.url=jdbc:postgresql://localhost/ 24 | spring.datasource.username=postgres 25 | spring.datasource.password=postgres 26 | spring.datasource.hikari.maximum-pool-size=8 27 | spring.datasource.hikari.auto-commit=false 28 | ``` 29 | 30 | ### Disable elasticsearch indexing 31 | ```properties 32 | conductor.app.asyncIndexingEnabled=false 33 | conductor.indexing.enabled=false 34 | ``` 35 | -------------------------------------------------------------------------------- /archive/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | 3 | implementation "com.netflix.conductor:conductor-common:${versions.revConductor}" 4 | implementation "com.netflix.conductor:conductor-core:${versions.revConductor}" 5 | implementation ("com.netflix.conductor:conductor-postgres-persistence:${versions.revConductor}") 6 | 7 | implementation 'org.springframework.boot:spring-boot-starter' 8 | implementation 'org.springframework.boot:spring-boot-starter-data-jdbc' 9 | implementation 'org.springframework.boot:spring-boot-starter-data-jpa' 10 | implementation 'org.springframework.boot:spring-boot-starter-security' 11 | 12 | implementation "com.fasterxml.jackson.core:jackson-databind" 13 | implementation "com.fasterxml.jackson.core:jackson-core" 14 | 15 | //Metrics 16 | implementation "io.micrometer:micrometer-registry-prometheus:1.7.5" 17 | implementation "io.micrometer:micrometer-core:1.7.5" 18 | implementation "com.netflix.spectator:spectator-reg-micrometer:${versions.revSpectator}" 19 | implementation "com.netflix.spectator:spectator-reg-metrics3:${versions.revSpectator}" 20 | implementation "com.netflix.spectator:spectator-api:${versions.revSpectator}" 21 | 22 | implementation 'org.springframework.retry:spring-retry' 23 | 24 | //Flyway for postgres configuration 25 | implementation "org.flywaydb:flyway-core" 26 | 27 | //Cache 28 | implementation "com.google.guava:guava:${versions.revGuava}" 29 | 30 | //Lucene 31 | implementation "org.apache.lucene:lucene-core:${versions.revLucene}" 32 | implementation "org.apache.lucene:lucene-analyzers-common:${versions.revLucene}" 33 | 34 | //spring 35 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 36 | testImplementation 'org.springframework.security:spring-security-test' 37 | 38 | 39 | testImplementation 'org.hamcrest:hamcrest' 40 | testImplementation "org.awaitility:awaitility:3.1.6" 41 | 42 | //postgres test container 43 | testImplementation "org.testcontainers:postgresql:${versions.revTestContainer}" 44 | 45 | //Fake data generator 46 | testImplementation ('com.github.javafaker:javafaker:1.0.2') { exclude module: 'snakeyaml' } 47 | // testImplementation group: 'org.yaml', name: 'snakeyaml', version: '2.2' 48 | } 49 | 50 | test { 51 | useJUnitPlatform() 52 | } -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/archive/ArchiveDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.archive; 14 | 15 | import java.util.List; 16 | 17 | import com.netflix.conductor.common.metadata.tasks.TaskExecLog; 18 | import com.netflix.conductor.model.WorkflowModel; 19 | 20 | public interface ArchiveDAO { 21 | 22 | // Workflow Methods 23 | 24 | void createOrUpdateWorkflow(WorkflowModel workflow); 25 | 26 | boolean removeWorkflow(String workflowId); 27 | 28 | WorkflowModel getWorkflow(String workflowId, boolean includeTasks); 29 | 30 | List getWorkflowIdsByType(String workflowName, Long startTime, Long endTime); 31 | 32 | List getWorkflowIdsByCorrelationId( 33 | String workflowName, String correlationId, boolean includeClosed, boolean includeTasks); 34 | 35 | ScrollableSearchResult searchWorkflows( 36 | String query, String freeText, int start, int count); 37 | 38 | List getTaskExecutionLogs(String taskId); 39 | 40 | void addTaskExecutionLogs(List logs); 41 | } 42 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/archive/DocumentStoreDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.archive; 14 | 15 | import com.netflix.conductor.model.WorkflowModel; 16 | 17 | /** 18 | * Document store is used to store completed workflow JSON data. 19 | * 20 | * @see io.orkes.conductor.dao.postgres.archive.PostgresArchiveDAO 21 | */ 22 | public interface DocumentStoreDAO { 23 | 24 | void createOrUpdateWorkflow(WorkflowModel workflow); 25 | 26 | boolean removeWorkflow(String workflowId); 27 | 28 | WorkflowModel getWorkflow(String workflowId, boolean includeTasks); 29 | } 30 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/archive/ScrollableSearchResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.archive; 14 | 15 | import java.util.List; 16 | 17 | import com.netflix.conductor.common.run.SearchResult; 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnore; 20 | import lombok.Getter; 21 | import lombok.NoArgsConstructor; 22 | import lombok.Setter; 23 | 24 | @Getter 25 | @Setter 26 | @NoArgsConstructor 27 | public class ScrollableSearchResult extends SearchResult { 28 | 29 | private String queryId; 30 | 31 | public ScrollableSearchResult(List results, String queryId) { 32 | super(0, results); 33 | this.queryId = queryId; 34 | } 35 | 36 | // With ScrollableSearchResult this will always be zero and it's confusing from an API client's 37 | // perspective. 38 | // That's why it's ignored. 39 | @Override 40 | @JsonIgnore 41 | public long getTotalHits() { 42 | return super.getTotalHits(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/indexer/IndexValueExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.indexer; 14 | 15 | import java.util.*; 16 | import java.util.stream.Collectors; 17 | 18 | import org.apache.lucene.analysis.en.EnglishAnalyzer; 19 | 20 | import com.netflix.conductor.model.TaskModel; 21 | import com.netflix.conductor.model.WorkflowModel; 22 | 23 | import lombok.extern.slf4j.Slf4j; 24 | 25 | @Slf4j 26 | public class IndexValueExtractor { 27 | 28 | private static final String splitWords = "\"|,|;|\\s|,"; 29 | private static final String replaceWords = "\"|,|;|\\s|,|:"; 30 | private static final Collection stopWords = EnglishAnalyzer.getDefaultStopSet(); 31 | 32 | public static Collection getIndexWords( 33 | WorkflowModel workflow, int maxWords, int maxWordLength) { 34 | 35 | try { 36 | 37 | List words = getIndexWords(workflow); 38 | return words.stream() 39 | .flatMap(value -> Arrays.asList(value.split(splitWords)).stream()) 40 | .filter(word -> word.length() < maxWordLength) 41 | .filter(word -> word.length() > 2) 42 | .filter(word -> !word.trim().isBlank()) 43 | .map(word -> word.toLowerCase().trim().replaceAll(replaceWords + "+$", "")) 44 | .filter(word -> !stopWords.contains(word)) 45 | .limit(maxWords) 46 | .collect(Collectors.toSet()); 47 | 48 | } catch (Exception e) { 49 | log.warn("Error serializing input/output map to text: " + e.getMessage(), e); 50 | return new ArrayList<>(); 51 | } 52 | } 53 | 54 | private static List getIndexWords(WorkflowModel workflow) { 55 | List words = new ArrayList<>(); 56 | append(words, workflow.getCorrelationId()); 57 | append(words, workflow.getInput()); 58 | append(words, workflow.getOutput()); 59 | append(words, workflow.getReasonForIncompletion()); 60 | append(words, workflow.getVariables()); 61 | 62 | for (TaskModel task : workflow.getTasks()) { 63 | append(words, task.getOutputData()); 64 | } 65 | return words; 66 | } 67 | 68 | private static void append(List words, Object value) { 69 | if (value instanceof String || value instanceof Number) { 70 | if (value != null) words.add(value.toString()); 71 | } else if (value instanceof List) { 72 | List values = (List) value; 73 | for (Object valueObj : values) { 74 | if (valueObj != null) words.add(valueObj.toString()); 75 | } 76 | } else if (value instanceof Map) { 77 | append(words, (Map) value); 78 | } 79 | } 80 | 81 | private static void append(List words, Map map) { 82 | 83 | map.values() 84 | .forEach( 85 | value -> { 86 | if (value instanceof String || value instanceof Number) { 87 | if (value != null) words.add(value.toString()); 88 | } else if (value instanceof Map) { 89 | append(words, (Map) value); 90 | } else if (value instanceof List) { 91 | List values = (List) value; 92 | for (Object valueObj : values) { 93 | if (valueObj != null) words.add(valueObj.toString()); 94 | } 95 | } 96 | }); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/indexer/IndexWorkerProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.indexer; 14 | 15 | import org.springframework.boot.context.properties.ConfigurationProperties; 16 | import org.springframework.context.annotation.Configuration; 17 | 18 | @Configuration 19 | @ConfigurationProperties("conductor.archive.db.indexer") 20 | public class IndexWorkerProperties { 21 | 22 | private int threadCount = 3; 23 | 24 | private int pollingInterval = 10; // in millisecond 25 | 26 | private int pollBatchSize = 10; 27 | 28 | public int getThreadCount() { 29 | return threadCount; 30 | } 31 | 32 | public void setThreadCount(int threadCount) { 33 | this.threadCount = threadCount; 34 | } 35 | 36 | public int getPollingInterval() { 37 | return pollingInterval; 38 | } 39 | 40 | public void setPollingInterval(int pollingInterval) { 41 | this.pollingInterval = pollingInterval; 42 | } 43 | 44 | public int getPollBatchSize() { 45 | return pollBatchSize; 46 | } 47 | 48 | public void setPollBatchSize(int pollBatchSize) { 49 | this.pollBatchSize = pollBatchSize; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/indexer/TaskIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.indexer; 14 | 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | import com.netflix.conductor.model.TaskModel; 19 | 20 | public class TaskIndex { 21 | 22 | private TaskModel task; 23 | 24 | private int maxValueLen; 25 | 26 | public TaskIndex(TaskModel task, int maxValueLen) { 27 | this.task = task; 28 | this.maxValueLen = maxValueLen; 29 | } 30 | 31 | public String toIndexString() { 32 | StringBuilder sb = new StringBuilder(); 33 | append(sb, task.getTaskType()); 34 | append(sb, task.getTaskId()); 35 | append(sb, task.getDomain()); 36 | append(sb, task.getReferenceTaskName()); 37 | append(sb, task.getStatus().toString()); 38 | append(sb, task.getExternalInputPayloadStoragePath()); 39 | append(sb, task.getExternalOutputPayloadStoragePath()); 40 | append(sb, task.getInputData()); 41 | append(sb, task.getOutputData()); 42 | append(sb, task.getReasonForIncompletion()); 43 | append(sb, task.getWorkerId()); 44 | 45 | return sb.toString(); 46 | } 47 | 48 | @Override 49 | public String toString() { 50 | return toIndexString(); 51 | } 52 | 53 | private String toString(Map map) { 54 | StringBuilder sb = new StringBuilder(); 55 | append(sb, map); 56 | return sb.toString(); 57 | } 58 | 59 | private void append(StringBuilder sb, Object value) { 60 | if (value instanceof String || value instanceof Number) { 61 | sb.append(" "); 62 | sb.append(value.toString()); 63 | } else if (value instanceof Map) { 64 | sb.append(" "); 65 | append(sb, (Map) value); 66 | } else if (value instanceof List) { 67 | List values = (List) value; 68 | for (Object valueObj : values) { 69 | sb.append(" "); 70 | append(sb, valueObj); 71 | } 72 | } 73 | } 74 | 75 | private void append(StringBuilder sb, Map map) { 76 | 77 | map.entrySet() 78 | .forEach( 79 | entry -> { 80 | String key = entry.getKey(); 81 | Object value = entry.getValue(); 82 | sb.append(" "); 83 | sb.append(key); 84 | if (value instanceof String) { 85 | sb.append(" "); 86 | String valueString = value.toString(); 87 | sb.append( 88 | valueString.substring( 89 | 0, Math.min(valueString.length(), maxValueLen))); 90 | } else if (value instanceof Number) { 91 | sb.append(" "); 92 | sb.append(value.toString()); 93 | } else if (value instanceof Map) { 94 | sb.append(" "); 95 | append(sb, (Map) value); 96 | } else if (value instanceof List) { 97 | List values = (List) value; 98 | for (Object valueObj : values) { 99 | sb.append(" "); 100 | append(sb, valueObj); 101 | } 102 | } 103 | }); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/indexer/WorkflowIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.indexer; 14 | 15 | import java.util.Collection; 16 | 17 | import com.netflix.conductor.model.WorkflowModel; 18 | 19 | public class WorkflowIndex { 20 | 21 | private WorkflowModel workflow; 22 | private int maxWords; 23 | private int maxWordLength; 24 | 25 | public WorkflowIndex(WorkflowModel workflow, int maxWords, int maxWordLength) { 26 | this.workflow = workflow; 27 | this.maxWords = maxWords; 28 | this.maxWordLength = maxWordLength; 29 | } 30 | 31 | public Collection toIndexWords() { 32 | return IndexValueExtractor.getIndexWords(workflow, maxWords, maxWordLength); 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return toIndexWords().toString(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/dao/postgres/archive/SearchQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.postgres.archive; 14 | 15 | import java.util.ArrayList; 16 | import java.util.Arrays; 17 | import java.util.List; 18 | import java.util.Objects; 19 | 20 | public class SearchQuery { 21 | 22 | private List workflowNames; 23 | 24 | private List taskTypes; 25 | 26 | private List workflowIds; 27 | 28 | private List correlationIds; 29 | 30 | private List taskIds; 31 | 32 | private List statuses; 33 | 34 | private long fromStartTime; 35 | 36 | private long toStartTime; 37 | 38 | private SearchQuery() {} 39 | 40 | public static SearchQuery parse(String query) { 41 | SearchQuery searchQuery = new SearchQuery(); 42 | 43 | String[] values = query.split(" AND "); 44 | for (String value : values) { 45 | value = value.trim(); 46 | if (value.startsWith("workflowId")) { 47 | searchQuery.workflowIds = getValues(value); 48 | } 49 | if (value.startsWith("correlationId")) { 50 | searchQuery.correlationIds = getValues(value); 51 | } else if (value.startsWith("taskId")) { 52 | searchQuery.taskIds = getValues(value); 53 | } else if (value.startsWith("workflowType")) { 54 | searchQuery.workflowNames = getValues(value); 55 | } else if (value.startsWith("taskType")) { 56 | searchQuery.taskTypes = getValues(value); 57 | } else if (value.startsWith("status")) { 58 | searchQuery.statuses = getValues(value); 59 | } else if (value.startsWith("startTime")) { 60 | 61 | if (value.contains(">")) { 62 | 63 | String[] kv = value.split(">"); 64 | if (kv.length > 0) { 65 | try { 66 | searchQuery.fromStartTime = Long.parseLong(kv[1].trim()); 67 | } catch (Exception e) { 68 | } 69 | } 70 | 71 | } else if (value.contains("<")) { 72 | 73 | String[] kv = value.split("<"); 74 | if (kv.length > 0) { 75 | try { 76 | searchQuery.toStartTime = Long.parseLong(kv[1].trim()); 77 | } catch (Exception e) { 78 | } 79 | } 80 | } 81 | } 82 | } 83 | return searchQuery; 84 | } 85 | 86 | private static List getValues(String keyValue) { 87 | List values = new ArrayList<>(); 88 | if (keyValue.contains("=")) { 89 | String[] kv = keyValue.split("="); 90 | if (kv.length > 0) { 91 | String value = kv[1].trim(); 92 | // remove quotes from the start and end 93 | value = value.substring(1, value.length() - 1); 94 | return Arrays.asList(value.trim()); 95 | } 96 | } else if (keyValue.contains(" IN ")) { 97 | 98 | String[] kv = keyValue.split(" IN "); 99 | if (kv.length > 0) { 100 | String[] inValues = kv[1].trim().substring(1, kv[1].length() - 1).split(","); 101 | for (String inValue : inValues) { 102 | values.add(inValue.trim()); 103 | } 104 | } 105 | } 106 | return values; 107 | } 108 | 109 | public List getWorkflowNames() { 110 | return workflowNames; 111 | } 112 | 113 | public List getWorkflowIds() { 114 | return workflowIds; 115 | } 116 | 117 | public List getCorrelationIds() { 118 | return correlationIds; 119 | } 120 | 121 | public List getStatuses() { 122 | return statuses; 123 | } 124 | 125 | public long getFromStartTime() { 126 | return fromStartTime; 127 | } 128 | 129 | public long getToStartTime() { 130 | if (toStartTime == 0) { 131 | toStartTime = System.currentTimeMillis(); 132 | } 133 | return toStartTime; 134 | } 135 | 136 | public List getTaskTypes() { 137 | return taskTypes; 138 | } 139 | 140 | public List getTaskIds() { 141 | return taskIds; 142 | } 143 | 144 | @Override 145 | public boolean equals(Object o) { 146 | if (this == o) return true; 147 | if (o == null || getClass() != o.getClass()) return false; 148 | SearchQuery that = (SearchQuery) o; 149 | return fromStartTime == that.fromStartTime 150 | && toStartTime == that.toStartTime 151 | && Objects.equals(workflowNames, that.workflowNames) 152 | && Objects.equals(workflowIds, that.workflowIds) 153 | && Objects.equals(statuses, that.statuses); 154 | } 155 | 156 | @Override 157 | public int hashCode() { 158 | return Objects.hash(workflowNames, workflowIds, statuses, fromStartTime, toStartTime); 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /archive/src/main/java/io/orkes/conductor/id/TimeBasedUUIDGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.id; 14 | 15 | import java.time.LocalDate; 16 | import java.util.Calendar; 17 | import java.util.TimeZone; 18 | import java.util.UUID; 19 | 20 | import org.apache.logging.log4j.core.util.UuidUtil; 21 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 22 | import org.springframework.stereotype.Component; 23 | 24 | import com.netflix.conductor.core.utils.IDGenerator; 25 | 26 | import lombok.extern.slf4j.Slf4j; 27 | 28 | @Component 29 | @ConditionalOnProperty(name = "conductor.id.generator", havingValue = "time_based") 30 | @Slf4j 31 | public class TimeBasedUUIDGenerator extends IDGenerator { 32 | 33 | private static final LocalDate JAN_1_2020 = LocalDate.of(2020, 1, 1); 34 | 35 | private static final int uuidLength = UUID.randomUUID().toString().length(); 36 | 37 | private static Calendar uuidEpoch = Calendar.getInstance(TimeZone.getTimeZone("UTC")); 38 | 39 | private static final long epochMillis; 40 | 41 | static { 42 | uuidEpoch.clear(); 43 | uuidEpoch.set(1582, 9, 15, 0, 0, 0); // 44 | epochMillis = uuidEpoch.getTime().getTime(); 45 | } 46 | 47 | public TimeBasedUUIDGenerator() { 48 | log.info("Using TimeBasedUUIDGenerator to generate Ids"); 49 | } 50 | 51 | public String generate() { 52 | UUID uuid = UuidUtil.getTimeBasedUuid(); 53 | return uuid.toString(); 54 | } 55 | 56 | public static long getDate(String id) { 57 | UUID uuid = UUID.fromString(id); 58 | if (uuid.version() != 1) { 59 | return 0; 60 | } 61 | long time = (uuid.timestamp() / 10000L) + epochMillis; 62 | return time; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /archive/src/main/resources/db/migration_archive_postgres/V99__initial_schema.sql: -------------------------------------------------------------------------------- 1 | -- Workflow 2 | CREATE TABLE workflow_archive ( 3 | workflow_id varchar(255) NOT NULL, 4 | created_on bigint, 5 | modified_on bigint, 6 | created_by varchar(255), 7 | correlation_id varchar(255), 8 | workflow_name varchar(255), 9 | status varchar(255), 10 | json_data TEXT, 11 | index_data text[], 12 | PRIMARY KEY (workflow_id) 13 | ) with (autovacuum_vacuum_scale_factor = 0.0, autovacuum_vacuum_threshold = 10000); 14 | 15 | CREATE INDEX workflow_archive_workflow_name_index ON workflow_archive (workflow_name, status, correlation_id, created_on); 16 | CREATE INDEX workflow_archive_search_index ON workflow_archive USING gin(index_data); 17 | CREATE INDEX workflow_archive_created_on_index ON workflow_archive (created_on desc); 18 | 19 | -- Task Logs 20 | CREATE TABLE task_logs ( 21 | task_id varchar(255) NOT NULL, 22 | seq bigserial, 23 | log TEXT, 24 | created_on bigint, 25 | PRIMARY KEY (task_id, seq) 26 | ); -------------------------------------------------------------------------------- /archive/src/test/java/io/orkes/conductor/dao/indexer/TestIndexValueExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.indexer; 14 | 15 | import java.util.Collection; 16 | import java.util.Map; 17 | import java.util.UUID; 18 | 19 | import org.junit.jupiter.api.Test; 20 | 21 | import com.netflix.conductor.common.config.ObjectMapperProvider; 22 | import com.netflix.conductor.model.TaskModel; 23 | import com.netflix.conductor.model.WorkflowModel; 24 | 25 | import static io.orkes.conductor.dao.indexer.IndexValueExtractor.getIndexWords; 26 | import static org.junit.jupiter.api.Assertions.*; 27 | 28 | public class TestIndexValueExtractor { 29 | 30 | @Test 31 | public void testIndexer() { 32 | WorkflowModel model = new WorkflowModel(); 33 | model.getTasks().add(new TaskModel()); 34 | 35 | String uuid = UUID.randomUUID().toString(); 36 | model.getInput().put("correlation_id", uuid); 37 | System.out.println("correlationId: " + uuid); 38 | model.getInput().put("keep", "abcd"); 39 | 40 | String json = 41 | "{\n" 42 | + " \"response\": \"java.lang.Exception: I/O error on GET request for \\\"https://orkes-services.web.app2/data.json\\\": orkes-services.web.app2: nodename nor servname provided, or not known; nested exception is java.net.UnknownHostException: orkes-services.web.app2: nodename nor servname provided, or not known\"\n" 43 | + "}"; 44 | try { 45 | Map output = new ObjectMapperProvider().getObjectMapper().readValue(json, Map.class); 46 | model.getOutput().putAll(output); 47 | } catch (Exception e) { 48 | 49 | } 50 | 51 | for (int i = 0; i < 100; i++) { 52 | model.getTasks().get(0).getOutputData().put("id" + i, UUID.randomUUID().toString()); 53 | } 54 | Collection words = getIndexWords(model, 2, 50); 55 | 56 | // Sine all the UUIDs are longer than max word length, they should all get filtered out 57 | assertNotNull(words); 58 | assertEquals(2, words.size()); 59 | assertTrue(words.contains("abcd")); 60 | assertTrue(words.contains(uuid), uuid + " not in the list of words : " + words); 61 | 62 | words = getIndexWords(model, 200, 50); 63 | System.out.println(words); 64 | System.out.println(words.size()); 65 | words.stream().forEach(System.out::println); 66 | 67 | // All UUIDs shouldbe present 68 | assertNotNull(words); 69 | assertTrue(words.contains("https://orkes-services.web.app2/data.json")); 70 | assertTrue(words.contains(uuid)); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresDAOTestUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.dao.postgres; 14 | 15 | import java.nio.file.Paths; 16 | import java.time.Duration; 17 | 18 | import javax.sql.DataSource; 19 | 20 | import org.flywaydb.core.Flyway; 21 | import org.flywaydb.core.api.configuration.FluentConfiguration; 22 | import org.testcontainers.containers.PostgreSQLContainer; 23 | 24 | import com.netflix.conductor.postgres.config.PostgresProperties; 25 | 26 | import com.fasterxml.jackson.databind.ObjectMapper; 27 | import com.zaxxer.hikari.HikariDataSource; 28 | 29 | import static org.mockito.Mockito.mock; 30 | import static org.mockito.Mockito.when; 31 | 32 | public class PostgresDAOTestUtil { 33 | 34 | private final HikariDataSource dataSource; 35 | private final PostgresProperties properties = mock(PostgresProperties.class); 36 | private final ObjectMapper objectMapper; 37 | 38 | public PostgresDAOTestUtil( 39 | PostgreSQLContainer postgreSQLContainer, ObjectMapper objectMapper) { 40 | 41 | this.objectMapper = objectMapper; 42 | 43 | this.dataSource = new HikariDataSource(); 44 | dataSource.setJdbcUrl(postgreSQLContainer.getJdbcUrl()); 45 | dataSource.setUsername(postgreSQLContainer.getUsername()); 46 | dataSource.setPassword(postgreSQLContainer.getPassword()); 47 | dataSource.setAutoCommit(false); 48 | // Prevent DB from getting exhausted during rapid testing 49 | dataSource.setMaximumPoolSize(8); 50 | 51 | when(properties.getTaskDefCacheRefreshInterval()).thenReturn(Duration.ofSeconds(60)); 52 | 53 | flywayMigrate(dataSource); 54 | } 55 | 56 | private void flywayMigrate(DataSource dataSource) { 57 | FluentConfiguration fluentConfiguration = 58 | Flyway.configure() 59 | .table("schema_version") 60 | .locations(Paths.get("db", "migration_archive_postgres").toString()) 61 | .dataSource(dataSource) 62 | .mixed(true) 63 | .placeholderReplacement(false); 64 | 65 | Flyway flyway = fluentConfiguration.load(); 66 | flyway.migrate(); 67 | } 68 | 69 | public HikariDataSource getDataSource() { 70 | return dataSource; 71 | } 72 | 73 | public PostgresProperties getTestProperties() { 74 | return properties; 75 | } 76 | 77 | public ObjectMapper getObjectMapper() { 78 | return objectMapper; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /archive/src/test/resources/application-testazurearchive.properties: -------------------------------------------------------------------------------- 1 | conductor.db.type=redis_standalone 2 | conductor.redis.hosts=localhost:6379:us-east-1c 3 | 4 | conductor.archive.db.enabled=true 5 | conductor.archive.db.type=postgres 6 | conductor.archive.db.indexer.threadCount=1 7 | conductor.archive.db.indexer.pollingInterval=10 8 | spring.datasource.url=jdbc:tc:postgresql:11.15-alpine:///conductordb 9 | 10 | conductor.archive.db.document.store.type=azureblob 11 | conductor.archive.db.document.store.type.azureblob.connectionString=DefaultEndpointsProtocol=https;AccountName=aiademo;AccountKey=nMNQta5j9ILcRoALQhlFELQLJ0ILbXVu5NeEfkXWQT15WcupsNuQ1wrMZP1O5K5gWyyd3tJpQXpVln/zctt19A==;EndpointSuffix=core.windows.net 12 | 13 | conductor.grpc-server.port=8091 14 | es.set.netty.runtime.available.processors=false 15 | 16 | -------------------------------------------------------------------------------- /archive/src/test/resources/application-tests3archive.properties: -------------------------------------------------------------------------------- 1 | conductor.db.type=memory 2 | conductor.redis.hosts=host1:port:rack;host2:port:rack:host3:port:rack 3 | 4 | conductor.archive.db.enabled=true 5 | conductor.archive.db.type=postgres 6 | conductor.archive.db.indexer.threadCount=1 7 | conductor.archive.db.indexer.pollingInterval=10 8 | spring.datasource.url=jdbc:tc:postgresql:11.15-alpine:///conductordb 9 | 10 | conductor.archive.db.document.store.type=s3 11 | 12 | conductor.grpc-server.port=8091 13 | es.set.netty.runtime.available.processors=false 14 | 15 | -------------------------------------------------------------------------------- /archive/src/test/resources/drop_all.sql: -------------------------------------------------------------------------------- 1 | DROP SCHEMA public CASCADE; -------------------------------------------------------------------------------- /archive/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | dependencies { 3 | classpath "org.springframework.boot:spring-boot-gradle-plugin:2.5.15" 4 | classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.+' 5 | } 6 | } 7 | 8 | plugins { 9 | id 'io.spring.dependency-management' version '1.0.11.RELEASE' 10 | } 11 | 12 | ext { 13 | group = 'io.orkes.conductor' 14 | appVersion = '0.0.1-SNAPSHOT' 15 | springBootVersion = '2.7.6' 16 | 17 | versions = [ 18 | revConductor : '3.15.0', 19 | revTestContainer : '1.17.2', 20 | revGuava : '32.0.0-jre', 21 | revLog4j : '2.17.1', 22 | revJedis : '3.8.0', 23 | revMockServerClient : '5.12.0', 24 | revCommonsLang : '3.12.0', 25 | revLombok : '1.18.24', 26 | revLucene : '7.7.3', 27 | revSpectator : '0.122.0', 28 | revJsonPath : '2.8.0', 29 | revOpenapi : '1.7.+', 30 | revAwsSdk : '1.12.549', 31 | revProtoBuf : '3.16.3', 32 | revRarefiedRedis : '0.0.17', 33 | revOrkesProtos : '0.9.2', 34 | revOrkesQueues : '1.0.6', 35 | ioGRPC : '1.53.0' 36 | ] 37 | } 38 | 39 | def relVersion = System.getenv('REL_VER') 40 | if (relVersion) { 41 | println "Inferred version from env variable 'REL_VER': $relVersion" 42 | appVersion = relVersion 43 | } 44 | 45 | subprojects { 46 | 47 | group = 'io.orkes.conductor' 48 | version = "${appVersion}" 49 | 50 | apply plugin: 'java' 51 | apply plugin: 'io.spring.dependency-management' 52 | apply plugin: 'maven-publish' 53 | apply plugin: 'signing' 54 | apply plugin: 'com.diffplug.spotless' 55 | 56 | repositories { 57 | mavenCentral() 58 | mavenLocal() 59 | } 60 | 61 | java { 62 | withSourcesJar() 63 | } 64 | 65 | configurations { 66 | compileOnly { 67 | extendsFrom annotationProcessor 68 | } 69 | testCompileOnly { 70 | extendsFrom annotationProcessor 71 | } 72 | all { 73 | exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl' 74 | exclude group: 'org.slf4j', module: 'slf4j-log4j12' 75 | } 76 | } 77 | 78 | dependencies { 79 | 80 | implementation('net.minidev:json-smart') { 81 | version { 82 | strictly '2.4.10' 83 | } 84 | } 85 | 86 | implementation 'com.amazonaws:aws-java-sdk-s3:1.12.548' 87 | implementation "redis.clients:jedis:${versions.revJedis}" 88 | 89 | implementation "org.apache.logging.log4j:log4j-core:${versions.revLog4j}!!" 90 | implementation "org.apache.logging.log4j:log4j-api:${versions.revLog4j}!!" 91 | implementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.revLog4j}!!" 92 | implementation "org.apache.logging.log4j:log4j-jul:${versions.revLog4j}!!" 93 | implementation "org.apache.logging.log4j:log4j-web:${versions.revLog4j}!!" 94 | implementation "org.apache.logging.log4j:log4j-to-slf4j:${versions.revLog4j}!!" 95 | 96 | compileOnly "org.projectlombok:lombok:${versions.revLombok}" 97 | annotationProcessor "org.projectlombok:lombok:${versions.revLombok}" 98 | testAnnotationProcessor "org.projectlombok:lombok:${versions.revLombok}" 99 | implementation "org.apache.commons:commons-lang3:${versions.revCommonsLang}" 100 | } 101 | 102 | allprojects { 103 | configurations.all { 104 | resolutionStrategy.eachDependency { DependencyResolveDetails details -> 105 | if (details.requested.group == 'com.fasterxml.jackson.core') { 106 | details.useVersion '2.15.2' 107 | } 108 | if (details.requested.group == 'com.fasterxml.jackson.dataformat') { 109 | details.useVersion '2.15.2' 110 | } 111 | if (details.requested.group == 'org.yaml') { 112 | details.useVersion '2.2' 113 | } 114 | if (details.requested.group == 'io.netty' && details.requested.version == '4.1.92.Final') { 115 | details.useVersion "4.1.94.Final" 116 | } 117 | } 118 | } 119 | } 120 | 121 | dependencyManagement { 122 | imports { 123 | mavenBom(org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES) 124 | } 125 | } 126 | 127 | test { 128 | useJUnitPlatform() 129 | testLogging { 130 | events = ["SKIPPED", "FAILED"] 131 | exceptionFormat = "full" 132 | showStandardStreams = false 133 | } 134 | } 135 | 136 | 137 | compileJava { 138 | sourceCompatibility = 17 139 | targetCompatibility = 17 140 | } 141 | 142 | spotless { 143 | java { 144 | googleJavaFormat().aosp() 145 | removeUnusedImports() 146 | importOrder('java', 'javax', 'org', 'com.netflix', 'io.orkes','', '\\#com.netflix', '\\#') 147 | licenseHeaderFile("$rootDir/licenseheader.txt") 148 | } 149 | } 150 | build.dependsOn(spotlessApply) 151 | } 152 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributing 3 | Thanks for your interest in contributing to Orkes. This guide helps to find the most efficient way to contribute, ask questions, and report issues. 4 | 5 | ## I want to contribute 6 | 7 | We welcome Pull Requests and already had many outstanding community contributions! Creating and reviewing Pull Requests take considerable time. This section helps you to set up a smooth Pull Request experience. 8 | 9 | We're currently accepting contributions for our [Community repository](https://github.com/orkes-io/orkes-conductor-community) 10 | Please create pull requests for your contributions against Orkes Conductor community repo only. 11 | 12 | Also, please consider that not every feature is a good fit for Orkes. A few things to consider for your contributions are: 13 | 14 | * Is it increasing complexity for the user, or might it be confusing? 15 | * Does it, in any way, break backward compatibility (this is seldom acceptable) 16 | * Does it require new dependencies (this is rarely acceptable for core modules) 17 | * Should the feature be opt-in or enabled by default. For integration with a new Queuing recipe or persistence module, a separate module which can be optionally enabled is the right choice. 18 | * Should the feature be implemented in the main Orkes-Conductor repository, or would it be better to set up a separate repository? Especially for integration with other systems, a separate repository is often the right choice because the life-cycle of it will be different. 19 | Of course, for more minor bug fixes and improvements, the process can be more light-weight. 20 | 21 | We'll try to be responsive to Pull Requests. Do keep in mind that because of the inherently distributed nature of open source projects, responses to a PR might take some time because of time zones, weekends, and other things we may be working on. 22 | 23 | Of course, for more minor bug fixes and improvements, the process can be more light-weight. 24 | 25 | We'll try to be responsive to Pull Requests. Do keep in mind that because of the inherently distributed nature of open source projects, responses to a PR might take some time because of time zones, weekends, and other things we may be working on. 26 | 27 | 28 | ##I want to report an issue 29 | ----- 30 | 31 | If you found a bug, it is much appreciated if you create an issue. Please include clear instructions on how to reproduce the issue, or even better, include a test case on a branch. Make sure to come up with a descriptive title for the issue because this helps while organizing issues. 32 | Please choose the right label against it. 33 | 34 | ## I have a great idea for a new feature 35 | Many features in Orkes have come from ideas from the community. If you think something is missing or certain use cases could be supported better, let us know! You can do so by opening a discussion on the discussion forum. Provide as much relevant context to why and when the feature would be helpful. Providing context is especially important for "Support XYZ" issues since we might not be familiar with what "XYZ" is and why it's useful. If you have an idea of how to implement the feature, include that as well. 36 | Once we have decided on a direction, it's time to summarize the idea by creating a new issue. 37 | 38 | Please note that any features will be added or worked on Orkes discretion. 39 | 40 | -------------------------------------------------------------------------------- /docker/DockerfileServer: -------------------------------------------------------------------------------- 1 | # 2 | # conductor:server - Combined Netflix conductor server & UI 3 | # 4 | # =========================================================================================================== 5 | # 0. Builder stage 6 | # =========================================================================================================== 7 | FROM alpine:3.18 AS builder 8 | 9 | MAINTAINER Orkes Inc 10 | 11 | # =========================================================================================================== 12 | # 0. Build Conductor Server UI 13 | # =========================================================================================================== 14 | 15 | 16 | # Install dependencies 17 | RUN apk add openjdk17 18 | RUN apk add git 19 | RUN apk add --update nodejs npm yarn 20 | 21 | COPY . /conductor-community 22 | WORKDIR conductor-community 23 | RUN ./gradlew clean build -x test 24 | 25 | WORKDIR / 26 | RUN git clone https://github.com/Netflix/conductor 27 | WORKDIR conductor/ui 28 | RUN yarn config set network-timeout 600000 -g 29 | RUN yarn install && yarn build 30 | RUN ls -ltr 31 | RUN echo "Done building UI" 32 | 33 | FROM alpine:3.18.3 34 | 35 | MAINTAINER Orkes Inc 36 | 37 | RUN apk add nginx 38 | 39 | RUN apk add openjdk17 40 | 41 | RUN apk add coreutils 42 | RUN apk add curl 43 | 44 | # Make app folders 45 | RUN mkdir -p /app/config /app/logs /app/libs /app/info 46 | 47 | # Add UI 48 | # Make sure to run build-ui.sh script before running the docker build to pull and build the UI 49 | WORKDIR /usr/share/nginx/html 50 | RUN rm -rf ./* 51 | COPY --from=builder /conductor/ui/build . 52 | COPY docker/config/nginx.conf /etc/nginx/http.d/default.conf 53 | 54 | # Startup script(s) 55 | COPY docker/config/startup.sh /app/startup.sh 56 | COPY docker/config/config.properties /app/config/config.properties 57 | COPY server/src/main/resources/banner.txt /app/config/banner.txt 58 | 59 | # JAR files 60 | COPY --from=builder conductor-community/server/build/libs/orkes-conductor-server-boot.jar /app/libs/server.jar 61 | 62 | # Server version 63 | #COPY assembled/libs/server-version.txt* /app/info 64 | 65 | RUN chmod +x /app/startup.sh 66 | 67 | HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1 68 | 69 | EXPOSE 5000 8080 70 | 71 | CMD ["/app/startup.sh"] 72 | ENTRYPOINT ["/bin/sh"] 73 | -------------------------------------------------------------------------------- /docker/DockerfileStandalone: -------------------------------------------------------------------------------- 1 | # 2 | # conductor:server - Combined Netflix conductor server & UI 3 | # 4 | # =========================================================================================================== 5 | # 0. Builder stage 6 | # =========================================================================================================== 7 | FROM alpine:3.18 AS builder 8 | 9 | MAINTAINER Orkes Inc 10 | 11 | # =========================================================================================================== 12 | # 0. Build Conductor Server UI 13 | # =========================================================================================================== 14 | 15 | 16 | # Install dependencies 17 | RUN apk add openjdk17 18 | RUN apk add git 19 | RUN apk add --update nodejs npm yarn 20 | 21 | COPY . /conductor-community 22 | WORKDIR conductor-community 23 | RUN ./gradlew clean build -x test 24 | 25 | WORKDIR / 26 | RUN git clone https://github.com/Netflix/conductor 27 | WORKDIR conductor/ui 28 | RUN yarn config set network-timeout 600000 -g 29 | RUN yarn install && yarn build 30 | RUN ls -ltr 31 | RUN echo "Done building UI" 32 | 33 | FROM alpine:3.18.3 34 | MAINTAINER Orkes Inc 35 | 36 | # Install software required to run conductor stack 37 | RUN apk add nginx 38 | RUN apk add coreutils 39 | RUN apk add openjdk17 40 | RUN apk add redis 41 | RUN apk add postgresql14 42 | 43 | # Make app folders 44 | RUN mkdir -p /app/config /app/logs /app/libs /app/info 45 | 46 | # Add UI 47 | # Make sure to run build-ui.sh script before running the docker build to pull and build the UI 48 | WORKDIR /usr/share/nginx/html 49 | RUN rm -rf ./* 50 | COPY --from=builder /conductor/ui/build . 51 | COPY docker/config/nginx.conf /etc/nginx/http.d/default.conf 52 | 53 | # Startup script(s) 54 | COPY docker/config/startup.sh /app/startup.sh 55 | COPY docker/config/config.properties /app/config/config.properties 56 | COPY docker/config/redis.conf /app/config/redis.conf 57 | COPY docker/config/start_all.sh /app/start_all.sh 58 | COPY server/src/main/resources/banner.txt /app/config/banner.txt 59 | 60 | # JAR files 61 | COPY --from=builder conductor-community/server/build/libs/orkes-conductor-server-boot.jar /app/libs/server.jar 62 | 63 | RUN chmod +x /app/startup.sh 64 | RUN touch /app/logs/server.log 65 | 66 | # setup postgres 67 | RUN mkdir /run/postgresql 68 | RUN chown postgres:postgres /run/postgresql/ 69 | 70 | HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1 71 | EXPOSE 5000 8080 72 | 73 | USER root 74 | CMD ["/app/start_all.sh"] 75 | ENTRYPOINT ["/bin/sh"] 76 | -------------------------------------------------------------------------------- /docker/build-ui.sh: -------------------------------------------------------------------------------- 1 | # clone and build conductor UI 2 | mkdir -p tmp/ui 3 | cd tmp/ui 4 | pwd 5 | git clone https://github.com/Netflix/conductor 6 | cd conductor/ui 7 | yarn config set network-timeout 600000 -g 8 | yarn install 9 | yarn build -------------------------------------------------------------------------------- /docker/config/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orkes-io/orkes-conductor-community/004d605db5d851ad99ffe84bbad961a97517b061/docker/config/config.properties -------------------------------------------------------------------------------- /docker/config/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 5000; 3 | server_name conductor; 4 | server_tokens off; 5 | 6 | gzip on; 7 | gzip_vary on; 8 | gzip_comp_level 6; 9 | gzip_types text/plain text/css application/json application/x-javascript application/javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/vnd.ms-fontobject application/x-font-ttf font/opentype; 10 | 11 | location / { 12 | add_header Referrer-Policy "strict-origin"; 13 | add_header X-Frame-Options "SAMEORIGIN"; 14 | add_header X-Content-Type-Options "nosniff"; 15 | add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; 16 | add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; 17 | # This would be the directory where your React app's static files are stored at 18 | root /usr/share/nginx/html; 19 | try_files $uri /index.html; 20 | } 21 | 22 | location /api { 23 | proxy_set_header X-Real-IP $remote_addr; 24 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 25 | proxy_set_header X-NginX-Proxy true; 26 | proxy_pass http://localhost:8080/api; 27 | proxy_ssl_session_reuse off; 28 | proxy_set_header Host $http_host; 29 | proxy_cache_bypass $http_upgrade; 30 | proxy_redirect off; 31 | } 32 | 33 | location /actuator { 34 | proxy_set_header X-Real-IP $remote_addr; 35 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 36 | proxy_set_header X-NginX-Proxy true; 37 | proxy_pass http://localhost:8080/actuator; 38 | proxy_ssl_session_reuse off; 39 | proxy_set_header Host $http_host; 40 | proxy_cache_bypass $http_upgrade; 41 | proxy_redirect off; 42 | } 43 | 44 | location /swagger-ui { 45 | proxy_set_header X-Real-IP $remote_addr; 46 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 47 | proxy_set_header X-NginX-Proxy true; 48 | proxy_pass http://localhost:8080/swagger-ui; 49 | proxy_ssl_session_reuse off; 50 | proxy_set_header Host $http_host; 51 | proxy_cache_bypass $http_upgrade; 52 | proxy_redirect off; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /docker/config/redis.conf: -------------------------------------------------------------------------------- 1 | appendonly yes -------------------------------------------------------------------------------- /docker/config/start_all.sh: -------------------------------------------------------------------------------- 1 | # configure and start redis 2 | mkdir -p /redis 3 | cd /redis 4 | nohup redis-server /app/config/redis.conf & 5 | 6 | # configure and start postgres 7 | mkdir -p /pgdata 8 | chown -R postgres:postgres /pgdata 9 | chmod 0700 /pgdata 10 | su postgres -c 'initdb -D /pgdata' 11 | su postgres -c 'pg_ctl start -D /pgdata' 12 | 13 | cat /app/config/banner.txt 14 | nohup /app/startup.sh & 15 | tail -f /app/logs/server.log -------------------------------------------------------------------------------- /docker/config/startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Starting Conductor Server and UI" 4 | echo "Running Nginx in background" 5 | # Start nginx as daemon 6 | nginx 7 | 8 | # Start the server 9 | cd /app/libs 10 | echo "Using config properties"; 11 | export config_file=/app/config/config.properties 12 | 13 | if [[ -z "${JVM_MEMORY_SETTINGS}" ]]; then 14 | JVM_MEMORY="-Xms512M -Xmx750M" 15 | else 16 | JVM_MEMORY="${JVM_MEMORY_SETTINGS}" 17 | fi 18 | 19 | echo "Starting Conductor with $JVM_MEMORY memory settings" 20 | export LOG_FILE=/app/logs/server.log 21 | 22 | java $JVM_MEMORY -jar -DCONDUCTOR_CONFIG_FILE=$config_file server.jar -------------------------------------------------------------------------------- /docker/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2.3' 3 | 4 | services: 5 | postgresql: 6 | image: postgres:11.15-alpine 7 | environment: 8 | - POSTGRES_USER=postgres 9 | - POSTGRES_PASSWORD=postgres 10 | - POSTGRES_DB=postgres 11 | ports: 12 | - "7432:5432" 13 | networks: 14 | - internal 15 | 16 | r1: 17 | image: redis:alpine 18 | restart: on-failure 19 | command: "redis-server --save 60 1 --appendonly yes --loglevel warning" 20 | healthcheck: 21 | test: [ "CMD", "redis-cli","ping" ] 22 | ports: 23 | - "7379:6379" 24 | networks: 25 | - internal 26 | 27 | server-1: 28 | container_name: conductor 29 | build: 30 | context: ../ 31 | dockerfile: docker/DockerfileServer 32 | environment: 33 | - spring.datasource.url=jdbc:postgresql://postgresql:5432/postgres 34 | - conductor.redis-lock.serverAddress=redis://r1:6379 35 | - conductor.redis.hosts=r1:6379:us-east-1c 36 | - conductor.queue.type=redis_standalone 37 | - conductor.db.type=redis_standalone 38 | healthcheck: 39 | test: [ "CMD", "curl", "-f", "http://localhost:8080/health" ] 40 | depends_on: 41 | r1: 42 | condition: service_healthy 43 | links: 44 | - r1 45 | - postgresql 46 | ports: 47 | - "9090:8080" 48 | - "6000:5000" 49 | networks: 50 | - internal 51 | 52 | networks: 53 | internal: 54 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | org.gradle.parallel=true -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orkes-io/orkes-conductor-community/004d605db5d851ad99ffe84bbad961a97517b061/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionSha256Sum=b586e04868a22fd817c8971330fec37e298f3242eb85c374181b12d637f80302 4 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /licenseheader.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright $YEAR Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ -------------------------------------------------------------------------------- /persistence/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | 3 | 4 | 5 | implementation "com.netflix.conductor:conductor-common:${versions.revConductor}" 6 | implementation "com.netflix.conductor:conductor-core:${versions.revConductor}" 7 | compileOnly 'org.springframework.boot:spring-boot-starter' 8 | 9 | implementation "redis.clients:jedis:${versions.revJedis}" 10 | 11 | implementation "com.google.guava:guava:${versions.revGuava}" 12 | 13 | //Object Mapper 14 | implementation "com.fasterxml.jackson.core:jackson-databind" 15 | implementation "com.fasterxml.jackson.core:jackson-core" 16 | 17 | //In Memory redis 18 | implementation "org.rarefiedredis.redis:redis-java:${versions.revRarefiedRedis}" 19 | 20 | //In memory 21 | implementation "com.jayway.jsonpath:json-path:2.4.0" 22 | 23 | 24 | //Micrometer 25 | implementation "io.micrometer:micrometer-core:1.7.5" 26 | 27 | //spring 28 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 29 | testImplementation 'org.junit.vintage:junit-vintage-engine' 30 | 31 | } -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/config/AnyRedisCondition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.config; 14 | 15 | import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; 16 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 17 | 18 | public class AnyRedisCondition extends AnyNestedCondition { 19 | 20 | public AnyRedisCondition() { 21 | super(ConfigurationPhase.PARSE_CONFIGURATION); 22 | } 23 | 24 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "dynomite") 25 | static class DynomiteClusterCondition {} 26 | 27 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "memory") 28 | static class InMemoryRedisCondition {} 29 | 30 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_cluster") 31 | static class RedisClusterConfiguration {} 32 | 33 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_sentinel") 34 | static class RedisSentinelConfiguration {} 35 | 36 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_standalone") 37 | static class RedisStandaloneConfiguration {} 38 | } 39 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/config/InMemoryRedisConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.config; 14 | 15 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 16 | import org.springframework.context.annotation.Bean; 17 | import org.springframework.context.annotation.Configuration; 18 | import org.springframework.context.annotation.Primary; 19 | 20 | import com.netflix.conductor.redis.dynoqueue.LocalhostHostSupplier; 21 | import com.netflix.conductor.redis.jedis.JedisMock; 22 | import com.netflix.conductor.redis.jedis.JedisStandalone; 23 | import com.netflix.conductor.redis.jedis.OrkesJedisCommands; 24 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 25 | import com.netflix.dyno.connectionpool.HostSupplier; 26 | 27 | import redis.clients.jedis.Jedis; 28 | import redis.clients.jedis.JedisPool; 29 | 30 | @Configuration(proxyBeanMethods = false) 31 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "memory") 32 | public class InMemoryRedisConfiguration { 33 | 34 | public static final JedisMock jedisMock = new JedisMock(); 35 | 36 | @Bean 37 | public HostSupplier hostSupplier(RedisProperties properties) { 38 | return new LocalhostHostSupplier(properties); 39 | } 40 | 41 | @Bean 42 | public JedisMock jedisMock() { 43 | return jedisMock; 44 | } 45 | 46 | @Bean 47 | public OrkesJedisCommands jedisCommands() { 48 | return new JedisStandalone(jedisPool()); 49 | } 50 | 51 | @Bean 52 | public JedisPool jedisPool() { 53 | return new JedisPool() { 54 | @Override 55 | public Jedis getResource() { 56 | return jedisMock; 57 | } 58 | }; 59 | } 60 | 61 | @Primary 62 | @Bean 63 | public OrkesJedisProxy OrkesJedisProxy() { 64 | System.out.println("OrkesJedisProxy created"); 65 | return new OrkesJedisProxy(jedisCommands()); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/config/RedisClusterConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.config; 14 | 15 | import java.util.List; 16 | import java.util.Set; 17 | import java.util.stream.Collectors; 18 | 19 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 23 | import org.springframework.context.annotation.Configuration; 24 | 25 | import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier; 26 | import com.netflix.conductor.redis.jedis.JedisCluster; 27 | import com.netflix.dyno.connectionpool.Host; 28 | 29 | import redis.clients.jedis.HostAndPort; 30 | import redis.clients.jedis.Jedis; 31 | import redis.clients.jedis.Protocol; 32 | 33 | @Configuration(proxyBeanMethods = false) 34 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_cluster") 35 | public class RedisClusterConfiguration { 36 | 37 | private static final Logger log = LoggerFactory.getLogger(RedisClusterConfiguration.class); 38 | 39 | // Same as redis.clients.jedis.BinaryJedisCluster 40 | protected static final int DEFAULT_MAX_ATTEMPTS = 5; 41 | 42 | public JedisCluster getJedisCluster(RedisProperties properties) { 43 | GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig<>(); 44 | genericObjectPoolConfig.setMaxTotal(properties.getMaxConnectionsPerHost()); 45 | ConfigurationHostSupplier hostSupplier = new ConfigurationHostSupplier(properties); 46 | Set hosts = 47 | hostSupplier.getHosts().stream() 48 | .map(h -> new HostAndPort(h.getHostName(), h.getPort())) 49 | .collect(Collectors.toSet()); 50 | String password = getPassword(hostSupplier.getHosts()); 51 | 52 | if (password != null) { 53 | log.info("Connecting to Redis Cluster with AUTH"); 54 | } 55 | 56 | return new JedisCluster( 57 | new redis.clients.jedis.JedisCluster( 58 | hosts, 59 | Protocol.DEFAULT_TIMEOUT, 60 | Protocol.DEFAULT_TIMEOUT, 61 | DEFAULT_MAX_ATTEMPTS, 62 | password, 63 | null, 64 | genericObjectPoolConfig, 65 | properties.isSsl())); 66 | } 67 | 68 | private String getPassword(List hosts) { 69 | return hosts.isEmpty() ? null : hosts.get(0).getPassword(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/config/RedisSentinelConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.config; 14 | 15 | import java.util.HashSet; 16 | import java.util.List; 17 | import java.util.Set; 18 | 19 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 23 | import org.springframework.context.annotation.Configuration; 24 | 25 | import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier; 26 | import com.netflix.conductor.redis.jedis.JedisSentinel; 27 | import com.netflix.dyno.connectionpool.Host; 28 | 29 | import redis.clients.jedis.Jedis; 30 | import redis.clients.jedis.JedisSentinelPool; 31 | import redis.clients.jedis.Protocol; 32 | 33 | @Configuration(proxyBeanMethods = false) 34 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_sentinel") 35 | public class RedisSentinelConfiguration { 36 | 37 | private static final Logger log = LoggerFactory.getLogger(RedisSentinelConfiguration.class); 38 | 39 | protected JedisSentinel getJedisSentinel(RedisProperties properties) { 40 | GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig<>(); 41 | genericObjectPoolConfig.setMinIdle(properties.getMinIdleConnections()); 42 | genericObjectPoolConfig.setMaxIdle(properties.getMaxIdleConnections()); 43 | genericObjectPoolConfig.setMaxTotal(properties.getMaxConnectionsPerHost()); 44 | genericObjectPoolConfig.setTestWhileIdle(properties.isTestWhileIdle()); 45 | genericObjectPoolConfig.setMinEvictableIdleTimeMillis( 46 | properties.getMinEvictableIdleTimeMillis()); 47 | genericObjectPoolConfig.setTimeBetweenEvictionRunsMillis( 48 | properties.getTimeBetweenEvictionRunsMillis()); 49 | genericObjectPoolConfig.setNumTestsPerEvictionRun(properties.getNumTestsPerEvictionRun()); 50 | ConfigurationHostSupplier hostSupplier = new ConfigurationHostSupplier(properties); 51 | 52 | log.info( 53 | "Starting conductor server using redis_sentinel and cluster " 54 | + properties.getClusterName()); 55 | Set sentinels = new HashSet<>(); 56 | for (Host host : hostSupplier.getHosts()) { 57 | sentinels.add(host.getHostName() + ":" + host.getPort()); 58 | } 59 | // We use the password of the first sentinel host as password and sentinelPassword 60 | String password = getPassword(hostSupplier.getHosts()); 61 | if (password != null) { 62 | return new JedisSentinel( 63 | new JedisSentinelPool( 64 | properties.getClusterName(), 65 | sentinels, 66 | genericObjectPoolConfig, 67 | Protocol.DEFAULT_TIMEOUT, 68 | Protocol.DEFAULT_TIMEOUT, 69 | password, 70 | properties.getDatabase(), 71 | null, 72 | Protocol.DEFAULT_TIMEOUT, 73 | Protocol.DEFAULT_TIMEOUT, 74 | password, 75 | null)); 76 | } else { 77 | return new JedisSentinel( 78 | new JedisSentinelPool( 79 | properties.getClusterName(), 80 | sentinels, 81 | genericObjectPoolConfig, 82 | Protocol.DEFAULT_TIMEOUT, 83 | null, 84 | properties.getDatabase())); 85 | } 86 | } 87 | 88 | private String getPassword(List hosts) { 89 | return hosts.isEmpty() ? null : hosts.get(0).getPassword(); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/config/RedisStandaloneConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.config; 14 | 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 18 | import org.springframework.context.annotation.Configuration; 19 | 20 | import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier; 21 | import com.netflix.dyno.connectionpool.Host; 22 | 23 | import redis.clients.jedis.JedisPool; 24 | import redis.clients.jedis.JedisPoolConfig; 25 | import redis.clients.jedis.Protocol; 26 | 27 | @Configuration(proxyBeanMethods = false) 28 | @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_standalone") 29 | public class RedisStandaloneConfiguration { 30 | 31 | private static final Logger log = LoggerFactory.getLogger(RedisStandaloneConfiguration.class); 32 | 33 | public JedisPool getJedisPool(RedisProperties redisProperties) { 34 | JedisPoolConfig config = new JedisPoolConfig(); 35 | config.setMinIdle(2); 36 | config.setMaxTotal(redisProperties.getMaxConnectionsPerHost()); 37 | log.info( 38 | "Starting conductor server using redis_standalone - use SSL? {}", 39 | redisProperties.isSsl()); 40 | ConfigurationHostSupplier hostSupplier = new ConfigurationHostSupplier(redisProperties); 41 | Host host = hostSupplier.getHosts().get(0); 42 | 43 | if (host.getPassword() != null) { 44 | log.info("Connecting to Redis Standalone with AUTH"); 45 | return new JedisPool( 46 | config, 47 | host.getHostName(), 48 | host.getPort(), 49 | Protocol.DEFAULT_TIMEOUT, 50 | host.getPassword(), 51 | redisProperties.getDatabase(), 52 | redisProperties.isSsl()); 53 | } else { 54 | return new JedisPool( 55 | config, 56 | host.getHostName(), 57 | host.getPort(), 58 | Protocol.DEFAULT_TIMEOUT, 59 | null, 60 | redisProperties.getDatabase(), 61 | redisProperties.isSsl()); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/dao/BaseDynoDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dao; 14 | 15 | import java.io.IOException; 16 | 17 | import org.apache.commons.lang3.StringUtils; 18 | 19 | import com.netflix.conductor.core.config.ConductorProperties; 20 | import com.netflix.conductor.metrics.Monitors; 21 | import com.netflix.conductor.redis.config.RedisProperties; 22 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 23 | 24 | import com.fasterxml.jackson.core.JsonProcessingException; 25 | import com.fasterxml.jackson.databind.ObjectMapper; 26 | 27 | public class BaseDynoDAO { 28 | 29 | private static final String NAMESPACE_SEP = "."; 30 | private static final String DAO_NAME = "redis"; 31 | private final String domain; 32 | private final RedisProperties properties; 33 | private final ConductorProperties conductorProperties; 34 | protected OrkesJedisProxy orkesJedisProxy; 35 | protected ObjectMapper objectMapper; 36 | 37 | protected BaseDynoDAO( 38 | OrkesJedisProxy jedisProxy, 39 | ObjectMapper objectMapper, 40 | ConductorProperties conductorProperties, 41 | RedisProperties properties) { 42 | this.orkesJedisProxy = jedisProxy; 43 | this.objectMapper = objectMapper; 44 | this.conductorProperties = conductorProperties; 45 | this.properties = properties; 46 | this.domain = properties.getKeyspaceDomain(); 47 | } 48 | 49 | String nsKey(String... nsValues) { 50 | String rootNamespace = properties.getWorkflowNamespacePrefix(); 51 | StringBuilder namespacedKey = new StringBuilder(); 52 | if (StringUtils.isNotBlank(rootNamespace)) { 53 | namespacedKey.append(rootNamespace).append(NAMESPACE_SEP); 54 | } 55 | String stack = conductorProperties.getStack(); 56 | if (StringUtils.isNotBlank(stack)) { 57 | namespacedKey.append(stack).append(NAMESPACE_SEP); 58 | } 59 | if (StringUtils.isNotBlank(domain)) { 60 | namespacedKey.append(domain).append(NAMESPACE_SEP); 61 | } 62 | for (String nsValue : nsValues) { 63 | namespacedKey.append(nsValue).append(NAMESPACE_SEP); 64 | } 65 | return StringUtils.removeEnd(namespacedKey.toString(), NAMESPACE_SEP); 66 | } 67 | 68 | String toJson(Object value) { 69 | try { 70 | return objectMapper.writeValueAsString(value); 71 | } catch (JsonProcessingException e) { 72 | throw new RuntimeException(e); 73 | } 74 | } 75 | 76 | T readValue(String json, Class clazz) { 77 | try { 78 | return objectMapper.readValue(json, clazz); 79 | } catch (IOException e) { 80 | throw new RuntimeException(e); 81 | } 82 | } 83 | 84 | void recordRedisDaoRequests(String action) { 85 | recordRedisDaoRequests(action, "n/a", "n/a"); 86 | } 87 | 88 | void recordRedisDaoRequests(String action, String taskType, String workflowType) { 89 | Monitors.recordDaoRequests(DAO_NAME, action, taskType, workflowType); 90 | } 91 | 92 | void recordRedisDaoEventRequests(String action, String event) { 93 | Monitors.recordDaoEventRequests(DAO_NAME, action, event); 94 | } 95 | 96 | void recordRedisDaoPayloadSize(String action, int size, String taskType, String workflowType) { 97 | Monitors.recordDaoPayloadSize( 98 | DAO_NAME, 99 | action, 100 | StringUtils.defaultIfBlank(taskType, ""), 101 | StringUtils.defaultIfBlank(workflowType, ""), 102 | size); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/dao/RedisPollDataDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dao; 14 | 15 | import java.time.Clock; 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.Map; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | 21 | import org.apache.commons.lang3.StringUtils; 22 | import org.springframework.context.annotation.Conditional; 23 | import org.springframework.stereotype.Component; 24 | 25 | import com.netflix.conductor.common.metadata.tasks.PollData; 26 | import com.netflix.conductor.core.config.ConductorProperties; 27 | import com.netflix.conductor.dao.PollDataDAO; 28 | import com.netflix.conductor.redis.config.AnyRedisCondition; 29 | import com.netflix.conductor.redis.config.RedisProperties; 30 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 31 | 32 | import com.fasterxml.jackson.databind.ObjectMapper; 33 | import com.google.common.base.Preconditions; 34 | import lombok.extern.slf4j.Slf4j; 35 | 36 | @Component 37 | @Conditional(AnyRedisCondition.class) 38 | @Slf4j 39 | public class RedisPollDataDAO extends BaseDynoDAO implements PollDataDAO { 40 | 41 | private static final String POLL_DATA = "POLL_DATA"; 42 | 43 | private final Clock clock; 44 | 45 | private final Map lastUpdateTimes = new ConcurrentHashMap<>(); 46 | 47 | private static final int MAX_UPDATE_FREQUENCY = 10_000; // 10 second 48 | 49 | public RedisPollDataDAO( 50 | OrkesJedisProxy orkesJedisProxy, 51 | ObjectMapper objectMapper, 52 | ConductorProperties conductorProperties, 53 | RedisProperties properties) { 54 | super(orkesJedisProxy, objectMapper, conductorProperties, properties); 55 | this.clock = Clock.systemDefaultZone(); 56 | log.info("Using OrkesPollDataDAO"); 57 | } 58 | 59 | @Override 60 | public void updateLastPollData(String taskDefName, String domain, String workerId) { 61 | 62 | long now = clock.millis(); 63 | lastUpdateTimes.compute( 64 | taskDefName, 65 | (s, lastUpdateTime) -> { 66 | if (lastUpdateTime == null || lastUpdateTime < (now - MAX_UPDATE_FREQUENCY)) { 67 | _updateLastPollData(taskDefName, domain, workerId); 68 | return now; 69 | } 70 | return lastUpdateTime; 71 | }); 72 | } 73 | 74 | private void _updateLastPollData(String taskDefName, String domain, String workerId) { 75 | Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null"); 76 | PollData pollData = new PollData(taskDefName, domain, workerId, System.currentTimeMillis()); 77 | 78 | String key = nsKey(POLL_DATA, pollData.getQueueName()); 79 | String field = (domain == null) ? "DEFAULT" : domain; 80 | 81 | String payload = toJson(pollData); 82 | recordRedisDaoRequests("updatePollData"); 83 | recordRedisDaoPayloadSize("updatePollData", payload.length(), "n/a", "n/a"); 84 | orkesJedisProxy.hset(key, field, payload); 85 | } 86 | 87 | @Override 88 | public PollData getPollData(String taskDefName, String domain) { 89 | Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null"); 90 | 91 | String key = nsKey(POLL_DATA, taskDefName); 92 | String field = (domain == null) ? "DEFAULT" : domain; 93 | 94 | String pollDataJsonString = orkesJedisProxy.hget(key, field); 95 | recordRedisDaoRequests("getPollData"); 96 | recordRedisDaoPayloadSize( 97 | "getPollData", StringUtils.length(pollDataJsonString), "n/a", "n/a"); 98 | 99 | PollData pollData = null; 100 | if (StringUtils.isNotBlank(pollDataJsonString)) { 101 | pollData = readValue(pollDataJsonString, PollData.class); 102 | } 103 | return pollData; 104 | } 105 | 106 | @Override 107 | public List getPollData(String taskDefName) { 108 | Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null"); 109 | 110 | String key = nsKey(POLL_DATA, taskDefName); 111 | 112 | Map pMapdata = orkesJedisProxy.hgetAll(key); 113 | List pollData = new ArrayList<>(); 114 | if (pMapdata != null) { 115 | pMapdata.values() 116 | .forEach( 117 | pollDataJsonString -> { 118 | pollData.add(readValue(pollDataJsonString, PollData.class)); 119 | recordRedisDaoRequests("getPollData"); 120 | recordRedisDaoPayloadSize( 121 | "getPollData", pollDataJsonString.length(), "n/a", "n/a"); 122 | }); 123 | } 124 | return pollData; 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/ConfigurationHostSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dynoqueue; 14 | 15 | import java.util.Arrays; 16 | import java.util.List; 17 | import java.util.stream.Collectors; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import com.netflix.conductor.redis.config.RedisProperties; 23 | import com.netflix.dyno.connectionpool.Host; 24 | import com.netflix.dyno.connectionpool.HostBuilder; 25 | import com.netflix.dyno.connectionpool.HostSupplier; 26 | 27 | public class ConfigurationHostSupplier implements HostSupplier { 28 | 29 | private static final Logger log = LoggerFactory.getLogger(ConfigurationHostSupplier.class); 30 | 31 | private final RedisProperties properties; 32 | 33 | public ConfigurationHostSupplier(RedisProperties properties) { 34 | this.properties = properties; 35 | } 36 | 37 | @Override 38 | public List getHosts() { 39 | return parseHostsFromConfig(); 40 | } 41 | 42 | private List parseHostsFromConfig() { 43 | String hosts = properties.getHosts(); 44 | if (hosts == null) { 45 | // FIXME This type of validation probably doesn't belong here. 46 | String message = 47 | "Missing dynomite/redis hosts. Ensure 'workflow.dynomite.cluster.hosts' has been set in the supplied configuration."; 48 | log.error(message); 49 | throw new RuntimeException(message); 50 | } 51 | return parseHostsFrom(hosts); 52 | } 53 | 54 | private List parseHostsFrom(String hostConfig) { 55 | List hostConfigs = Arrays.asList(hostConfig.split(";")); 56 | 57 | return hostConfigs.stream() 58 | .map( 59 | hc -> { 60 | String[] hostConfigValues = hc.split(":"); 61 | String host = hostConfigValues[0]; 62 | int port = Integer.parseInt(hostConfigValues[1]); 63 | String rack = hostConfigValues[2]; 64 | 65 | if (hostConfigValues.length >= 4) { 66 | String password = hostConfigValues[3]; 67 | return new HostBuilder() 68 | .setHostname(host) 69 | .setPort(port) 70 | .setRack(rack) 71 | .setStatus(Host.Status.Up) 72 | .setPassword(password) 73 | .createHost(); 74 | } 75 | return new HostBuilder() 76 | .setHostname(host) 77 | .setPort(port) 78 | .setRack(rack) 79 | .setStatus(Host.Status.Up) 80 | .createHost(); 81 | }) 82 | .collect(Collectors.toList()); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/LocalhostHostSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dynoqueue; 14 | 15 | import java.util.Arrays; 16 | import java.util.List; 17 | 18 | import com.netflix.conductor.redis.config.RedisProperties; 19 | import com.netflix.dyno.connectionpool.Host; 20 | import com.netflix.dyno.connectionpool.HostBuilder; 21 | import com.netflix.dyno.connectionpool.HostSupplier; 22 | 23 | public class LocalhostHostSupplier implements HostSupplier { 24 | 25 | private final RedisProperties properties; 26 | 27 | public LocalhostHostSupplier(RedisProperties properties) { 28 | this.properties = properties; 29 | } 30 | 31 | @Override 32 | public List getHosts() { 33 | Host dynoHost = 34 | new HostBuilder() 35 | .setHostname("localhost") 36 | .setIpAddress("0") 37 | .setRack(properties.getAvailabilityZone()) 38 | .setStatus(Host.Status.Up) 39 | .createHost(); 40 | return Arrays.asList(dynoHost); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/conductor/redis/jedis/OrkesJedisCommands.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.jedis; 14 | 15 | import redis.clients.jedis.commands.JedisCommands; 16 | 17 | public interface OrkesJedisCommands extends JedisCommands { 18 | 19 | String set(byte[] key, byte[] value); 20 | 21 | byte[] getBytes(byte[] key); 22 | } 23 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/dyno/connectionpool/HostBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.dyno.connectionpool; 14 | 15 | import static com.netflix.dyno.connectionpool.Host.DEFAULT_DATASTORE_PORT; 16 | import static com.netflix.dyno.connectionpool.Host.DEFAULT_PORT; 17 | 18 | public class HostBuilder { 19 | private String hostname; 20 | private int port = DEFAULT_PORT; 21 | private String rack; 22 | private String ipAddress = null; 23 | private int securePort = DEFAULT_PORT; 24 | private int datastorePort = DEFAULT_DATASTORE_PORT; 25 | private String datacenter = null; 26 | private Host.Status status = Host.Status.Down; 27 | private String hashtag = null; 28 | private String password = null; 29 | 30 | public HostBuilder setPort(int port) { 31 | this.port = port; 32 | return this; 33 | } 34 | 35 | public HostBuilder setRack(String rack) { 36 | this.rack = rack; 37 | return this; 38 | } 39 | 40 | public HostBuilder setHostname(String hostname) { 41 | this.hostname = hostname; 42 | return this; 43 | } 44 | 45 | public HostBuilder setIpAddress(String ipAddress) { 46 | this.ipAddress = ipAddress; 47 | return this; 48 | } 49 | 50 | public HostBuilder setSecurePort(int securePort) { 51 | this.securePort = securePort; 52 | return this; 53 | } 54 | 55 | public HostBuilder setDatacenter(String datacenter) { 56 | this.datacenter = datacenter; 57 | return this; 58 | } 59 | 60 | public HostBuilder setStatus(Host.Status status) { 61 | this.status = status; 62 | return this; 63 | } 64 | 65 | public HostBuilder setHashtag(String hashtag) { 66 | this.hashtag = hashtag; 67 | return this; 68 | } 69 | 70 | public HostBuilder setPassword(String password) { 71 | this.password = password; 72 | return this; 73 | } 74 | 75 | public HostBuilder setDatastorePort(int datastorePort) { 76 | this.datastorePort = datastorePort; 77 | return this; 78 | } 79 | 80 | public Host createHost() { 81 | return new Host( 82 | hostname, 83 | ipAddress, 84 | port, 85 | securePort, 86 | datastorePort, 87 | rack, 88 | datacenter, 89 | status, 90 | hashtag, 91 | password); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /persistence/src/main/java/com/netflix/dyno/connectionpool/HostSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.dyno.connectionpool; 14 | 15 | import java.util.List; 16 | 17 | public interface HostSupplier { 18 | 19 | public List getHosts(); 20 | } 21 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/common/config/TestObjectMapperConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.common.config; 14 | 15 | import org.springframework.context.annotation.Bean; 16 | import org.springframework.context.annotation.Configuration; 17 | 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | 20 | /** Supplies the standard Conductor {@link ObjectMapper} for tests that need them. */ 21 | @Configuration 22 | public class TestObjectMapperConfiguration { 23 | 24 | @Bean 25 | public ObjectMapper testObjectMapper() { 26 | return new ObjectMapperProvider().getObjectMapper(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/dao/PollDataDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.dao; 14 | 15 | import java.util.List; 16 | 17 | import org.junit.Ignore; 18 | import org.junit.Test; 19 | 20 | import com.netflix.conductor.common.metadata.tasks.PollData; 21 | 22 | import static org.junit.Assert.*; 23 | 24 | public abstract class PollDataDAOTest { 25 | 26 | protected abstract PollDataDAO getPollDataDAO(); 27 | 28 | // TODO review this test 29 | @Ignore 30 | @Test 31 | public void testPollData() { 32 | getPollDataDAO().updateLastPollData("taskDef", null, "workerId1"); 33 | PollData pollData = getPollDataDAO().getPollData("taskDef", null); 34 | assertNotNull(pollData); 35 | assertTrue(pollData.getLastPollTime() > 0); 36 | assertEquals(pollData.getQueueName(), "taskDef"); 37 | assertNull(pollData.getDomain()); 38 | assertEquals(pollData.getWorkerId(), "workerId1"); 39 | 40 | getPollDataDAO().updateLastPollData("taskDef", "domain1", "workerId1"); 41 | pollData = getPollDataDAO().getPollData("taskDef", "domain1"); 42 | assertNotNull(pollData); 43 | assertTrue(pollData.getLastPollTime() > 0); 44 | assertEquals(pollData.getQueueName(), "taskDef"); 45 | assertEquals(pollData.getDomain(), "domain1"); 46 | assertEquals(pollData.getWorkerId(), "workerId1"); 47 | 48 | List pData = getPollDataDAO().getPollData("taskDef"); 49 | assertEquals(pData.size(), 2); 50 | 51 | pollData = getPollDataDAO().getPollData("taskDef", "domain2"); 52 | assertNull(pollData); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/redis/dao/BaseDynoDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dao; 14 | 15 | import org.junit.Before; 16 | import org.junit.Test; 17 | import org.mockito.Mock; 18 | 19 | import com.netflix.conductor.core.config.ConductorProperties; 20 | import com.netflix.conductor.redis.config.RedisProperties; 21 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 22 | 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | 25 | import static org.junit.Assert.assertEquals; 26 | import static org.mockito.Mockito.mock; 27 | import static org.mockito.Mockito.when; 28 | 29 | public class BaseDynoDAOTest { 30 | 31 | @Mock private OrkesJedisProxy jedisProxy; 32 | 33 | @Mock private ObjectMapper objectMapper; 34 | 35 | private RedisProperties properties; 36 | private ConductorProperties conductorProperties; 37 | 38 | private BaseDynoDAO baseDynoDAO; 39 | 40 | @Before 41 | public void setUp() { 42 | properties = mock(RedisProperties.class); 43 | conductorProperties = mock(ConductorProperties.class); 44 | this.baseDynoDAO = 45 | new BaseDynoDAO(jedisProxy, objectMapper, conductorProperties, properties); 46 | } 47 | 48 | @Test 49 | public void testNsKey() { 50 | assertEquals("", baseDynoDAO.nsKey()); 51 | 52 | String[] keys = {"key1", "key2"}; 53 | assertEquals("key1.key2", baseDynoDAO.nsKey(keys)); 54 | 55 | when(properties.getWorkflowNamespacePrefix()).thenReturn("test"); 56 | assertEquals("test", baseDynoDAO.nsKey()); 57 | 58 | assertEquals("test.key1.key2", baseDynoDAO.nsKey(keys)); 59 | 60 | when(conductorProperties.getStack()).thenReturn("stack"); 61 | assertEquals("test.stack.key1.key2", baseDynoDAO.nsKey(keys)); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/redis/dao/RedisEventHandlerDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dao; 14 | 15 | import java.time.Duration; 16 | import java.util.List; 17 | import java.util.UUID; 18 | 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.springframework.beans.factory.annotation.Autowired; 23 | import org.springframework.test.context.ContextConfiguration; 24 | import org.springframework.test.context.junit4.SpringRunner; 25 | 26 | import com.netflix.conductor.common.config.TestObjectMapperConfiguration; 27 | import com.netflix.conductor.common.metadata.events.EventHandler; 28 | import com.netflix.conductor.common.metadata.events.EventHandler.Action; 29 | import com.netflix.conductor.common.metadata.events.EventHandler.Action.Type; 30 | import com.netflix.conductor.common.metadata.events.EventHandler.StartWorkflow; 31 | import com.netflix.conductor.core.config.ConductorProperties; 32 | import com.netflix.conductor.redis.config.RedisProperties; 33 | import com.netflix.conductor.redis.jedis.JedisMock; 34 | import com.netflix.conductor.redis.jedis.JedisStandalone; 35 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 36 | 37 | import com.fasterxml.jackson.databind.ObjectMapper; 38 | import redis.clients.jedis.JedisPool; 39 | 40 | import static org.junit.Assert.assertEquals; 41 | import static org.junit.Assert.assertNotNull; 42 | import static org.mockito.Mockito.mock; 43 | import static org.mockito.Mockito.when; 44 | 45 | @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) 46 | @RunWith(SpringRunner.class) 47 | public class RedisEventHandlerDAOTest { 48 | 49 | private RedisEventHandlerDAO redisEventHandlerDAO; 50 | 51 | @Autowired private ObjectMapper objectMapper; 52 | 53 | @Before 54 | public void init() { 55 | ConductorProperties conductorProperties = mock(ConductorProperties.class); 56 | RedisProperties properties = mock(RedisProperties.class); 57 | when(properties.getTaskDefCacheRefreshInterval()).thenReturn(Duration.ofSeconds(60)); 58 | JedisPool jedisPool = mock(JedisPool.class); 59 | when(jedisPool.getResource()).thenReturn(new JedisMock()); 60 | OrkesJedisProxy orkesJedisProxy = new OrkesJedisProxy(new JedisStandalone(jedisPool)); 61 | 62 | redisEventHandlerDAO = 63 | new RedisEventHandlerDAO( 64 | orkesJedisProxy, objectMapper, conductorProperties, properties); 65 | } 66 | 67 | @Test 68 | public void testEventHandlers() { 69 | String event1 = "SQS::arn:account090:sqstest1"; 70 | String event2 = "SQS::arn:account090:sqstest2"; 71 | 72 | EventHandler eventHandler = new EventHandler(); 73 | eventHandler.setName(UUID.randomUUID().toString()); 74 | eventHandler.setActive(false); 75 | Action action = new Action(); 76 | action.setAction(Type.start_workflow); 77 | action.setStart_workflow(new StartWorkflow()); 78 | action.getStart_workflow().setName("test_workflow"); 79 | eventHandler.getActions().add(action); 80 | eventHandler.setEvent(event1); 81 | 82 | redisEventHandlerDAO.addEventHandler(eventHandler); 83 | List allEventHandlers = redisEventHandlerDAO.getAllEventHandlers(); 84 | assertNotNull(allEventHandlers); 85 | assertEquals(1, allEventHandlers.size()); 86 | assertEquals(eventHandler.getName(), allEventHandlers.get(0).getName()); 87 | assertEquals(eventHandler.getEvent(), allEventHandlers.get(0).getEvent()); 88 | 89 | List byEvents = redisEventHandlerDAO.getEventHandlersForEvent(event1, true); 90 | assertNotNull(byEvents); 91 | assertEquals(0, byEvents.size()); // event is marked as in-active 92 | 93 | eventHandler.setActive(true); 94 | eventHandler.setEvent(event2); 95 | redisEventHandlerDAO.updateEventHandler(eventHandler); 96 | 97 | allEventHandlers = redisEventHandlerDAO.getAllEventHandlers(); 98 | assertNotNull(allEventHandlers); 99 | assertEquals(1, allEventHandlers.size()); 100 | 101 | byEvents = redisEventHandlerDAO.getEventHandlersForEvent(event1, true); 102 | assertNotNull(byEvents); 103 | assertEquals(0, byEvents.size()); 104 | 105 | byEvents = redisEventHandlerDAO.getEventHandlersForEvent(event2, true); 106 | assertNotNull(byEvents); 107 | assertEquals(1, byEvents.size()); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/redis/dao/RedisExecutionDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dao; 14 | 15 | import java.time.Duration; 16 | import java.util.Collections; 17 | import java.util.List; 18 | 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.springframework.beans.factory.annotation.Autowired; 23 | import org.springframework.test.context.ContextConfiguration; 24 | import org.springframework.test.context.junit4.SpringRunner; 25 | 26 | import com.netflix.conductor.common.config.TestObjectMapperConfiguration; 27 | import com.netflix.conductor.common.metadata.tasks.TaskDef; 28 | import com.netflix.conductor.core.config.ConductorProperties; 29 | import com.netflix.conductor.dao.ExecutionDAO; 30 | import com.netflix.conductor.dao.ExecutionDAOTest; 31 | import com.netflix.conductor.model.TaskModel; 32 | import com.netflix.conductor.redis.config.RedisProperties; 33 | import com.netflix.conductor.redis.jedis.JedisMock; 34 | import com.netflix.conductor.redis.jedis.JedisStandalone; 35 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 36 | 37 | import com.fasterxml.jackson.databind.ObjectMapper; 38 | import redis.clients.jedis.JedisPool; 39 | 40 | import static org.junit.Assert.assertEquals; 41 | import static org.junit.Assert.assertNotNull; 42 | import static org.mockito.Mockito.mock; 43 | import static org.mockito.Mockito.when; 44 | 45 | @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) 46 | @RunWith(SpringRunner.class) 47 | public class RedisExecutionDAOTest extends ExecutionDAOTest { 48 | 49 | private RedisExecutionDAO executionDAO; 50 | 51 | @Autowired private ObjectMapper objectMapper; 52 | 53 | @Before 54 | public void init() { 55 | ConductorProperties conductorProperties = mock(ConductorProperties.class); 56 | RedisProperties properties = mock(RedisProperties.class); 57 | when(properties.getTaskDefCacheRefreshInterval()).thenReturn(Duration.ofSeconds(60)); 58 | JedisPool jedisPool = mock(JedisPool.class); 59 | when(jedisPool.getResource()).thenReturn(new JedisMock()); 60 | OrkesJedisProxy orkesJedisProxy = new OrkesJedisProxy(new JedisStandalone(jedisPool)); 61 | 62 | executionDAO = 63 | new RedisExecutionDAO( 64 | orkesJedisProxy, objectMapper, conductorProperties, properties); 65 | } 66 | 67 | @Test 68 | public void testCorrelateTaskToWorkflowInDS() { 69 | String workflowId = "workflowId"; 70 | String taskId = "taskId1"; 71 | String taskDefName = "task1"; 72 | 73 | TaskDef def = new TaskDef(); 74 | def.setName("task1"); 75 | def.setConcurrentExecLimit(1); 76 | 77 | TaskModel task = new TaskModel(); 78 | task.setTaskId(taskId); 79 | task.setWorkflowInstanceId(workflowId); 80 | task.setReferenceTaskName("ref_name"); 81 | task.setTaskDefName(taskDefName); 82 | task.setTaskType(taskDefName); 83 | task.setStatus(TaskModel.Status.IN_PROGRESS); 84 | List tasks = executionDAO.createTasks(Collections.singletonList(task)); 85 | assertNotNull(tasks); 86 | assertEquals(1, tasks.size()); 87 | 88 | executionDAO.correlateTaskToWorkflowInDS(taskId, workflowId); 89 | tasks = executionDAO.getTasksForWorkflow(workflowId); 90 | assertNotNull(tasks); 91 | assertEquals(workflowId, tasks.get(0).getWorkflowInstanceId()); 92 | assertEquals(taskId, tasks.get(0).getTaskId()); 93 | } 94 | 95 | @Override 96 | protected ExecutionDAO getExecutionDAO() { 97 | return executionDAO; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/redis/dao/RedisPollDataDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dao; 14 | 15 | import java.time.Duration; 16 | 17 | import org.junit.Before; 18 | import org.junit.runner.RunWith; 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.test.context.ContextConfiguration; 21 | import org.springframework.test.context.junit4.SpringRunner; 22 | 23 | import com.netflix.conductor.common.config.TestObjectMapperConfiguration; 24 | import com.netflix.conductor.core.config.ConductorProperties; 25 | import com.netflix.conductor.dao.PollDataDAO; 26 | import com.netflix.conductor.dao.PollDataDAOTest; 27 | import com.netflix.conductor.redis.config.RedisProperties; 28 | import com.netflix.conductor.redis.jedis.JedisMock; 29 | import com.netflix.conductor.redis.jedis.JedisStandalone; 30 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 31 | 32 | import com.fasterxml.jackson.databind.ObjectMapper; 33 | import redis.clients.jedis.JedisPool; 34 | 35 | import static org.mockito.Mockito.mock; 36 | import static org.mockito.Mockito.when; 37 | 38 | @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) 39 | @RunWith(SpringRunner.class) 40 | public class RedisPollDataDAOTest extends PollDataDAOTest { 41 | 42 | private PollDataDAO redisPollDataDAO; 43 | 44 | @Autowired private ObjectMapper objectMapper; 45 | 46 | @Before 47 | public void init() { 48 | ConductorProperties conductorProperties = mock(ConductorProperties.class); 49 | RedisProperties properties = mock(RedisProperties.class); 50 | when(properties.getTaskDefCacheRefreshInterval()).thenReturn(Duration.ofSeconds(60)); 51 | JedisPool jedisPool = mock(JedisPool.class); 52 | when(jedisPool.getResource()).thenReturn(new JedisMock()); 53 | OrkesJedisProxy orkesJedisProxy = new OrkesJedisProxy(new JedisStandalone(jedisPool)); 54 | 55 | redisPollDataDAO = 56 | new RedisPollDataDAO( 57 | orkesJedisProxy, objectMapper, conductorProperties, properties); 58 | } 59 | 60 | @Override 61 | protected PollDataDAO getPollDataDAO() { 62 | return redisPollDataDAO; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/redis/dao/RedisRateLimitDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.dao; 14 | 15 | import java.time.Duration; 16 | import java.util.UUID; 17 | 18 | import org.junit.Before; 19 | import org.junit.Test; 20 | import org.junit.runner.RunWith; 21 | import org.springframework.beans.factory.annotation.Autowired; 22 | import org.springframework.test.context.ContextConfiguration; 23 | import org.springframework.test.context.junit4.SpringRunner; 24 | 25 | import com.netflix.conductor.common.config.TestObjectMapperConfiguration; 26 | import com.netflix.conductor.common.metadata.tasks.TaskDef; 27 | import com.netflix.conductor.core.config.ConductorProperties; 28 | import com.netflix.conductor.model.TaskModel; 29 | import com.netflix.conductor.redis.config.RedisProperties; 30 | import com.netflix.conductor.redis.jedis.JedisMock; 31 | import com.netflix.conductor.redis.jedis.JedisStandalone; 32 | import com.netflix.conductor.redis.jedis.OrkesJedisProxy; 33 | 34 | import com.fasterxml.jackson.databind.ObjectMapper; 35 | import redis.clients.jedis.JedisPool; 36 | 37 | import static org.junit.Assert.assertFalse; 38 | import static org.junit.Assert.assertTrue; 39 | import static org.mockito.Mockito.mock; 40 | import static org.mockito.Mockito.when; 41 | 42 | @ContextConfiguration(classes = {TestObjectMapperConfiguration.class}) 43 | @RunWith(SpringRunner.class) 44 | public class RedisRateLimitDAOTest { 45 | 46 | private RedisRateLimitingDAO rateLimitingDao; 47 | 48 | @Autowired private ObjectMapper objectMapper; 49 | 50 | @Before 51 | public void init() { 52 | ConductorProperties conductorProperties = mock(ConductorProperties.class); 53 | RedisProperties properties = mock(RedisProperties.class); 54 | when(properties.getTaskDefCacheRefreshInterval()).thenReturn(Duration.ofSeconds(60)); 55 | JedisPool jedisPool = mock(JedisPool.class); 56 | when(jedisPool.getResource()).thenReturn(new JedisMock()); 57 | OrkesJedisProxy orkesJedisProxy = new OrkesJedisProxy(new JedisStandalone(jedisPool)); 58 | 59 | rateLimitingDao = 60 | new RedisRateLimitingDAO( 61 | orkesJedisProxy, objectMapper, conductorProperties, properties); 62 | } 63 | 64 | @Test 65 | public void testExceedsRateLimitWhenNoRateLimitSet() { 66 | TaskDef taskDef = new TaskDef("TestTaskDefinition"); 67 | TaskModel task = new TaskModel(); 68 | task.setTaskId(UUID.randomUUID().toString()); 69 | task.setTaskDefName(taskDef.getName()); 70 | assertFalse(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); 71 | } 72 | 73 | @Test 74 | public void testExceedsRateLimitWithinLimit() { 75 | TaskDef taskDef = new TaskDef("TestTaskDefinition"); 76 | taskDef.setRateLimitFrequencyInSeconds(60); 77 | taskDef.setRateLimitPerFrequency(20); 78 | TaskModel task = new TaskModel(); 79 | task.setTaskId(UUID.randomUUID().toString()); 80 | task.setTaskDefName(taskDef.getName()); 81 | assertFalse(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); 82 | } 83 | 84 | @Test 85 | public void testExceedsRateLimitOutOfLimit() { 86 | TaskDef taskDef = new TaskDef("TestTaskDefinition"); 87 | taskDef.setRateLimitFrequencyInSeconds(60); 88 | taskDef.setRateLimitPerFrequency(1); 89 | TaskModel task = new TaskModel(); 90 | task.setTaskId(UUID.randomUUID().toString()); 91 | task.setTaskDefName(taskDef.getName()); 92 | assertFalse(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); 93 | assertTrue(rateLimitingDao.exceedsRateLimitPerFrequency(task, taskDef)); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /persistence/src/test/java/com/netflix/conductor/redis/jedis/ConfigurationHostSupplierTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.conductor.redis.jedis; 14 | 15 | import java.util.List; 16 | 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | 20 | import com.netflix.conductor.redis.config.RedisProperties; 21 | import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier; 22 | import com.netflix.dyno.connectionpool.Host; 23 | 24 | import static org.junit.Assert.assertEquals; 25 | import static org.junit.Assert.assertTrue; 26 | import static org.mockito.Mockito.mock; 27 | import static org.mockito.Mockito.when; 28 | 29 | public class ConfigurationHostSupplierTest { 30 | 31 | private RedisProperties properties; 32 | 33 | private ConfigurationHostSupplier configurationHostSupplier; 34 | 35 | @Before 36 | public void setUp() { 37 | properties = mock(RedisProperties.class); 38 | configurationHostSupplier = new ConfigurationHostSupplier(properties); 39 | } 40 | 41 | @Test 42 | public void getHost() { 43 | when(properties.getHosts()).thenReturn("dyno1:8102:us-east-1c"); 44 | 45 | List hosts = configurationHostSupplier.getHosts(); 46 | assertEquals(1, hosts.size()); 47 | 48 | Host firstHost = hosts.get(0); 49 | assertEquals("dyno1", firstHost.getHostName()); 50 | assertEquals(8102, firstHost.getPort()); 51 | assertEquals("us-east-1c", firstHost.getRack()); 52 | assertTrue(firstHost.isUp()); 53 | } 54 | 55 | @Test 56 | public void getMultipleHosts() { 57 | when(properties.getHosts()).thenReturn("dyno1:8102:us-east-1c;dyno2:8103:us-east-1c"); 58 | 59 | List hosts = configurationHostSupplier.getHosts(); 60 | assertEquals(2, hosts.size()); 61 | 62 | Host firstHost = hosts.get(0); 63 | assertEquals("dyno1", firstHost.getHostName()); 64 | assertEquals(8102, firstHost.getPort()); 65 | assertEquals("us-east-1c", firstHost.getRack()); 66 | assertTrue(firstHost.isUp()); 67 | 68 | Host secondHost = hosts.get(1); 69 | assertEquals("dyno2", secondHost.getHostName()); 70 | assertEquals(8103, secondHost.getPort()); 71 | assertEquals("us-east-1c", secondHost.getRack()); 72 | assertTrue(secondHost.isUp()); 73 | } 74 | 75 | @Test 76 | public void getAuthenticatedHost() { 77 | when(properties.getHosts()).thenReturn("redis1:6432:us-east-1c:password"); 78 | 79 | List hosts = configurationHostSupplier.getHosts(); 80 | assertEquals(1, hosts.size()); 81 | 82 | Host firstHost = hosts.get(0); 83 | assertEquals("redis1", firstHost.getHostName()); 84 | assertEquals(6432, firstHost.getPort()); 85 | assertEquals("us-east-1c", firstHost.getRack()); 86 | assertEquals("password", firstHost.getPassword()); 87 | assertTrue(firstHost.isUp()); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /run_tests.sh: -------------------------------------------------------------------------------- 1 | docker rm -f conductor_test_container 2 | docker build -f docker/DockerfileStandalone . -t conductor_test_container 3 | docker run -dit --name conductor_test_container -p 8899:8080 -p 4535:5000 -t conductor_test_container 4 | COUNTER=0 5 | MAX_TIME=120 6 | while ! curl -s http://localhost:8899/api/metadata/workflow -o /dev/null 7 | do 8 | echo "$(date) - still trying - since $COUNTER second, will wait for $MAX_TIME" 9 | sleep 1 10 | let COUNTER=COUNTER+1 11 | if [ $COUNTER -gt $MAX_TIME ]; 12 | then 13 | echo "Exceeded wait time of $MAX_TIME seconds. Terminating the build" 14 | exit 1 15 | fi 16 | done 17 | sleep 5 18 | echo "All set - starting tests now" 19 | ./gradlew clean 20 | ./gradlew -PIntegrationTests orkes-conductor-test-harness:test 21 | docker rm -f conductor_test_container 22 | 23 | -------------------------------------------------------------------------------- /scripts/run_local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | UI_PORT=1234 3 | read -p "Enter the port for UI [1234]: " UI_PORT 4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.config; 14 | 15 | import org.springframework.context.annotation.Configuration; 16 | import org.springframework.security.config.annotation.web.builders.HttpSecurity; 17 | import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; 18 | 19 | @Configuration 20 | public class SecurityDisabledConfig extends WebSecurityConfigurerAdapter { 21 | 22 | public SecurityDisabledConfig() { 23 | super(true); // Disable defaults 24 | } 25 | 26 | @Override 27 | protected void configure(HttpSecurity http) throws Exception {} 28 | } 29 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/execution/tasks/HttpSync.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.execution.tasks; 14 | 15 | import org.springframework.stereotype.Component; 16 | 17 | import com.netflix.conductor.core.execution.WorkflowExecutor; 18 | import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; 19 | import com.netflix.conductor.model.TaskModel; 20 | import com.netflix.conductor.model.WorkflowModel; 21 | import com.netflix.conductor.tasks.http.HttpTask; 22 | import com.netflix.conductor.tasks.http.providers.RestTemplateProvider; 23 | 24 | import com.fasterxml.jackson.databind.ObjectMapper; 25 | import lombok.extern.slf4j.Slf4j; 26 | 27 | import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_HTTP; 28 | 29 | @Slf4j 30 | @Component(TASK_TYPE_HTTP) 31 | public class HttpSync extends WorkflowSystemTask { 32 | 33 | private HttpTask httpTask; 34 | 35 | public HttpSync(RestTemplateProvider restTemplateProvider, ObjectMapper objectMapper) { 36 | super(TASK_TYPE_HTTP); 37 | httpTask = new HttpTask(restTemplateProvider, objectMapper); 38 | log.info("Using {}", restTemplateProvider); 39 | } 40 | 41 | @SuppressWarnings("unchecked") 42 | @Override 43 | public boolean isAsync() { 44 | return false; 45 | } 46 | 47 | @Override 48 | public boolean execute(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { 49 | httpTask.execute(workflow, task, executor); 50 | return true; 51 | } 52 | 53 | public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor executor) { 54 | try { 55 | httpTask.start(workflow, task, executor); 56 | } catch (Exception e) { 57 | log.error(e.getMessage(), e); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/execution/tasks/OrkesRestTemplateProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.execution.tasks; 14 | 15 | import java.time.Duration; 16 | import java.util.List; 17 | import java.util.Optional; 18 | 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.beans.factory.annotation.Value; 21 | import org.springframework.boot.web.client.RestTemplateBuilder; 22 | import org.springframework.context.annotation.Primary; 23 | import org.springframework.http.client.ClientHttpRequestInterceptor; 24 | import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; 25 | import org.springframework.stereotype.Component; 26 | import org.springframework.web.client.RestTemplate; 27 | 28 | import com.netflix.conductor.tasks.http.HttpTask; 29 | import com.netflix.conductor.tasks.http.providers.RestTemplateProvider; 30 | 31 | import lombok.AllArgsConstructor; 32 | 33 | /** 34 | * Provider for a customized RestTemplateBuilder. This class provides a default {@link 35 | * RestTemplateBuilder} which can be configured or extended as needed. 36 | */ 37 | @Component 38 | @Primary 39 | public class OrkesRestTemplateProvider implements RestTemplateProvider { 40 | 41 | @AllArgsConstructor 42 | private static class RestTemplateHolder { 43 | RestTemplate restTemplate; 44 | HttpComponentsClientHttpRequestFactory requestFactory; 45 | int readTimeout; 46 | int connectTimeout; 47 | } 48 | 49 | private final ThreadLocal threadLocalRestTemplate; 50 | private final int defaultReadTimeout; 51 | private final int defaultConnectTimeout; 52 | 53 | @Autowired 54 | public OrkesRestTemplateProvider( 55 | @Value("${conductor.tasks.http.readTimeout:250ms}") Duration readTimeout, 56 | @Value("${conductor.tasks.http.connectTimeout:250ms}") Duration connectTimeout, 57 | Optional interceptor) { 58 | this.defaultReadTimeout = (int) readTimeout.toMillis(); 59 | this.defaultConnectTimeout = (int) connectTimeout.toMillis(); 60 | this.threadLocalRestTemplate = 61 | ThreadLocal.withInitial( 62 | () -> { 63 | RestTemplate restTemplate = new RestTemplate(); 64 | HttpComponentsClientHttpRequestFactory requestFactory = 65 | new HttpComponentsClientHttpRequestFactory(); 66 | requestFactory.setReadTimeout(defaultReadTimeout); 67 | requestFactory.setConnectTimeout(defaultConnectTimeout); 68 | restTemplate.setRequestFactory(requestFactory); 69 | interceptor.ifPresent(it -> addInterceptor(restTemplate, it)); 70 | return new RestTemplateHolder( 71 | restTemplate, 72 | requestFactory, 73 | defaultReadTimeout, 74 | defaultConnectTimeout); 75 | }); 76 | } 77 | 78 | @Override 79 | public RestTemplate getRestTemplate(HttpTask.Input input) { 80 | RestTemplateHolder holder = threadLocalRestTemplate.get(); 81 | RestTemplate restTemplate = holder.restTemplate; 82 | HttpComponentsClientHttpRequestFactory requestFactory = holder.requestFactory; 83 | 84 | int newReadTimeout = Optional.ofNullable(input.getReadTimeOut()).orElse(defaultReadTimeout); 85 | int newConnectTimeout = 86 | Optional.ofNullable(input.getConnectionTimeOut()).orElse(defaultConnectTimeout); 87 | if (newReadTimeout != holder.readTimeout || newConnectTimeout != holder.connectTimeout) { 88 | holder.readTimeout = newReadTimeout; 89 | holder.connectTimeout = newConnectTimeout; 90 | requestFactory.setReadTimeout(newReadTimeout); 91 | requestFactory.setConnectTimeout(newConnectTimeout); 92 | } 93 | 94 | return restTemplate; 95 | } 96 | 97 | private void addInterceptor( 98 | RestTemplate restTemplate, ClientHttpRequestInterceptor interceptor) { 99 | List interceptors = restTemplate.getInterceptors(); 100 | if (!interceptors.contains(interceptor)) { 101 | interceptors.add(interceptor); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/execution/tasks/SubWorkflowSync.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.execution.tasks; 14 | 15 | import org.springframework.stereotype.Component; 16 | 17 | import com.netflix.conductor.core.execution.WorkflowExecutor; 18 | import com.netflix.conductor.core.execution.tasks.SubWorkflow; 19 | import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; 20 | import com.netflix.conductor.core.operation.StartWorkflowOperation; 21 | import com.netflix.conductor.model.TaskModel; 22 | import com.netflix.conductor.model.WorkflowModel; 23 | 24 | import com.fasterxml.jackson.databind.ObjectMapper; 25 | import lombok.extern.slf4j.Slf4j; 26 | 27 | import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_SUB_WORKFLOW; 28 | 29 | @Component(TASK_TYPE_SUB_WORKFLOW) 30 | @Slf4j 31 | public class SubWorkflowSync extends WorkflowSystemTask { 32 | 33 | private final SubWorkflow subWorkflow; 34 | 35 | public SubWorkflowSync( 36 | ObjectMapper objectMapper, StartWorkflowOperation startWorkflowOperation) { 37 | super(TASK_TYPE_SUB_WORKFLOW); 38 | this.subWorkflow = new SubWorkflow(objectMapper, startWorkflowOperation); 39 | } 40 | 41 | @Override 42 | public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { 43 | subWorkflow.start(workflow, task, workflowExecutor); 44 | } 45 | 46 | @Override 47 | public boolean execute( 48 | WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { 49 | return subWorkflow.execute(workflow, task, workflowExecutor); 50 | } 51 | 52 | @Override 53 | public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { 54 | subWorkflow.cancel(workflow, task, workflowExecutor); 55 | } 56 | 57 | @Override 58 | public boolean isAsync() { 59 | return false; 60 | } 61 | 62 | @Override 63 | public boolean isAsyncComplete(TaskModel task) { 64 | return true; 65 | } 66 | 67 | @Override 68 | public String toString() { 69 | return subWorkflow.toString(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/rest/VersionResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.rest; 14 | 15 | import org.springframework.web.bind.annotation.GetMapping; 16 | import org.springframework.web.bind.annotation.RestController; 17 | 18 | import io.swagger.v3.oas.annotations.Operation; 19 | import lombok.RequiredArgsConstructor; 20 | 21 | import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE; 22 | 23 | @RestController 24 | @RequiredArgsConstructor 25 | public class VersionResource { 26 | 27 | @GetMapping(value = "/api/version", produces = TEXT_PLAIN_VALUE) 28 | @Operation(summary = "Get the server's version") 29 | public String getVersion() { 30 | String version = getClass().getPackage().getImplementationVersion(); 31 | return version == null ? "n/a" : version; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/rest/WorkflowResourceSync.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.rest; 14 | 15 | import java.util.ArrayList; 16 | import java.util.concurrent.*; 17 | 18 | import javax.annotation.PostConstruct; 19 | 20 | import org.springframework.web.bind.annotation.*; 21 | 22 | import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; 23 | import com.netflix.conductor.common.run.Workflow; 24 | import com.netflix.conductor.service.WorkflowService; 25 | 26 | import io.orkes.conductor.common.model.WorkflowRun; 27 | 28 | import com.google.common.util.concurrent.Uninterruptibles; 29 | import io.swagger.v3.oas.annotations.Operation; 30 | import lombok.RequiredArgsConstructor; 31 | import lombok.SneakyThrows; 32 | import lombok.extern.slf4j.Slf4j; 33 | 34 | import static com.netflix.conductor.rest.config.RequestMappingConstants.WORKFLOW; 35 | 36 | import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; 37 | 38 | @RestController 39 | @RequestMapping(WORKFLOW) 40 | @Slf4j 41 | @RequiredArgsConstructor 42 | public class WorkflowResourceSync { 43 | 44 | public static final String REQUEST_ID_KEY = "_X-Request-Id"; 45 | 46 | private final WorkflowService workflowService; 47 | 48 | private final ScheduledExecutorService executionMonitor = 49 | Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() * 2); 50 | 51 | @PostConstruct 52 | public void startMonitor() { 53 | log.info("Starting execution monitors"); 54 | } 55 | 56 | @PostMapping(value = "execute/{name}/{version}", produces = APPLICATION_JSON_VALUE) 57 | @Operation(summary = "Execute a workflow synchronously", tags = "workflow-resource") 58 | @SneakyThrows 59 | public WorkflowRun executeWorkflow( 60 | @PathVariable("name") String name, 61 | @PathVariable(value = "version", required = false) Integer version, 62 | @RequestParam(value = "requestId", required = true) String requestId, 63 | @RequestParam(value = "waitUntilTaskRef", required = false) String waitUntilTaskRef, 64 | @RequestBody StartWorkflowRequest request) { 65 | 66 | request.setName(name); 67 | request.setVersion(version); 68 | String workflowId = workflowService.startWorkflow(request); 69 | request.getInput().put(REQUEST_ID_KEY, requestId); 70 | Workflow workflow = workflowService.getExecutionStatus(workflowId, true); 71 | if (workflow.getStatus().isTerminal() 72 | || workflow.getTasks().stream() 73 | .anyMatch( 74 | t -> t.getReferenceTaskName().equalsIgnoreCase(waitUntilTaskRef))) { 75 | return toWorkflowRun(workflow); 76 | } 77 | int maxTimeInMilis = 5_000; // 5 sec 78 | int sleepTime = 100; // millis 79 | int loopCount = maxTimeInMilis / sleepTime; 80 | for (int i = 0; i < loopCount; i++) { 81 | Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS); 82 | workflow = workflowService.getExecutionStatus(workflowId, true); 83 | if (workflow.getStatus().isTerminal() 84 | || workflow.getTasks().stream() 85 | .anyMatch( 86 | t -> 87 | t.getReferenceTaskName() 88 | .equalsIgnoreCase(waitUntilTaskRef))) { 89 | return toWorkflowRun(workflow); 90 | } 91 | } 92 | workflow = workflowService.getExecutionStatus(workflowId, true); 93 | return toWorkflowRun(workflow); 94 | } 95 | 96 | public static WorkflowRun toWorkflowRun(Workflow workflow) { 97 | WorkflowRun run = new WorkflowRun(); 98 | 99 | run.setWorkflowId(workflow.getWorkflowId()); 100 | run.setRequestId((String) workflow.getInput().get(REQUEST_ID_KEY)); 101 | run.setCorrelationId(workflow.getCorrelationId()); 102 | run.setInput(workflow.getInput()); 103 | run.setCreatedBy(workflow.getCreatedBy()); 104 | run.setCreateTime(workflow.getCreateTime()); 105 | run.setOutput(workflow.getOutput()); 106 | run.setTasks(new ArrayList<>()); 107 | workflow.getTasks().forEach(task -> run.getTasks().add(task)); 108 | run.setPriority(workflow.getPriority()); 109 | if (workflow.getUpdateTime() != null) { 110 | run.setUpdateTime(workflow.getUpdateTime()); 111 | } 112 | run.setStatus(Workflow.WorkflowStatus.valueOf(workflow.getStatus().name())); 113 | run.setVariables(workflow.getVariables()); 114 | 115 | return run; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/server/service/OrkesSweeperProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.server.service; 14 | 15 | import org.springframework.boot.context.properties.ConfigurationProperties; 16 | import org.springframework.context.annotation.Configuration; 17 | 18 | import lombok.Getter; 19 | import lombok.Setter; 20 | import lombok.ToString; 21 | 22 | @Configuration 23 | @ConfigurationProperties("conductor.orkes.sweeper") 24 | @Getter 25 | @Setter 26 | @ToString 27 | public class OrkesSweeperProperties { 28 | private int frequencyMillis = 10; 29 | private int sweepBatchSize = 5; 30 | private int queuePopTimeout = 100; 31 | } 32 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/server/service/OrkesWorkflowSweeper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.server.service; 14 | 15 | import java.util.List; 16 | import java.util.concurrent.CompletableFuture; 17 | 18 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 19 | import org.springframework.scheduling.annotation.EnableScheduling; 20 | import org.springframework.scheduling.annotation.Scheduled; 21 | import org.springframework.stereotype.Component; 22 | 23 | import com.netflix.conductor.core.LifecycleAwareComponent; 24 | import com.netflix.conductor.core.config.ConductorProperties; 25 | import com.netflix.conductor.dao.QueueDAO; 26 | import com.netflix.conductor.metrics.Monitors; 27 | 28 | import lombok.extern.slf4j.Slf4j; 29 | 30 | import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; 31 | 32 | @Component 33 | @EnableScheduling 34 | @ConditionalOnProperty(name = "conductor.orkes.sweeper.enabled", havingValue = "true") 35 | @Slf4j 36 | public class OrkesWorkflowSweeper extends LifecycleAwareComponent { 37 | 38 | private final QueueDAO queueDAO; 39 | private final OrkesSweeperProperties sweeperProperties; 40 | private final OrkesWorkflowSweepWorker sweepWorker; 41 | 42 | public OrkesWorkflowSweeper( 43 | OrkesWorkflowSweepWorker sweepWorker, 44 | QueueDAO queueDAO, 45 | ConductorProperties properties, 46 | OrkesSweeperProperties sweeperProperties) { 47 | this.sweepWorker = sweepWorker; 48 | this.queueDAO = queueDAO; 49 | this.sweeperProperties = sweeperProperties; 50 | log.info("Initializing sweeper with {} threads", properties.getSweeperThreadCount()); 51 | } 52 | 53 | // Reuse com.netflix.conductor.core.config.SchedulerConfiguration 54 | @Scheduled( 55 | fixedDelayString = "${conductor.orkes.sweeper.frequencyMillis:10}", 56 | initialDelayString = "${conductor.orkes.sweeper.frequencyMillis:10}") 57 | public void pollAndSweep() { 58 | try { 59 | if (!isRunning()) { 60 | log.trace("Component stopped, skip workflow sweep"); 61 | } else { 62 | List workflowIds = 63 | queueDAO.pop( 64 | DECIDER_QUEUE, 65 | sweeperProperties.getSweepBatchSize(), 66 | sweeperProperties.getQueuePopTimeout()); 67 | if (workflowIds != null && workflowIds.size() > 0) { 68 | // wait for all workflow ids to be "swept" 69 | CompletableFuture.allOf( 70 | workflowIds.stream() 71 | .map(sweepWorker::sweepAsync) 72 | .toArray(CompletableFuture[]::new)) 73 | .get(); 74 | log.debug( 75 | "Sweeper processed {} workflow from the decider queue, workflowIds: {}", 76 | workflowIds.size(), 77 | workflowIds); 78 | } 79 | // NOTE: Disabling the sweeper implicitly disables this metric. 80 | recordQueueDepth(); 81 | } 82 | } catch (Exception e) { 83 | Monitors.error(OrkesWorkflowSweeper.class.getSimpleName(), "poll"); 84 | log.error("Error when polling for workflows", e); 85 | if (e instanceof InterruptedException) { 86 | // Restore interrupted state... 87 | Thread.currentThread().interrupt(); 88 | } 89 | } 90 | } 91 | 92 | private void recordQueueDepth() { 93 | int currentQueueSize = queueDAO.getSize(DECIDER_QUEUE); 94 | Monitors.recordGauge(DECIDER_QUEUE, currentQueueSize); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /server/src/main/java/io/orkes/conductor/ui/UIContextGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.ui; 14 | 15 | import java.io.File; 16 | import java.io.FileWriter; 17 | import java.io.IOException; 18 | import java.nio.file.Files; 19 | import java.nio.file.Paths; 20 | import java.util.Map; 21 | 22 | import org.springframework.beans.factory.annotation.Value; 23 | import org.springframework.core.env.Environment; 24 | import org.springframework.stereotype.Component; 25 | 26 | import com.fasterxml.jackson.core.JsonProcessingException; 27 | import com.fasterxml.jackson.databind.ObjectMapper; 28 | import com.fasterxml.jackson.databind.ObjectWriter; 29 | import lombok.extern.slf4j.Slf4j; 30 | 31 | @Component 32 | @Slf4j 33 | public class UIContextGenerator { 34 | 35 | private final Environment environment; 36 | private final ObjectMapper objectMapper; 37 | private final boolean isSecurityEnabled; 38 | private final String contextFilePath; 39 | 40 | public UIContextGenerator( 41 | Environment environment, 42 | ObjectMapper objectMapper, 43 | @Value("${conductor.ui.context.file:/usr/share/nginx/html/context.js}") 44 | String contextFilePath) { 45 | this.environment = environment; 46 | this.objectMapper = objectMapper; 47 | this.isSecurityEnabled = false; 48 | this.contextFilePath = contextFilePath; 49 | } 50 | 51 | public boolean writeUIContextFile() throws IOException { 52 | log.info("Writing UI Context file: {}", contextFilePath); 53 | File contextFile = new File(contextFilePath); 54 | if (Files.notExists(Paths.get(contextFile.toURI()))) { 55 | log.info("UI Context File doesn't exist, skipping"); 56 | return false; 57 | } 58 | 59 | try (FileWriter fileWriter = new FileWriter(contextFile)) { 60 | ObjectWriter jsonWriter = objectMapper.writerWithDefaultPrettyPrinter(); 61 | fileWriter.write( 62 | String.format( 63 | "window.conductor = %s;\n\n", conductorFeaturesAsJson(jsonWriter))); 64 | fileWriter.write( 65 | String.format( 66 | "\nwindow.auth0Identifiers = %s;\n", 67 | jsonWriter.writeValueAsString( 68 | Map.of( 69 | "clientId", "NOT_ENABLED", 70 | "domain", "NOT_ENABLED")))); 71 | 72 | fileWriter.flush(); 73 | } 74 | 75 | return true; 76 | } 77 | 78 | private String conductorFeaturesAsJson(ObjectWriter jsonWriter) throws JsonProcessingException { 79 | return jsonWriter.writeValueAsString( 80 | Map.of( 81 | "TASK_VISIBILITY", 82 | environment.getProperty( 83 | "conductor.security.default.taskVisibility", "READ"), 84 | "ACCESS_MANAGEMENT", isSecurityEnabled, 85 | "CREATOR_ENABLE_CREATOR", 86 | environment.getProperty( 87 | "conductor.creatorUi.featuresEnabled", "false"), 88 | "CREATOR_ENABLE_REAFLOW_DIAGRAM", 89 | environment.getProperty( 90 | "conductor.creatorUi.reaflowDiagramEnabled", "false"))); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /server/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | conductor.app.workflow-execution-lock-enabled=true 3 | conductor.workflow-execution-lock.type=redis 4 | conductor.redis-lock.serverAddress=redis://localhost:6379 5 | #in millisecond the amount of time to wait to obtain the lock on workflow 6 | conductor.app.lockTimeToTry=50 7 | 8 | conductor.db.type=redis_standalone 9 | conductor.queue.type=redis_standalone 10 | conductor.id.generator=time_based 11 | 12 | conductor.redis.hosts=localhost:6379:us-east-1c 13 | 14 | #Misc conductor server configuration 15 | conductor.default-event-queue.type=sqs 16 | conductor.metrics-logger.enabled=false 17 | conductor.app.owner-email-mandatory=false 18 | 19 | #Redis Properties 20 | conductor.redis.queueNamespacePrefix=conductor_queues 21 | conductor.redis.workflowNamespacePrefix=conductor 22 | conductor.redis.taskDefCacheRefreshInterval=1 23 | 24 | #Workflow archival and indexing 25 | conductor.archive.db.enabled=true 26 | conductor.archive.db.type=postgres 27 | conductor.archive.db.indexer.threadCount=4 28 | conductor.archive.db.indexer.pollingInterval=10 29 | 30 | #postgres database 31 | spring.datasource.url=jdbc:postgresql://localhost:5432/postgres 32 | spring.datasource.username=postgres 33 | spring.datasource.password=postgres 34 | 35 | #JDBC datasource configuration 36 | spring.datasource.hikari.connection-init-sql=SET statement_timeout = '30s' 37 | spring.datasource.hikari.maximum-pool-size=8 38 | spring.datasource.hikari.auto-commit=true 39 | spring.search-datasource.hikari.maximum-pool-size=8 40 | spring.search-datasource.hikari.auto-commit=true 41 | 42 | #System Task Workers 43 | conductor.app.systemTaskWorkerPollInterval=1 44 | conductor.app.systemTaskMaxPollCount=10 45 | conductor.app.systemTaskWorkerThreadCount=10 46 | 47 | #Background sweeper job 48 | #Disable default 49 | conductor.workflow-reconciler.enabled=false 50 | conductor.workflow-repair-service.enabled=false 51 | #Enable the Orkes version 52 | conductor.orkes.sweeper.enabled=true 53 | conductor.orkes.sweeper.frequencyMillis=100 54 | conductor.orkes.sweeper.sweepBatchSize=10 55 | conductor.orkes.sweeper.queuePopTimeout=100 56 | #shares the same sweeper executor 57 | conductor.app.sweeperThreadCount=16 58 | 59 | #Monitor 60 | conductor.workflow-monitor.enabled=true 61 | 62 | #metrics -- only enable what is necessary 63 | management.endpoints.web.exposure.include=prometheus,health 64 | management.metrics.web.server.request.autotime.percentiles=0.50,0.75,0.90,0.95,0.99 65 | 66 | # MAX Payload configuration 67 | conductor.app.maxTaskOutputPayloadSizeThreshold=102400 68 | conductor.app.maxTaskInputPayloadSizeThreshold=102400 69 | conductor.app.taskOutputPayloadSizeThreshold=102400 70 | conductor.app.taskInputPayloadSizeThreshold=102400 71 | 72 | # Additional modules for metrics collection exposed to Datadog (optional) 73 | management.metrics.export.datadog.enabled=${conductor.metrics-datadog.enabled:false} 74 | management.metrics.export.datadog.api-key=${conductor.metrics-datadog.api-key:} 75 | 76 | #Swagger - OpenAPI configuration 77 | springdoc.swagger-ui.tagsSorter=alpha 78 | springdoc.swagger-ui.operationsSorter=alpha 79 | springdoc.writer-with-order-by-keys=true 80 | springdoc.api-docs.path=/api-docs 81 | springdoc.swagger-ui.disable-swagger-default-url=true 82 | springdoc.swagger-ui.queryConfigEnabled=false 83 | springdoc.swagger-ui.filter=true 84 | 85 | conductor.swagger.url=/ 86 | 87 | -------------------------------------------------------------------------------- /server/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | 2 | ______ .______ __ ___ _______ _______. 3 | / __ \ | _ \ | |/ / | ____| / | 4 | | | | | | |_) | | ' / | |__ | (----` 5 | | | | | | / | < | __| \ \ 6 | | `--' | | |\ \----.| . \ | |____.----) | 7 | \______/ | _| `._____||__|\__\ |_______|_______/ 8 | 9 | ______ ______ .__ __. _______ __ __ ______ .___________. ______ .______ 10 | / | / __ \ | \ | | | \ | | | | / || | / __ \ | _ \ 11 | | ,----'| | | | | \| | | .--. || | | | | ,----'`---| |----`| | | | | |_) | 12 | | | | | | | | . ` | | | | || | | | | | | | | | | | | / 13 | | `----.| `--' | | |\ | | '--' || `--' | | `----. | | | `--' | | |\ \----. 14 | \______| \______/ |__| \__| |_______/ \______/ \______| |__| \______/ | _| `._____| 15 | 16 | Licensed under Orkes Conductor Community License 17 | -------------------------------------------------------------------------------- /server/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable 9 | 10 | 11 | 12 | 13 | 14 | 16 | ${LOG_FILE} 17 | 19 | 20 | ${LOG_FILE}.%d{yyyy-MM-dd}.%i 21 | 22 | 200MB 23 | 10 24 | 10GB 25 | 26 | 27 | %d{ISO8601} %-5level [%t]: %msg%n%throwable 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /server/src/main/resources/tasks.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "createTime": 1662241150448, 4 | "createdBy": "", 5 | "name": "simple_task_5", 6 | "description": "Edit or extend this sample task. Set the task name to get started", 7 | "retryCount": 3, 8 | "timeoutSeconds": 3600, 9 | "inputKeys": [], 10 | "outputKeys": [], 11 | "timeoutPolicy": "TIME_OUT_WF", 12 | "retryLogic": "FIXED", 13 | "retryDelaySeconds": 60, 14 | "responseTimeoutSeconds": 600, 15 | "inputTemplate": {}, 16 | "rateLimitPerFrequency": 0, 17 | "rateLimitFrequencyInSeconds": 1, 18 | "ownerEmail": "viren@orkes.io", 19 | "backoffScaleFactor": 1 20 | }, 21 | { 22 | "createTime": 1662241156390, 23 | "createdBy": "", 24 | "name": "simple_task_6", 25 | "description": "Edit or extend this sample task. Set the task name to get started", 26 | "retryCount": 3, 27 | "timeoutSeconds": 3600, 28 | "inputKeys": [], 29 | "outputKeys": [], 30 | "timeoutPolicy": "TIME_OUT_WF", 31 | "retryLogic": "FIXED", 32 | "retryDelaySeconds": 60, 33 | "responseTimeoutSeconds": 600, 34 | "inputTemplate": {}, 35 | "rateLimitPerFrequency": 0, 36 | "rateLimitFrequencyInSeconds": 1, 37 | "ownerEmail": "viren@orkes.io", 38 | "backoffScaleFactor": 1 39 | }, 40 | { 41 | "createTime": 1662241120549, 42 | "createdBy": "", 43 | "name": "simple_task_3", 44 | "description": "Edit or extend this sample task. Set the task name to get started", 45 | "retryCount": 3, 46 | "timeoutSeconds": 3600, 47 | "inputKeys": [], 48 | "outputKeys": [], 49 | "timeoutPolicy": "TIME_OUT_WF", 50 | "retryLogic": "FIXED", 51 | "retryDelaySeconds": 60, 52 | "responseTimeoutSeconds": 600, 53 | "inputTemplate": {}, 54 | "rateLimitPerFrequency": 0, 55 | "rateLimitFrequencyInSeconds": 1, 56 | "ownerEmail": "viren@orkes.io", 57 | "backoffScaleFactor": 1 58 | }, 59 | { 60 | "createTime": 1662241125214, 61 | "createdBy": "", 62 | "name": "simple_task_4", 63 | "description": "Edit or extend this sample task. Set the task name to get started", 64 | "retryCount": 3, 65 | "timeoutSeconds": 3600, 66 | "inputKeys": [], 67 | "outputKeys": [], 68 | "timeoutPolicy": "TIME_OUT_WF", 69 | "retryLogic": "FIXED", 70 | "retryDelaySeconds": 60, 71 | "responseTimeoutSeconds": 600, 72 | "inputTemplate": {}, 73 | "rateLimitPerFrequency": 0, 74 | "rateLimitFrequencyInSeconds": 1, 75 | "ownerEmail": "viren@orkes.io", 76 | "backoffScaleFactor": 1 77 | }, 78 | { 79 | "createTime": 1662241109699, 80 | "createdBy": "", 81 | "name": "simple_task_1", 82 | "description": "Edit or extend this sample task. Set the task name to get started", 83 | "retryCount": 3, 84 | "timeoutSeconds": 3600, 85 | "inputKeys": [], 86 | "outputKeys": [], 87 | "timeoutPolicy": "TIME_OUT_WF", 88 | "retryLogic": "FIXED", 89 | "retryDelaySeconds": 60, 90 | "responseTimeoutSeconds": 600, 91 | "inputTemplate": {}, 92 | "rateLimitPerFrequency": 0, 93 | "rateLimitFrequencyInSeconds": 1, 94 | "ownerEmail": "viren@orkes.io", 95 | "backoffScaleFactor": 1 96 | }, 97 | { 98 | "createTime": 1662241116155, 99 | "createdBy": "", 100 | "name": "simple_task_2", 101 | "description": "Edit or extend this sample task. Set the task name to get started", 102 | "retryCount": 3, 103 | "timeoutSeconds": 3600, 104 | "inputKeys": [], 105 | "outputKeys": [], 106 | "timeoutPolicy": "TIME_OUT_WF", 107 | "retryLogic": "FIXED", 108 | "retryDelaySeconds": 60, 109 | "responseTimeoutSeconds": 600, 110 | "inputTemplate": {}, 111 | "rateLimitPerFrequency": 0, 112 | "rateLimitFrequencyInSeconds": 1, 113 | "ownerEmail": "viren@orkes.io", 114 | "backoffScaleFactor": 1 115 | } 116 | ] -------------------------------------------------------------------------------- /server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'orkes-conductor' 2 | include 'archive' 3 | include 'persistence' 4 | include 'server' 5 | include 'test-harness' 6 | 7 | rootProject.children.each { it.name = "${rootProject.name}-${it.name}" } -------------------------------------------------------------------------------- /test-harness/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | dependencies { 3 | 4 | 5 | 6 | implementation "org.awaitility:awaitility:3.1.6" 7 | implementation "org.testcontainers:postgresql:${versions.revTestContainer}" 8 | implementation "org.testcontainers:testcontainers:${versions.revTestContainer}" 9 | implementation 'ch.qos.logback:logback-classic:1.4.5' 10 | 11 | 12 | 13 | implementation 'io.orkes.conductor:orkes-conductor-client:1.1.19' 14 | implementation 'javax.annotation:javax.annotation-api:1.3.2' 15 | 16 | implementation "com.google.protobuf:protobuf-java:${versions.revProtoBuf}" 17 | implementation "io.netty:netty-tcnative-boringssl-static:2.0.51.Final" 18 | implementation "com.google.guava:guava:${versions.revGuava}" 19 | 20 | implementation "org.apache.commons:commons-lang3:${versions.revCommonsLang}" 21 | 22 | testImplementation 'org.testcontainers:testcontainers:1.17.1' 23 | testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.0' 24 | testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.0' 25 | } 26 | 27 | test { 28 | useJUnitPlatform() 29 | testLogging { 30 | events = ["SKIPPED", "FAILED"] 31 | exceptionFormat = "short" 32 | showStandardStreams = true 33 | } 34 | minHeapSize = "2g" // initial heap size 35 | maxHeapSize = "4g" // maximum heap size 36 | } 37 | 38 | tasks.withType(Test) { 39 | maxParallelForks = 10 40 | } 41 | 42 | tasks.forEach(task -> task.onlyIf { project.hasProperty('IntegrationTests') }) -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/AbstractConductorTest.java: -------------------------------------------------------------------------------- 1 | package io.orkes.conductor.client.e2e; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.netflix.conductor.common.config.ObjectMapperProvider; 5 | import com.netflix.conductor.common.metadata.workflow.WorkflowDef; 6 | import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; 7 | import io.orkes.conductor.client.ApiClient; 8 | import io.orkes.conductor.client.http.OrkesMetadataClient; 9 | import io.orkes.conductor.client.http.OrkesTaskClient; 10 | import io.orkes.conductor.client.http.OrkesWorkflowClient; 11 | import lombok.SneakyThrows; 12 | import lombok.extern.slf4j.Slf4j; 13 | import org.junit.jupiter.api.AfterAll; 14 | import org.junit.jupiter.api.BeforeAll; 15 | import org.testcontainers.containers.GenericContainer; 16 | 17 | import java.io.InputStream; 18 | import java.io.InputStreamReader; 19 | import java.util.List; 20 | 21 | @Slf4j 22 | public abstract class AbstractConductorTest { 23 | 24 | protected static final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper(); 25 | 26 | protected static GenericContainer conductor; 27 | 28 | protected static ApiClient apiClient; 29 | protected static io.orkes.conductor.client.WorkflowClient workflowClient; 30 | protected static io.orkes.conductor.client.TaskClient taskClient; 31 | protected static io.orkes.conductor.client.MetadataClient metadataClient; 32 | protected static WorkflowExecutor executor; 33 | 34 | protected static final String[] workflows = new String[]{"/metadata/rerun.json", "/metadata/popminmax.json", "/metadata/fail.json"}; 35 | 36 | @SneakyThrows 37 | @BeforeAll 38 | public static final void setup() { 39 | String url = "http://localhost:8899/api"; 40 | 41 | apiClient = new ApiClient(url); 42 | workflowClient = new OrkesWorkflowClient(apiClient); 43 | metadataClient = new OrkesMetadataClient(apiClient); 44 | taskClient = new OrkesTaskClient(apiClient); 45 | executor = new WorkflowExecutor(taskClient, workflowClient, metadataClient, 1000); 46 | 47 | for (String workflow : workflows) { 48 | InputStream resource = AbstractConductorTest.class.getResourceAsStream(workflow); 49 | WorkflowDef workflowDef = objectMapper.readValue(new InputStreamReader(resource), WorkflowDef.class); 50 | metadataClient.updateWorkflowDefs(List.of(workflowDef), true); 51 | } 52 | } 53 | 54 | @AfterAll 55 | public static void cleanup() { 56 | executor.shutdown(); 57 | } 58 | 59 | 60 | 61 | 62 | 63 | } 64 | 65 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/ConductorStartupTest.java: -------------------------------------------------------------------------------- 1 | package io.orkes.conductor.client.e2e; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | public class ConductorStartupTest extends AbstractConductorTest { 6 | 7 | 8 | @Test 9 | public void test() { 10 | 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/GraaljsTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | import java.util.Map; 18 | import java.util.concurrent.ExecutionException; 19 | import java.util.concurrent.TimeUnit; 20 | import java.util.concurrent.TimeoutException; 21 | 22 | import org.apache.commons.lang3.RandomStringUtils; 23 | import org.junit.After; 24 | import org.junit.jupiter.api.Test; 25 | 26 | import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; 27 | import com.netflix.conductor.common.metadata.workflow.WorkflowDef; 28 | import com.netflix.conductor.common.run.Workflow; 29 | 30 | import io.orkes.conductor.client.model.WorkflowStatus; 31 | 32 | import static io.orkes.conductor.client.e2e.util.RegistrationUtil.registerWorkflowDef; 33 | import static org.junit.jupiter.api.Assertions.assertEquals; 34 | import static org.testcontainers.shaded.org.awaitility.Awaitility.await; 35 | 36 | public class GraaljsTests extends AbstractConductorTest { 37 | 38 | List workflowNames = new ArrayList<>(); 39 | List taskNames = new ArrayList<>(); 40 | 41 | @Test 42 | public void testInfiniteExecution() 43 | throws ExecutionException, InterruptedException, TimeoutException { 44 | String workflowName = RandomStringUtils.randomAlphanumeric(5).toUpperCase(); 45 | String taskName1 = RandomStringUtils.randomAlphanumeric(5).toUpperCase(); 46 | String taskName2 = RandomStringUtils.randomAlphanumeric(5).toUpperCase(); 47 | // Register workflow 48 | registerWorkflowDef(workflowName, taskName1, taskName2, metadataClient); 49 | WorkflowDef workflowDef = metadataClient.getWorkflowDef(workflowName, 1); 50 | workflowDef 51 | .getTasks() 52 | .get(0) 53 | .setInputParameters( 54 | Map.of( 55 | "evaluatorType", 56 | "graaljs", 57 | "expression", 58 | "function e() { while(true){} }; e();")); 59 | metadataClient.updateWorkflowDefs(List.of(workflowDef), true); 60 | workflowNames.add(workflowName); 61 | taskNames.add(taskName1); 62 | taskNames.add(taskName2); 63 | 64 | StartWorkflowRequest startWorkflowRequest = new StartWorkflowRequest(); 65 | startWorkflowRequest.setName(workflowName); 66 | startWorkflowRequest.setVersion(1); 67 | 68 | String workflowId = workflowClient.startWorkflow(startWorkflowRequest); 69 | 70 | // Wait for workflow to get failed since inline task will failed 71 | await().atMost(30, TimeUnit.SECONDS) 72 | .untilAsserted( 73 | () -> { 74 | Workflow workflow = workflowClient.getWorkflow(workflowId, true); 75 | assertEquals( 76 | workflow.getStatus().name(), 77 | WorkflowStatus.StatusEnum.FAILED.name()); 78 | }); 79 | } 80 | 81 | @After 82 | public void cleanUp() { 83 | for (String workflowName : workflowNames) { 84 | try { 85 | metadataClient.unregisterWorkflowDef(workflowName, 1); 86 | } catch (Exception e) { 87 | } 88 | } 89 | for (String taskName : taskNames) { 90 | try { 91 | metadataClient.unregisterTaskDef(taskName); 92 | } catch (Exception e) { 93 | } 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/JSONJQTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | import java.util.Map; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import org.apache.commons.lang3.RandomStringUtils; 21 | import org.junit.jupiter.api.Test; 22 | 23 | import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; 24 | import com.netflix.conductor.common.metadata.workflow.WorkflowDef; 25 | import com.netflix.conductor.common.metadata.workflow.WorkflowTask; 26 | import com.netflix.conductor.common.run.Workflow; 27 | 28 | import io.orkes.conductor.client.ApiClient; 29 | import io.orkes.conductor.client.WorkflowClient; 30 | import io.orkes.conductor.client.http.OrkesWorkflowClient; 31 | import io.orkes.conductor.client.e2e.util.Commons; 32 | 33 | import com.google.common.util.concurrent.Uninterruptibles; 34 | 35 | import static org.junit.jupiter.api.Assertions.assertEquals; 36 | 37 | public class JSONJQTests extends AbstractConductorTest{ 38 | @Test 39 | public void testJQOutputIsReachableWhenSyncSystemTaskIsNext() { 40 | 41 | String workflowName = RandomStringUtils.randomAlphanumeric(10).toUpperCase(); 42 | 43 | var request = new StartWorkflowRequest(); 44 | WorkflowDef workflowDef = new WorkflowDef(); 45 | workflowDef.setName(workflowName); 46 | workflowDef.setVersion(1); 47 | workflowDef.setOwnerEmail(Commons.OWNER_EMAIL); 48 | workflowDef.setTimeoutSeconds(60); 49 | workflowDef.setTimeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF); 50 | List tasks = new ArrayList<>(); 51 | 52 | WorkflowTask jqTask = new WorkflowTask(); 53 | jqTask.setName("jqTaskName"); 54 | jqTask.setTaskReferenceName("generate_operators_ref"); 55 | jqTask.setInputParameters(Map.of("queryExpression", "{\"as\": \"+\", \"md\": \"/\"}")); 56 | jqTask.setType("JSON_JQ_TRANSFORM"); 57 | 58 | WorkflowTask setVariableTask = new WorkflowTask(); 59 | setVariableTask.setName("setvartaskname"); 60 | setVariableTask.setTaskReferenceName("setvartaskname_ref"); 61 | setVariableTask.setInputParameters( 62 | Map.of("name", "${generate_operators_ref.output.result.md}")); 63 | setVariableTask.setType("SET_VARIABLE"); 64 | 65 | tasks.add(jqTask); 66 | tasks.add(setVariableTask); 67 | workflowDef.setTasks(tasks); 68 | request.setName(workflowName); 69 | request.setVersion(1); 70 | request.setWorkflowDef(workflowDef); 71 | 72 | List workflowIds = new ArrayList<>(); 73 | for (var i = 0; i < 40; ++i) { 74 | Uninterruptibles.sleepUninterruptibly(5, TimeUnit.MILLISECONDS); 75 | workflowIds.add(workflowClient.startWorkflow(request)); 76 | } 77 | assertEquals(40, workflowIds.size()); 78 | workflowIds.forEach( 79 | id -> { 80 | var workflow = workflowClient.getWorkflow(id, true); 81 | assertEquals(Workflow.WorkflowStatus.COMPLETED, workflow.getStatus()); 82 | assertEquals("/", workflow.getTasks().get(1).getInputData().get("name")); 83 | }); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/JavaSDKTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e; 14 | 15 | import java.math.BigDecimal; 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | import java.util.concurrent.CompletableFuture; 19 | import java.util.concurrent.ExecutionException; 20 | import java.util.concurrent.TimeUnit; 21 | import java.util.concurrent.TimeoutException; 22 | 23 | import org.junit.jupiter.api.AfterAll; 24 | import org.junit.jupiter.api.BeforeAll; 25 | import org.junit.jupiter.api.Test; 26 | 27 | import com.netflix.conductor.client.http.MetadataClient; 28 | import com.netflix.conductor.client.http.TaskClient; 29 | import com.netflix.conductor.client.http.WorkflowClient; 30 | import com.netflix.conductor.common.run.Workflow; 31 | import com.netflix.conductor.sdk.workflow.def.ConductorWorkflow; 32 | import com.netflix.conductor.sdk.workflow.def.tasks.SimpleTask; 33 | import com.netflix.conductor.sdk.workflow.def.tasks.Switch; 34 | import com.netflix.conductor.sdk.workflow.executor.WorkflowExecutor; 35 | import com.netflix.conductor.sdk.workflow.task.WorkerTask; 36 | 37 | import io.orkes.conductor.client.ApiClient; 38 | import io.orkes.conductor.client.OrkesClients; 39 | 40 | import static org.junit.jupiter.api.Assertions.*; 41 | 42 | public class JavaSDKTests extends AbstractConductorTest { 43 | 44 | @BeforeAll 45 | public static void init() { 46 | executor.initWorkers("io.orkes.conductor.client.e2e"); 47 | } 48 | 49 | @Test 50 | public void testSDK() throws ExecutionException, InterruptedException, TimeoutException { 51 | ConductorWorkflow> workflow = new ConductorWorkflow<>(executor); 52 | workflow.setName("sdk_integration_test"); 53 | workflow.setVersion(1); 54 | workflow.setOwnerEmail("test@orkes.io"); 55 | workflow.setVariables(new HashMap<>()); 56 | workflow.add(new SimpleTask("task1", "task1").input("name", "orkes")); 57 | 58 | Switch decision = new Switch("decide_ref", "${workflow.input.caseValue}"); 59 | decision.switchCase( 60 | "caseA", new SimpleTask("task1", "task1"), new SimpleTask("task1", "task11")); 61 | decision.switchCase("caseB", new SimpleTask("task2", "task2")); 62 | decision.defaultCase(new SimpleTask("task1", "default_task")); 63 | 64 | CompletableFuture future = workflow.executeDynamic(new HashMap<>()); 65 | assertNotNull(future); 66 | Workflow run = future.get(20, TimeUnit.SECONDS); 67 | assertNotNull(run); 68 | assertEquals(Workflow.WorkflowStatus.COMPLETED, run.getStatus()); 69 | assertEquals(1, run.getTasks().size()); 70 | assertEquals("Hello, orkes", run.getTasks().get(0).getOutputData().get("greetings")); 71 | } 72 | 73 | @AfterAll 74 | public static void cleanup() { 75 | if (executor != null) { 76 | executor.shutdown(); 77 | } 78 | } 79 | 80 | @WorkerTask("sum_numbers") 81 | public BigDecimal sum(BigDecimal num1, BigDecimal num2) { 82 | return num1.add(num2); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/SDKWorkers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e; 14 | 15 | import com.netflix.conductor.sdk.workflow.task.InputParam; 16 | import com.netflix.conductor.sdk.workflow.task.OutputParam; 17 | import com.netflix.conductor.sdk.workflow.task.WorkerTask; 18 | 19 | public class SDKWorkers { 20 | 21 | @WorkerTask(value = "task1", pollingInterval = 1000) 22 | public @OutputParam("greetings") String task1(@InputParam("name") String name) { 23 | return "Hello, " + name; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/WaitTaskTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e; 14 | 15 | import java.time.Duration; 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | import java.util.concurrent.CompletableFuture; 19 | import java.util.concurrent.ExecutionException; 20 | import java.util.concurrent.TimeUnit; 21 | import java.util.concurrent.TimeoutException; 22 | 23 | import org.junit.jupiter.api.Test; 24 | 25 | import com.netflix.conductor.common.run.Workflow; 26 | import com.netflix.conductor.sdk.workflow.def.ConductorWorkflow; 27 | import com.netflix.conductor.sdk.workflow.def.tasks.Wait; 28 | 29 | import static java.time.temporal.ChronoUnit.SECONDS; 30 | import static org.junit.jupiter.api.Assertions.*; 31 | 32 | public class WaitTaskTest extends AbstractConductorTest { 33 | 34 | 35 | 36 | @Test 37 | public void testWaitTimeout() throws ExecutionException, InterruptedException, TimeoutException { 38 | ConductorWorkflow> workflow = new ConductorWorkflow<>(executor); 39 | workflow.setName("wait_task_test"); 40 | workflow.setVersion(1); 41 | workflow.setVariables(new HashMap<>()); 42 | workflow.add(new Wait("wait_for_2_second", Duration.of(2, SECONDS))); 43 | CompletableFuture future = workflow.executeDynamic(new HashMap<>()); 44 | assertNotNull(future); 45 | Workflow run = future.get(60, TimeUnit.SECONDS); 46 | assertNotNull(run); 47 | assertEquals(Workflow.WorkflowStatus.COMPLETED, run.getStatus()); 48 | assertEquals(1, run.getTasks().size()); 49 | long timeToExecute = run.getTasks().get(0).getEndTime() - run.getTasks().get(0).getScheduledTime(); 50 | System.out.println("Wait task completed in " + timeToExecute + " millis - should have completed in around 2 second mark"); 51 | // Ensure the wait completes within 1sec buffer 52 | assertTrue(timeToExecute < 3000, "Wait task did not complete in time, took " + timeToExecute + " millis"); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/util/Commons.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e.util; 14 | 15 | import com.netflix.conductor.common.metadata.tasks.TaskDef; 16 | import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; 17 | 18 | public class Commons { 19 | public static String WORKFLOW_NAME = "test-sdk-java-workflow"; 20 | public static String TASK_NAME = "test-sdk-java-task"; 21 | public static String OWNER_EMAIL = "example@orkes.io"; 22 | public static int WORKFLOW_VERSION = 1; 23 | 24 | 25 | public static TaskDef getTaskDef() { 26 | TaskDef taskDef = new TaskDef(); 27 | taskDef.setName(Commons.TASK_NAME); 28 | return taskDef; 29 | } 30 | 31 | public static StartWorkflowRequest getStartWorkflowRequest() { 32 | return new StartWorkflowRequest().withName(WORKFLOW_NAME).withVersion(WORKFLOW_VERSION); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/util/SimpleWorker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e.util; 14 | 15 | import com.netflix.conductor.client.worker.Worker; 16 | import com.netflix.conductor.common.metadata.tasks.Task; 17 | import com.netflix.conductor.common.metadata.tasks.TaskResult; 18 | 19 | public class SimpleWorker implements Worker { 20 | @Override 21 | public String getTaskDefName() { 22 | return Commons.TASK_NAME; 23 | } 24 | 25 | @Override 26 | public TaskResult execute(Task task) { 27 | task.setStatus(Task.Status.COMPLETED); 28 | task.getOutputData().put("key", "value"); 29 | task.getOutputData().put("key2", 42); 30 | return new TaskResult(task); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /test-harness/src/test/java/io/orkes/conductor/client/e2e/util/WorkflowUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Orkes, Inc. 3 | *

4 | * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | *

7 | * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt 8 | *

9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package io.orkes.conductor.client.e2e.util; 14 | 15 | import java.util.List; 16 | 17 | import com.netflix.conductor.common.metadata.workflow.WorkflowDef; 18 | import com.netflix.conductor.common.metadata.workflow.WorkflowTask; 19 | 20 | public class WorkflowUtil { 21 | public static WorkflowDef getWorkflowDef() { 22 | WorkflowDef workflowDef = new WorkflowDef(); 23 | workflowDef.setName(Commons.WORKFLOW_NAME); 24 | workflowDef.setVersion(Commons.WORKFLOW_VERSION); 25 | workflowDef.setOwnerEmail(Commons.OWNER_EMAIL); 26 | workflowDef.setTimeoutSeconds(600); 27 | workflowDef.setTimeoutPolicy(WorkflowDef.TimeoutPolicy.TIME_OUT_WF); 28 | WorkflowTask workflowTask = new WorkflowTask(); 29 | workflowTask.setName(Commons.TASK_NAME); 30 | workflowTask.setTaskReferenceName(Commons.TASK_NAME); 31 | workflowDef.setTasks(List.of(workflowTask)); 32 | return workflowDef; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test-harness/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | %black(%d{ISO8601}) %highlight(%-5level): %msg%n%throwable 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /test-harness/src/test/resources/metadata/fail.json: -------------------------------------------------------------------------------- 1 | { 2 | "createTime": 1685744302411, 3 | "updateTime": 1685743597519, 4 | "name": "this_will_fail", 5 | "description": "Edit or extend this sample workflow. Set the workflow name to get started", 6 | "version": 1, 7 | "tasks": [ 8 | { 9 | "name": "http_task_lvome", 10 | "taskReferenceName": "http_task_lvome_ref", 11 | "inputParameters": { 12 | "http_request": { 13 | "uri": "https://orkes-api-tester.orkesconductor.com/api", 14 | "method": "GET", 15 | "connectionTimeOut": 3000, 16 | "readTimeOut": "3000", 17 | "accept": "application/json", 18 | "contentType": "application/json" 19 | } 20 | }, 21 | "type": "HTTP", 22 | "decisionCases": {}, 23 | "defaultCase": [], 24 | "forkTasks": [], 25 | "startDelay": 0, 26 | "joinOn": [], 27 | "optional": false, 28 | "defaultExclusiveJoinTask": [], 29 | "asyncComplete": false, 30 | "loopOver": [], 31 | "onStateChange": {} 32 | }, 33 | { 34 | "name": "set_variable_task_lxzgc", 35 | "taskReferenceName": "set_variable_task_lxzgc_ref", 36 | "inputParameters": { 37 | "name": "Orkes" 38 | }, 39 | "type": "SET_VARIABLE", 40 | "decisionCases": {}, 41 | "defaultCase": [], 42 | "forkTasks": [], 43 | "startDelay": 0, 44 | "joinOn": [], 45 | "optional": false, 46 | "defaultExclusiveJoinTask": [], 47 | "asyncComplete": false, 48 | "loopOver": [], 49 | "onStateChange": {} 50 | }, 51 | { 52 | "name": "json_transform_task_518l3h", 53 | "taskReferenceName": "json_transform_task_518l3h_ref", 54 | "inputParameters": { 55 | "persons": [ 56 | { 57 | "name": "some", 58 | "last": "name", 59 | "email": "mail@mail.com", 60 | "id": 1 61 | }, 62 | { 63 | "name": "some2", 64 | "last": "name2", 65 | "email": "mail2@mail.com", 66 | "id": 2 67 | } 68 | ], 69 | "queryExpression": ".persons | map({user:{email,id}})" 70 | }, 71 | "type": "JSON_JQ_TRANSFORM", 72 | "decisionCases": {}, 73 | "defaultCase": [], 74 | "forkTasks": [], 75 | "startDelay": 0, 76 | "joinOn": [], 77 | "optional": false, 78 | "defaultExclusiveJoinTask": [], 79 | "asyncComplete": false, 80 | "loopOver": [], 81 | "onStateChange": {} 82 | }, 83 | { 84 | "name": "get_random_fact2", 85 | "taskReferenceName": "get_random_fact", 86 | "inputParameters": { 87 | "http_request": { 88 | "uri": "https://orkes-api-tester.orkesconductor.com/dddd", 89 | "method": "GET", 90 | "connectionTimeOut": 3000, 91 | "readTimeOut": 3000, 92 | "accept": "application/json", 93 | "contentType": "application/json" 94 | } 95 | }, 96 | "type": "HTTP", 97 | "decisionCases": {}, 98 | "defaultCase": [], 99 | "forkTasks": [], 100 | "startDelay": 0, 101 | "joinOn": [], 102 | "optional": false, 103 | "defaultExclusiveJoinTask": [], 104 | "asyncComplete": false, 105 | "loopOver": [], 106 | "onStateChange": {} 107 | } 108 | ], 109 | "inputParameters": [], 110 | "outputParameters": { 111 | "data": "${get_random_fact.output.response.body.fact}" 112 | }, 113 | "failureWorkflow": "cat_facts", 114 | "schemaVersion": 2, 115 | "restartable": true, 116 | "workflowStatusListenerEnabled": false, 117 | "ownerEmail": "viren@orkes.io", 118 | "timeoutPolicy": "ALERT_ONLY", 119 | "timeoutSeconds": 0, 120 | "variables": {}, 121 | "inputTemplate": {}, 122 | "onStateChange": {} 123 | } -------------------------------------------------------------------------------- /test-harness/src/test/resources/metadata/popminmax.json: -------------------------------------------------------------------------------- 1 | { 2 | "createTime": 1670136356629, 3 | "updateTime": 1694687962674, 4 | "name": "PopulationMinMax", 5 | "description": "Edit or extend this sample workflow. Set the workflow name to get started", 6 | "version": 1, 7 | "tasks": [ 8 | { 9 | "name": "set_variable_task_jqc56h_ref", 10 | "taskReferenceName": "set_variable_task_jqc56h_ref", 11 | "inputParameters": { 12 | "name": "Orkes" 13 | }, 14 | "type": "SET_VARIABLE", 15 | "decisionCases": {}, 16 | "defaultCase": [], 17 | "forkTasks": [], 18 | "startDelay": 0, 19 | "joinOn": [], 20 | "optional": false, 21 | "defaultExclusiveJoinTask": [], 22 | "asyncComplete": false, 23 | "loopOver": [], 24 | "onStateChange": {} 25 | } 26 | ], 27 | "inputParameters": [], 28 | "outputParameters": {}, 29 | "schemaVersion": 2, 30 | "restartable": true, 31 | "workflowStatusListenerEnabled": false, 32 | "ownerEmail": "29219ff2-5bb4-4e52-8004-3570829d6970@apps.orkes.io", 33 | "timeoutPolicy": "ALERT_ONLY", 34 | "timeoutSeconds": 0, 35 | "variables": {}, 36 | "inputTemplate": {}, 37 | "onStateChange": {} 38 | } -------------------------------------------------------------------------------- /test-harness/src/test/resources/metadata/sub_workflow_tests.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "name": "sub_workflow", 3 | "description": "sub_workflow", 4 | "version": 1, 5 | "tasks": [ 6 | { 7 | "name": "simple_task_in_sub_wf", 8 | "taskReferenceName": "t1", 9 | "inputParameters": {}, 10 | "type": "SIMPLE", 11 | "decisionCases": {}, 12 | "defaultCase": [], 13 | "forkTasks": [], 14 | "startDelay": 0, 15 | "joinOn": [], 16 | "optional": false, 17 | "defaultExclusiveJoinTask": [], 18 | "asyncComplete": false, 19 | "loopOver": [] 20 | } 21 | ], 22 | "inputParameters": [], 23 | "outputParameters": {}, 24 | "schemaVersion": 2, 25 | "restartable": true, 26 | "workflowStatusListenerEnabled": false, 27 | "timeoutPolicy": "ALERT_ONLY", 28 | "timeoutSeconds": 0, 29 | "ownerEmail": "test@harness.com" 30 | },{ 31 | "name": "integration_test_wf", 32 | "description": "integration_test_wf", 33 | "version": 1, 34 | "tasks": [ 35 | { 36 | "name": "integration_task_1", 37 | "taskReferenceName": "t1", 38 | "inputParameters": { 39 | "p1": "${workflow.input.param1}", 40 | "p2": "${workflow.input.param2}", 41 | "p3": "${CPEWF_TASK_ID}", 42 | "someNullKey": null 43 | }, 44 | "type": "SIMPLE" 45 | }, 46 | { 47 | "name": "integration_task_2", 48 | "taskReferenceName": "t2", 49 | "inputParameters": { 50 | "tp1": "${workflow.input.param1}", 51 | "tp2": "${t1.output.op}", 52 | "tp3": "${CPEWF_TASK_ID}" 53 | }, 54 | "type": "SIMPLE" 55 | } 56 | ], 57 | "inputParameters": [ 58 | "param1", 59 | "param2" 60 | ], 61 | "outputParameters": { 62 | "o1": "${workflow.input.param1}", 63 | "o2": "${t2.output.uuid}", 64 | "o3": "${t1.output.op}" 65 | }, 66 | "failureWorkflow": "$workflow.input.failureWfName", 67 | "schemaVersion": 2, 68 | "restartable": true, 69 | "workflowStatusListenerEnabled": false, 70 | "timeoutPolicy": "ALERT_ONLY", 71 | "timeoutSeconds": 0, 72 | "ownerEmail": "test@harness.com" 73 | },{ 74 | "name": "integration_test_wf_with_sub_wf", 75 | "description": "integration_test_wf_with_sub_wf", 76 | "version": 1, 77 | "tasks": [ 78 | { 79 | "name": "integration_task_1", 80 | "taskReferenceName": "t1", 81 | "inputParameters": { 82 | "p1": "${workflow.input.param1}", 83 | "p2": "${workflow.input.param2}", 84 | "someNullKey": null 85 | }, 86 | "type": "SIMPLE", 87 | "decisionCases": {}, 88 | "defaultCase": [], 89 | "forkTasks": [], 90 | "startDelay": 0, 91 | "joinOn": [], 92 | "optional": false, 93 | "defaultExclusiveJoinTask": [], 94 | "asyncComplete": false, 95 | "loopOver": [] 96 | }, 97 | { 98 | "name": "sub_workflow_task", 99 | "taskReferenceName": "t2", 100 | "inputParameters": { 101 | "param1": "${workflow.input.param1}", 102 | "param2": "${workflow.input.param2}", 103 | "subwf": "${workflow.input.nextSubwf}" 104 | }, 105 | "type": "SUB_WORKFLOW", 106 | "subWorkflowParam": { 107 | "name": "${workflow.input.subwf}", 108 | "version": 1 109 | }, 110 | "startDelay": 0, 111 | "joinOn": [], 112 | "optional": false, 113 | "defaultExclusiveJoinTask": [], 114 | "asyncComplete": false, 115 | "loopOver": [], 116 | "retryCount": 0 117 | } 118 | ], 119 | "inputParameters": [ 120 | "param1", 121 | "param2" 122 | ], 123 | "failureWorkflow": "$workflow.input.failureWfName", 124 | "schemaVersion": 2, 125 | "restartable": true, 126 | "workflowStatusListenerEnabled": false, 127 | "timeoutPolicy": "TIME_OUT_WF", 128 | "timeoutSeconds": 5, 129 | "ownerEmail": "test@harness.com" 130 | } 131 | ] -------------------------------------------------------------------------------- /test-harness/src/test/resources/sample_tasks.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "createTime": 1651856131126, 4 | "createdBy": "", 5 | "name": "image_compression", 6 | "description": "Edit or extend this sample task. Set the task name to get started", 7 | "retryCount": 3, 8 | "timeoutSeconds": 600, 9 | "inputKeys": [], 10 | "outputKeys": [], 11 | "timeoutPolicy": "ALERT_ONLY", 12 | "retryLogic": "FIXED", 13 | "retryDelaySeconds": 6, 14 | "responseTimeoutSeconds": 10, 15 | "inputTemplate": {}, 16 | "rateLimitPerFrequency": 0, 17 | "rateLimitFrequencyInSeconds": 1, 18 | "ownerEmail": "test@orkes.io", 19 | "backoffScaleFactor": 1 20 | }, 21 | { 22 | "createTime": 1651853134126, 23 | "createdBy": "", 24 | "name": "download_file_from_ec2", 25 | "description": "Edit or extend this sample task. Set the task name to get started", 26 | "retryCount": 1, 27 | "timeoutSeconds": 300, 28 | "inputKeys": [], 29 | "outputKeys": [], 30 | "timeoutPolicy": "TIME_OUT_WF", 31 | "retryLogic": "FIXED", 32 | "retryDelaySeconds": 3, 33 | "responseTimeoutSeconds": 50, 34 | "inputTemplate": {}, 35 | "rateLimitPerFrequency": 0, 36 | "rateLimitFrequencyInSeconds": 1, 37 | "ownerEmail": "test@orkes.io", 38 | "backoffScaleFactor": 1 39 | }, 40 | { 41 | "createTime": 1652856134126, 42 | "createdBy": "", 43 | "name": "update_database", 44 | "description": "Edit or extend this sample task. Set the task name to get started", 45 | "retryCount": 3, 46 | "timeoutSeconds": 900, 47 | "inputKeys": [], 48 | "outputKeys": [], 49 | "timeoutPolicy": "TIME_OUT_WF", 50 | "retryLogic": "FIXED", 51 | "retryDelaySeconds": 10, 52 | "responseTimeoutSeconds": 60, 53 | "inputTemplate": {}, 54 | "rateLimitPerFrequency": 0, 55 | "rateLimitFrequencyInSeconds": 1, 56 | "ownerEmail": "test@orkes.io", 57 | "backoffScaleFactor": 1 58 | } 59 | ] -------------------------------------------------------------------------------- /test-harness/src/test/resources/sample_workflow.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Do_While_Workflow", 3 | "description": "Do_While_Workflow", 4 | "version": 1, 5 | "tasks": [ 6 | { 7 | "name": "loopTask", 8 | "taskReferenceName": "loopTask", 9 | "inputParameters": { 10 | "value": "${workflow.input.loop}" 11 | }, 12 | "type": "DO_WHILE", 13 | "decisionCases": {}, 14 | "defaultCase": [], 15 | "forkTasks": [], 16 | "startDelay": 0, 17 | "joinOn": [], 18 | "optional": false, 19 | "defaultExclusiveJoinTask": [], 20 | "asyncComplete": false, 21 | "loopCondition": "if ($.loopTask['iteration'] < $.value) { true; } else { false;} ", 22 | "loopOver": [ 23 | { 24 | "name": "integration_task_0", 25 | "taskReferenceName": "integration_task_0", 26 | "inputParameters": {}, 27 | "type": "SIMPLE", 28 | "decisionCases": {}, 29 | "defaultCase": [], 30 | "forkTasks": [], 31 | "startDelay": 0, 32 | "joinOn": [], 33 | "optional": false, 34 | "defaultExclusiveJoinTask": [], 35 | "asyncComplete": false, 36 | "loopOver": [] 37 | }, 38 | { 39 | "name": "fork", 40 | "taskReferenceName": "fork", 41 | "inputParameters": {}, 42 | "type": "FORK_JOIN", 43 | "decisionCases": {}, 44 | "defaultCase": [], 45 | "forkTasks": [ 46 | [ 47 | { 48 | "name": "integration_task_1", 49 | "taskReferenceName": "integration_task_1", 50 | "inputParameters": {}, 51 | "type": "SIMPLE", 52 | "decisionCases": {}, 53 | "defaultCase": [], 54 | "forkTasks": [], 55 | "startDelay": 0, 56 | "joinOn": [], 57 | "optional": false, 58 | "defaultExclusiveJoinTask": [], 59 | "asyncComplete": false, 60 | "loopOver": [] 61 | } 62 | ], 63 | [ 64 | { 65 | "name": "integration_task_2", 66 | "taskReferenceName": "integration_task_2", 67 | "inputParameters": {}, 68 | "type": "SIMPLE", 69 | "decisionCases": {}, 70 | "defaultCase": [], 71 | "forkTasks": [], 72 | "startDelay": 0, 73 | "joinOn": [], 74 | "optional": false, 75 | "defaultExclusiveJoinTask": [], 76 | "asyncComplete": false, 77 | "loopOver": [] 78 | } 79 | ] 80 | ], 81 | "startDelay": 0, 82 | "joinOn": [], 83 | "optional": false, 84 | "defaultExclusiveJoinTask": [], 85 | "asyncComplete": false, 86 | "loopOver": [] 87 | }, 88 | { 89 | "name": "join", 90 | "taskReferenceName": "join", 91 | "inputParameters": {}, 92 | "type": "JOIN", 93 | "decisionCases": {}, 94 | "defaultCase": [], 95 | "forkTasks": [], 96 | "startDelay": 0, 97 | "joinOn": [ 98 | "integration_task_1", 99 | "integration_task_2" 100 | ], 101 | "optional": false, 102 | "defaultExclusiveJoinTask": [], 103 | "asyncComplete": false, 104 | "loopOver": [] 105 | } 106 | ] 107 | } 108 | ], 109 | "inputParameters": [], 110 | "outputParameters": {}, 111 | "schemaVersion": 2, 112 | "restartable": true, 113 | "workflowStatusListenerEnabled": false, 114 | "timeoutPolicy": "ALERT_ONLY", 115 | "timeoutSeconds": 0, 116 | "ownerEmail": "test@harness.com" 117 | } --------------------------------------------------------------------------------