├── .editorconfig ├── .github └── workflows │ ├── CODEOWNERS │ ├── build-push.yaml │ ├── maven-publish.yml │ └── run-tests.yaml ├── .gitignore ├── LICENSE ├── README.md ├── development.md ├── examples ├── .editorconfig ├── pom.xml └── src │ ├── main │ └── java │ │ └── io │ │ └── numaproj │ │ └── numaflow │ │ └── examples │ │ ├── accumulator │ │ └── sorter │ │ │ └── StreamSorterFactory.java │ │ ├── batchmap │ │ └── flatmap │ │ │ └── BatchFlatMap.java │ │ ├── map │ │ ├── evenodd │ │ │ └── EvenOddFunction.java │ │ ├── flatmap │ │ │ └── FlatMapFunction.java │ │ └── forward │ │ │ └── ForwardFunction.java │ │ ├── mapstream │ │ └── flatmapstream │ │ │ └── FlatMapStreamFunction.java │ │ ├── reduce │ │ ├── count │ │ │ ├── Config.java │ │ │ └── CounterFactory.java │ │ └── sum │ │ │ ├── SumFactory.java │ │ │ └── SumFunction.java │ │ ├── reducesession │ │ └── counter │ │ │ ├── CountFactory.java │ │ │ └── CountFunction.java │ │ ├── reducestreamer │ │ └── sum │ │ │ ├── SumFactory.java │ │ │ └── SumFunction.java │ │ ├── servingstore │ │ └── memory │ │ │ └── ServingInMemoryStore.java │ │ ├── sideinput │ │ ├── Config.java │ │ ├── README.md │ │ ├── simple │ │ │ └── SimpleSideInput.java │ │ └── udf │ │ │ ├── SideInputWatcher.java │ │ │ └── SimpleMapWithSideInput.java │ │ ├── sink │ │ ├── forkjoin │ │ │ └── ConcurrentSink.java │ │ └── simple │ │ │ └── SimpleSink.java │ │ ├── source │ │ └── simple │ │ │ └── SimpleSource.java │ │ └── sourcetransformer │ │ └── eventtimefilter │ │ └── EventTimeFilterFunction.java │ └── test │ └── java │ └── io │ └── numaproj │ └── numaflow │ └── examples │ ├── map │ ├── evenodd │ │ └── EvenOddFunctionTest.java │ └── flatmap │ │ └── FlatMapFunctionTest.java │ ├── server │ └── ServerTest.java │ ├── sink │ ├── forkjoin │ │ └── ConcurrentSinkTest.java │ └── simple │ │ └── SimpleSinkTest.java │ ├── source │ └── simple │ │ └── SimpleSourceTest.java │ └── sourcetransformer │ └── eventtimefilter │ └── EventTimeFilterFunctionTest.java ├── hack └── update_examples.sh ├── lombok.config ├── pom.xml ├── releases.md ├── settings.xml └── src ├── main ├── java │ └── io │ │ └── numaproj │ │ └── numaflow │ │ ├── accumulator │ │ ├── AccumulatorActor.java │ │ ├── AccumulatorSupervisorActor.java │ │ ├── Constants.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── OutputActor.java │ │ ├── OutputStreamObserverImpl.java │ │ ├── Server.java │ │ ├── Service.java │ │ ├── ShutdownActor.java │ │ └── model │ │ │ ├── Accumulator.java │ │ │ ├── AccumulatorFactory.java │ │ │ ├── Datum.java │ │ │ ├── Message.java │ │ │ └── OutputStreamObserver.java │ │ ├── batchmapper │ │ ├── BatchMapper.java │ │ ├── BatchResponse.java │ │ ├── BatchResponses.java │ │ ├── Constants.java │ │ ├── Datum.java │ │ ├── DatumIterator.java │ │ ├── DatumIteratorImpl.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── Message.java │ │ ├── Server.java │ │ └── Service.java │ │ ├── errors │ │ └── PersistCriticalError.java │ │ ├── info │ │ ├── ContainerType.java │ │ ├── Language.java │ │ ├── Protocol.java │ │ ├── ServerInfo.java │ │ ├── ServerInfoAccessor.java │ │ └── ServerInfoAccessorImpl.java │ │ ├── mapper │ │ ├── Constants.java │ │ ├── Datum.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── MapSupervisorActor.java │ │ ├── Mapper.java │ │ ├── MapperActor.java │ │ ├── MapperTestKit.java │ │ ├── Message.java │ │ ├── MessageList.java │ │ ├── Server.java │ │ └── Service.java │ │ ├── mapstreamer │ │ ├── Constants.java │ │ ├── Datum.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── MapStreamSupervisorActor.java │ │ ├── MapStreamer.java │ │ ├── MapStreamerActor.java │ │ ├── Message.java │ │ ├── OutputObserver.java │ │ ├── OutputObserverImpl.java │ │ ├── Server.java │ │ └── Service.java │ │ ├── reducer │ │ ├── ActorRequest.java │ │ ├── ActorResponse.java │ │ ├── Constants.java │ │ ├── Datum.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── IntervalWindow.java │ │ ├── Message.java │ │ ├── MessageList.java │ │ ├── Metadata.java │ │ ├── ReduceActor.java │ │ ├── ReduceShutdownActor.java │ │ ├── ReduceSupervisorActor.java │ │ ├── Reducer.java │ │ ├── ReducerFactory.java │ │ ├── ReducerTestKit.java │ │ ├── Server.java │ │ ├── Service.java │ │ └── metadata │ │ │ ├── IntervalWindowImpl.java │ │ │ └── MetadataImpl.java │ │ ├── reducestreamer │ │ ├── ActorRequest.java │ │ ├── ActorResponse.java │ │ ├── Constants.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── IntervalWindowImpl.java │ │ ├── MetadataImpl.java │ │ ├── OutputActor.java │ │ ├── OutputStreamObserverImpl.java │ │ ├── ReduceStreamerActor.java │ │ ├── Server.java │ │ ├── Service.java │ │ ├── ShutdownActor.java │ │ ├── SupervisorActor.java │ │ └── model │ │ │ ├── Datum.java │ │ │ ├── IntervalWindow.java │ │ │ ├── Message.java │ │ │ ├── Metadata.java │ │ │ ├── OutputStreamObserver.java │ │ │ ├── ReduceStreamer.java │ │ │ └── ReduceStreamerFactory.java │ │ ├── servingstore │ │ ├── Constants.java │ │ ├── GRPCConfig.java │ │ ├── GetDatum.java │ │ ├── GetDatumImpl.java │ │ ├── Payload.java │ │ ├── PutDatum.java │ │ ├── PutDatumImpl.java │ │ ├── Server.java │ │ ├── Service.java │ │ ├── ServingStorer.java │ │ └── StoredResult.java │ │ ├── sessionreducer │ │ ├── ActorRequest.java │ │ ├── ActorRequestType.java │ │ ├── ActorResponse.java │ │ ├── Constants.java │ │ ├── GRPCConfig.java │ │ ├── GetAccumulatorRequest.java │ │ ├── GetAccumulatorResponse.java │ │ ├── HandlerDatum.java │ │ ├── MergeAccumulatorRequest.java │ │ ├── MergeDoneResponse.java │ │ ├── OutputActor.java │ │ ├── OutputStreamObserverImpl.java │ │ ├── Server.java │ │ ├── Service.java │ │ ├── SessionReducerActor.java │ │ ├── ShutdownActor.java │ │ ├── SupervisorActor.java │ │ ├── UniqueIdGenerator.java │ │ └── model │ │ │ ├── Datum.java │ │ │ ├── Message.java │ │ │ ├── OutputStreamObserver.java │ │ │ ├── SessionReducer.java │ │ │ └── SessionReducerFactory.java │ │ ├── shared │ │ ├── ExceptionUtils.java │ │ ├── GrpcConfigRetriever.java │ │ ├── GrpcServerUtils.java │ │ ├── GrpcServerWrapper.java │ │ └── ThreadUtils.java │ │ ├── sideinput │ │ ├── Constants.java │ │ ├── GRPCConfig.java │ │ ├── Message.java │ │ ├── Server.java │ │ ├── Service.java │ │ └── SideInputRetriever.java │ │ ├── sinker │ │ ├── Constants.java │ │ ├── Datum.java │ │ ├── DatumIterator.java │ │ ├── DatumIteratorImpl.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── Response.java │ │ ├── ResponseList.java │ │ ├── Server.java │ │ ├── Service.java │ │ ├── Sinker.java │ │ └── SinkerTestKit.java │ │ ├── sourcer │ │ ├── AckRequest.java │ │ ├── AckRequestImpl.java │ │ ├── Constants.java │ │ ├── GRPCConfig.java │ │ ├── Message.java │ │ ├── NackRequest.java │ │ ├── NackRequestImpl.java │ │ ├── Offset.java │ │ ├── OutputObserver.java │ │ ├── OutputObserverImpl.java │ │ ├── ReadRequest.java │ │ ├── ReadRequestImpl.java │ │ ├── Server.java │ │ ├── Service.java │ │ ├── Sourcer.java │ │ └── SourcerTestKit.java │ │ └── sourcetransformer │ │ ├── Constants.java │ │ ├── Datum.java │ │ ├── GRPCConfig.java │ │ ├── HandlerDatum.java │ │ ├── Message.java │ │ ├── MessageList.java │ │ ├── Server.java │ │ ├── Service.java │ │ ├── SourceTransformer.java │ │ ├── SourceTransformerTestKit.java │ │ ├── TransformSupervisorActor.java │ │ └── TransformerActor.java ├── proto │ ├── accumulator │ │ └── v1 │ │ │ └── accumulator.proto │ ├── map │ │ └── v1 │ │ │ └── map.proto │ ├── reduce │ │ └── v1 │ │ │ └── reduce.proto │ ├── serving │ │ └── v1 │ │ │ └── store.proto │ ├── sessionreduce │ │ └── v1 │ │ │ └── sessionreduce.proto │ ├── sideinput │ │ └── v1 │ │ │ └── sideinput.proto │ ├── sink │ │ └── v1 │ │ │ └── sink.proto │ ├── source │ │ └── v1 │ │ │ └── source.proto │ └── sourcetransform │ │ └── v1 │ │ └── sourcetransformer.proto └── resources │ └── numaflow-java-sdk-version.properties └── test └── java └── io └── numaproj └── numaflow ├── accumulator ├── AccumulatorStreamObserver.java ├── GRPCConfigTest.java ├── ServerErrTest.java └── ServerTest.java ├── batchmapper ├── BatchMapOutputStreamObserver.java ├── DatumStreamImplTest.java ├── GRPCConfigTest.java ├── HandlerDatumTest.java ├── ServerErrTest.java └── ServerTest.java ├── errors └── PersistCriticalErrorTest.java ├── info └── ServerInfoAccessorImplTest.java ├── mapper ├── GRPCConfigTest.java ├── MapOutputStreamObserver.java ├── ServerErrTest.java └── ServerTest.java ├── mapstreamer ├── GRPCConfigTest.java ├── MapStreamOutputStreamObserver.java ├── ServiceErrTest.java └── ServiceTest.java ├── reducer ├── GRPCConfigTest.java ├── ReduceErrTestFactory.java ├── ReduceOutputStreamObserver.java ├── ReduceTestFactory.java ├── ServerErrTest.java ├── ServerTest.java ├── ShutDownActorTest.java └── SupervisorActorTest.java ├── reducestreamer ├── GRPCConfigTest.java ├── ReduceOutputStreamObserver.java ├── ServerErrTest.java ├── ServerTest.java ├── ShutdownActorTest.java └── SupervisorActorTest.java ├── servingstore ├── GRPCConfigTest.java ├── ServerErrTest.java └── ServerTest.java ├── sessionreducer ├── GRPCConfigTest.java ├── ReduceOutputStreamObserver.java ├── ServerErrTest.java ├── ServerTest.java └── ShutdownActorTest.java ├── shared ├── ExceptionUtilsTest.java └── GrpcServerUtilsTest.java ├── sideinput ├── GRPCConfigTest.java └── ServerTest.java ├── sinker ├── DatumStreamImplTest.java ├── GRPCConfigTest.java ├── ServerErrTest.java ├── ServerTest.java └── SinkOutputStreamObserver.java ├── sourcer ├── AckOutputStreamObserver.java ├── GRPCConfigTest.java ├── NackOutputStreamObserver.java ├── ReadOutputStreamObserver.java ├── ServerErrTest.java └── ServerTest.java └── sourcetransformer ├── GRPCConfigTest.java ├── ServerErrTest.java ├── ServerTest.java └── TransformerOutputStreamObserver.java /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | max_line_length = 100 8 | 9 | [*.java] 10 | indent_size = 4 11 | indent_style = space 12 | tab_width = 4 13 | ij_continuation_indent_size = 8 14 | ij_java_binary_operation_sign_on_next_line = true 15 | ij_java_binary_operation_wrap = normal 16 | ij_java_call_parameters_new_line_after_left_paren = true 17 | ij_java_call_parameters_wrap = on_every_item 18 | ij_java_class_count_to_use_import_on_demand = 9999 19 | ij_java_doc_add_blank_line_after_param_comments = true 20 | ij_java_doc_add_blank_line_after_return = true 21 | ij_java_doc_align_exception_comments = false 22 | ij_java_doc_align_param_comments = false 23 | ij_java_doc_do_not_wrap_if_one_line = true 24 | ij_java_doc_enable_formatting = true 25 | ij_java_doc_indent_on_continuation = true 26 | ij_java_doc_keep_empty_lines = true 27 | ij_java_doc_preserve_line_breaks = false 28 | ij_java_layout_static_imports_separately = true 29 | ij_java_method_call_chain_wrap = on_every_item 30 | ij_java_method_parameters_new_line_after_left_paren = true 31 | ij_java_method_parameters_wrap = on_every_item 32 | ij_java_names_count_to_use_import_on_demand = 9999 33 | ij_java_spaces_around_equality_operators = true 34 | ij_java_variable_annotation_wrap = normal 35 | ij_java_wrap_first_method_in_call_chain = true 36 | -------------------------------------------------------------------------------- /.github/workflows/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in 2 | # the repo. Unless a later match takes precedence 3 | * @whynowy @vigith @KeranYang @yhl25 @ashwinidulams 4 | -------------------------------------------------------------------------------- /.github/workflows/build-push.yaml: -------------------------------------------------------------------------------- 1 | name: Docker Publish 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | tags: 7 | - '*' 8 | 9 | jobs: 10 | docker_publish: 11 | # run workflow only on numaproj/numaflow-java repository 12 | if: ${{ github.repository }} == "numaproj/numaflow-java" 13 | name: Build, Tag, and Push Image 14 | runs-on: ubuntu-latest 15 | 16 | strategy: 17 | matrix: 18 | execution_ids: [ 19 | "batch-map-flatmap", "mapt-event-time-filter-function", "flat-map-stream", "map-flatmap", 20 | "even-odd", "simple-sink", "reduce-sum", "reduce-stream-sum", 21 | "map-forward-message", "reduce-counter", "sideinput-example", 22 | "udf-sideinput-example", "source-simple-source", "session-reduce-count", "stream-sorter" 23 | ] 24 | 25 | steps: 26 | - name: Check out repository 27 | uses: actions/checkout@v3 28 | - name: Set up JDK 11 29 | uses: actions/setup-java@v3 30 | with: 31 | java-version: '11' 32 | distribution: 'temurin' 33 | - name: Set up QEMU 34 | uses: docker/setup-qemu-action@v3 35 | - name: Set up Docker Buildx 36 | uses: docker/setup-buildx-action@v3 37 | - name: Login to Quay.io registry 38 | uses: docker/login-action@v3 39 | with: 40 | registry: quay.io 41 | username: ${{ secrets.NUMAIO_USERNAME }} 42 | password: ${{ secrets.NUMAIO_PASSWORD }} 43 | - name: Build, tag, and push images 44 | run: ./hack/update_examples.sh --build-push-example ${{ matrix.execution_ids }} 45 | -------------------------------------------------------------------------------- /.github/workflows/maven-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a package using Maven and then publish it to Maven Central and GitHub packages when a release is created 2 | 3 | name: Publish to Maven Central and Github Packages 4 | on: 5 | release: 6 | types: [ created ] 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | contents: read 12 | packages: write 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Import GPG key 17 | id: import_gpg 18 | uses: crazy-max/ghaction-import-gpg@v5 19 | with: 20 | gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} 21 | passphrase: ${{ secrets.GPG_PASSPHRASE }} 22 | 23 | - name: Set up Java for publishing to Maven Central Repository 24 | uses: actions/setup-java@v3 25 | with: 26 | java-version: '11' 27 | distribution: 'temurin' 28 | server-id: central 29 | server-username: MAVEN_USERNAME 30 | server-password: MAVEN_PASSWORD 31 | settings-path: ${{ github.workspace }} # location for the settings.xml file 32 | 33 | - name: Publish to the Maven Central Repository 34 | run: mvn -DcentralRelease=true -P central deploy -s $GITHUB_WORKSPACE/settings.xml 35 | env: 36 | MAVEN_USERNAME: ${{ secrets.MVN_CENTRAL_USERNAME }} 37 | MAVEN_PASSWORD: ${{ secrets.MVN_CENTRAL_PASSWORD }} 38 | 39 | - name: Set up Java for publishing to GitHub Packages 40 | uses: actions/setup-java@v3 41 | with: 42 | java-version: '11' 43 | distribution: 'temurin' 44 | server-id: github # Value of the distributionManagement/repository/id field of the pom.xml 45 | settings-path: ${{ github.workspace }} #location for the settings.xml file 46 | - name: Publish to GitHub Packages 47 | run: mvn -DgithubRelease=true -P github deploy -s $GITHUB_WORKSPACE/settings.xml 48 | env: 49 | GITHUB_TOKEN: ${{ github.token }} 50 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yaml: -------------------------------------------------------------------------------- 1 | name: Numaflow Java CI 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v4 13 | - name: Set up JDK 11 14 | uses: actions/setup-java@v4 15 | with: 16 | java-version: '11' 17 | distribution: 'temurin' 18 | - name: Build with Maven, run unit tests and the coverage check 19 | run: mvn clean install 20 | - name: Build Examples 21 | run: cd examples && mvn clean install 22 | - name: Upload coverage reports to Codecov 23 | uses: codecov/codecov-action@v4 24 | with: 25 | token: ${{ secrets.CODECOV_TOKEN }} 26 | slug: numaproj/numaflow-java 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | 25 | # IDEs 26 | *.idea/ 27 | *.iml 28 | 29 | # Maven, see https://github.com/github/gitignore/blob/main/Maven.gitignore 30 | target/ 31 | pom.xml.tag 32 | pom.xml.releaseBackup 33 | pom.xml.versionsBackup 34 | pom.xml.next 35 | release.properties 36 | dependency-reduced-pom.xml 37 | buildNumber.properties 38 | .mvn/timing.properties 39 | .mvn/wrapper/maven-wrapper.jar 40 | 41 | # IDE 42 | .vscode/ 43 | .DS_Store 44 | */**/.DS_Store 45 | -------------------------------------------------------------------------------- /examples/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | max_line_length = 100 8 | 9 | [*.java] 10 | indent_size = 4 11 | indent_style = space 12 | tab_width = 4 13 | ij_continuation_indent_size = 8 14 | ij_java_binary_operation_sign_on_next_line = true 15 | ij_java_binary_operation_wrap = normal 16 | ij_java_call_parameters_new_line_after_left_paren = true 17 | ij_java_call_parameters_wrap = on_every_item 18 | ij_java_class_count_to_use_import_on_demand = 9999 19 | ij_java_doc_add_blank_line_after_param_comments = true 20 | ij_java_doc_add_blank_line_after_return = true 21 | ij_java_doc_align_exception_comments = false 22 | ij_java_doc_align_param_comments = false 23 | ij_java_doc_do_not_wrap_if_one_line = true 24 | ij_java_doc_enable_formatting = true 25 | ij_java_doc_indent_on_continuation = true 26 | ij_java_doc_keep_empty_lines = true 27 | ij_java_doc_preserve_line_breaks = false 28 | ij_java_layout_static_imports_separately = true 29 | ij_java_method_call_chain_wrap = on_every_item 30 | ij_java_method_parameters_new_line_after_left_paren = true 31 | ij_java_method_parameters_wrap = on_every_item 32 | ij_java_names_count_to_use_import_on_demand = 9999 33 | ij_java_spaces_around_equality_operators = true 34 | ij_java_variable_annotation_wrap = normal 35 | ij_java_wrap_first_method_in_call_chain = true 36 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/batchmap/flatmap/BatchFlatMap.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.batchmap.flatmap; 2 | 3 | import io.numaproj.numaflow.batchmapper.BatchMapper; 4 | import io.numaproj.numaflow.batchmapper.BatchResponse; 5 | import io.numaproj.numaflow.batchmapper.BatchResponses; 6 | import io.numaproj.numaflow.batchmapper.Datum; 7 | import io.numaproj.numaflow.batchmapper.DatumIterator; 8 | import io.numaproj.numaflow.batchmapper.Message; 9 | import io.numaproj.numaflow.batchmapper.Server; 10 | import lombok.extern.slf4j.Slf4j; 11 | 12 | @Slf4j 13 | public class BatchFlatMap extends BatchMapper { 14 | @Override 15 | public BatchResponses processMessage(DatumIterator datumStream) { 16 | BatchResponses batchResponses = new BatchResponses(); 17 | while (true) { 18 | Datum datum = null; 19 | try { 20 | datum = datumStream.next(); 21 | } catch (InterruptedException e) { 22 | Thread.currentThread().interrupt(); 23 | continue; 24 | } 25 | // null means the iterator is closed so we are good to break the loop. 26 | if (datum == null) { 27 | break; 28 | } 29 | try { 30 | String msg = new String(datum.getValue()); 31 | String[] strs = msg.split(","); 32 | BatchResponse batchResponse = new BatchResponse(datum.getId()); 33 | for (String str : strs) { 34 | batchResponse.append(new Message(str.getBytes())); 35 | } 36 | batchResponses.append(batchResponse); 37 | } catch (Exception e) { 38 | batchResponses.append(new BatchResponse(datum.getId())); 39 | } 40 | } 41 | return batchResponses; 42 | } 43 | 44 | public static void main(String[] args) throws Exception { 45 | Server server = new Server(new BatchFlatMap()); 46 | 47 | // Start the server 48 | server.start(); 49 | 50 | // wait for the server to shutdown 51 | server.awaitTermination(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/map/evenodd/EvenOddFunction.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.map.evenodd; 2 | 3 | import io.numaproj.numaflow.mapper.Datum; 4 | import io.numaproj.numaflow.mapper.Mapper; 5 | import io.numaproj.numaflow.mapper.Message; 6 | import io.numaproj.numaflow.mapper.MessageList; 7 | import io.numaproj.numaflow.mapper.Server; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | /** 11 | * This is a simple User Defined Function example which receives a message, 12 | * and attaches keys to the message based on the value, if the value is even 13 | * the key will be set as "even" if the value is odd the key will be set as 14 | * "odd" 15 | */ 16 | 17 | @Slf4j 18 | public class EvenOddFunction extends Mapper { 19 | 20 | public static void main(String[] args) throws Exception { 21 | Server server = new Server(new EvenOddFunction()); 22 | 23 | // Start the server 24 | server.start(); 25 | 26 | // Wait for the server to shutdown 27 | server.awaitTermination(); 28 | } 29 | 30 | public MessageList processMessage(String[] keys, Datum data) { 31 | int value = 0; 32 | try { 33 | value = Integer.parseInt(new String(data.getValue())); 34 | } catch (NumberFormatException e) { 35 | log.error("Error occurred while parsing int"); 36 | return MessageList.newBuilder().addMessage(Message.toDrop()).build(); 37 | } 38 | 39 | String[] outputKeys = value % 2 == 0 ? new String[]{"even"} : new String[]{"odd"}; 40 | 41 | // tags will be used for conditional forwarding 42 | String[] tags = value % 2 == 0 ? new String[]{"even-tag"} : new String[]{"odd-tag"}; 43 | 44 | return MessageList 45 | .newBuilder() 46 | .addMessage(new Message(data.getValue(), outputKeys, tags)) 47 | .build(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/map/flatmap/FlatMapFunction.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.map.flatmap; 2 | 3 | import io.numaproj.numaflow.mapper.Datum; 4 | import io.numaproj.numaflow.mapper.Mapper; 5 | import io.numaproj.numaflow.mapper.Message; 6 | import io.numaproj.numaflow.mapper.MessageList; 7 | import io.numaproj.numaflow.mapper.Server; 8 | 9 | /** 10 | * This is a simple User Defined Function example which processes the input message 11 | * and produces more than one output messages(flatMap) 12 | * example : if the input message is "dog,cat", it produces two output messages 13 | * "dog" and "cat" 14 | */ 15 | 16 | public class FlatMapFunction extends Mapper { 17 | 18 | public static void main(String[] args) throws Exception { 19 | Server server = new Server(new FlatMapFunction()); 20 | 21 | // Start the server 22 | server.start(); 23 | 24 | // Wait for the server to shut down 25 | server.awaitTermination(); 26 | } 27 | 28 | public MessageList processMessage(String[] keys, Datum data) { 29 | String msg = new String(data.getValue()); 30 | String[] strs = msg.split(","); 31 | MessageList.MessageListBuilder listBuilder = MessageList.newBuilder(); 32 | 33 | for (String str : strs) { 34 | listBuilder.addMessage(new Message(str.getBytes())); 35 | } 36 | 37 | return listBuilder.build(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/map/forward/ForwardFunction.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.map.forward; 2 | 3 | import io.numaproj.numaflow.mapper.Datum; 4 | import io.numaproj.numaflow.mapper.Mapper; 5 | import io.numaproj.numaflow.mapper.Message; 6 | import io.numaproj.numaflow.mapper.MessageList; 7 | import io.numaproj.numaflow.mapper.Server; 8 | 9 | /** 10 | * This is a simple User Defined Function example which forwards the message as is. 11 | */ 12 | 13 | public class ForwardFunction extends Mapper { 14 | public static void main(String[] args) throws Exception { 15 | Server server = new Server(new ForwardFunction()); 16 | 17 | // Start the server 18 | server.start(); 19 | 20 | // Wait for the server to shut down 21 | server.awaitTermination(); 22 | } 23 | 24 | public MessageList processMessage(String[] keys, Datum data) { 25 | return MessageList 26 | .newBuilder() 27 | .addMessage(new Message(data.getValue(), keys)) 28 | .build(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/mapstream/flatmapstream/FlatMapStreamFunction.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.mapstream.flatmapstream; 2 | 3 | import io.numaproj.numaflow.mapstreamer.Datum; 4 | import io.numaproj.numaflow.mapstreamer.MapStreamer; 5 | import io.numaproj.numaflow.mapstreamer.Message; 6 | import io.numaproj.numaflow.mapstreamer.OutputObserver; 7 | import io.numaproj.numaflow.mapstreamer.Server; 8 | 9 | 10 | /** 11 | * This is a simple User Defined Function example which processes the input message 12 | * and produces more than one output messages(flatMap) in a streaming mode 13 | * example : if the input message is "dog,cat", it streams two output messages 14 | * "dog" and "cat" 15 | */ 16 | 17 | public class FlatMapStreamFunction extends MapStreamer { 18 | 19 | public static void main(String[] args) throws Exception { 20 | Server server = new Server(new FlatMapStreamFunction()); 21 | 22 | // Start the server 23 | server.start(); 24 | 25 | // wait for the server to shutdown 26 | server.awaitTermination(); 27 | } 28 | 29 | public void processMessage(String[] keys, Datum data, OutputObserver outputObserver) { 30 | String msg = new String(data.getValue()); 31 | String[] strs = msg.split(","); 32 | 33 | for (String str : strs) { 34 | outputObserver.send(new Message(str.getBytes())); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/reduce/count/Config.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.reduce.count; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | 6 | @Getter 7 | @AllArgsConstructor 8 | public class Config { 9 | private int incrementBy; 10 | } 11 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/reduce/sum/SumFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.reduce.sum; 2 | 3 | import io.numaproj.numaflow.reducer.ReducerFactory; 4 | import io.numaproj.numaflow.reducer.Server; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | @Slf4j 8 | public class SumFactory extends ReducerFactory { 9 | 10 | public static void main(String[] args) throws Exception { 11 | log.info("Starting sum udf server"); 12 | Server server = new Server(new SumFactory()); 13 | 14 | // Start the server 15 | server.start(); 16 | 17 | // wait for the server to shut down 18 | server.awaitTermination(); 19 | } 20 | 21 | @Override 22 | public SumFunction createReducer() { 23 | return new SumFunction(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/reduce/sum/SumFunction.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.reduce.sum; 2 | 3 | import io.numaproj.numaflow.reducer.Datum; 4 | import io.numaproj.numaflow.reducer.Message; 5 | import io.numaproj.numaflow.reducer.MessageList; 6 | import io.numaproj.numaflow.reducer.Metadata; 7 | import io.numaproj.numaflow.reducer.Reducer; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | 11 | @Slf4j 12 | public class SumFunction extends Reducer { 13 | 14 | private int sum = 0; 15 | 16 | @Override 17 | public void addMessage(String[] keys, Datum datum, Metadata md) { 18 | try { 19 | sum += Integer.parseInt(new String(datum.getValue())); 20 | } catch (NumberFormatException e) { 21 | log.info("error while parsing integer - {}", e.getMessage()); 22 | } 23 | } 24 | 25 | @Override 26 | public MessageList getOutput(String[] keys, Metadata md) { 27 | return MessageList 28 | .newBuilder() 29 | .addMessage(new Message(String.valueOf(sum).getBytes())) 30 | .build(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/reducesession/counter/CountFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.reducesession.counter; 2 | 3 | import io.numaproj.numaflow.sessionreducer.Server; 4 | import io.numaproj.numaflow.sessionreducer.model.SessionReducerFactory; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | /** 8 | * CountFactory extends SessionReducerFactory to support creating instances of SumFunction. 9 | * It also provides a main function to start a server for handling the session reduce stream. 10 | */ 11 | @Slf4j 12 | public class CountFactory extends SessionReducerFactory { 13 | 14 | public static void main(String[] args) throws Exception { 15 | log.info("count udf was invoked"); 16 | Server server = new Server(new CountFactory()); 17 | 18 | // Start the server 19 | server.start(); 20 | 21 | // wait for the server to shut down 22 | server.awaitTermination(); 23 | } 24 | 25 | @Override 26 | public CountFunction createSessionReducer() { 27 | return new CountFunction(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/reducesession/counter/CountFunction.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.reducesession.counter; 2 | 3 | import io.numaproj.numaflow.sessionreducer.model.Datum; 4 | import io.numaproj.numaflow.sessionreducer.model.Message; 5 | import io.numaproj.numaflow.sessionreducer.model.SessionReducer; 6 | import lombok.extern.slf4j.Slf4j; 7 | 8 | import java.util.concurrent.atomic.AtomicInteger; 9 | 10 | /** 11 | * CountFunction is a simple session reducer which counts the number of events in a session. 12 | */ 13 | @Slf4j 14 | public class CountFunction extends SessionReducer { 15 | 16 | private final AtomicInteger count = new AtomicInteger(0); 17 | 18 | @Override 19 | public void processMessage( 20 | String[] keys, 21 | Datum datum, 22 | io.numaproj.numaflow.sessionreducer.model.OutputStreamObserver outputStreamObserver) { 23 | this.count.incrementAndGet(); 24 | } 25 | 26 | @Override 27 | public void handleEndOfStream( 28 | String[] keys, 29 | io.numaproj.numaflow.sessionreducer.model.OutputStreamObserver outputStreamObserver) { 30 | outputStreamObserver.send(new Message(String.valueOf(this.count.get()).getBytes())); 31 | } 32 | 33 | @Override 34 | public byte[] accumulator() { 35 | return String.valueOf(this.count.get()).getBytes(); 36 | } 37 | 38 | @Override 39 | public void mergeAccumulator(byte[] accumulator) { 40 | int value = 0; 41 | try { 42 | value = Integer.parseInt(new String(accumulator)); 43 | } catch (NumberFormatException e) { 44 | log.info("error while parsing integer - {}", e.getMessage()); 45 | } 46 | this.count.addAndGet(value); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/reducestreamer/sum/SumFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.reducestreamer.sum; 2 | 3 | import io.numaproj.numaflow.reducestreamer.Server; 4 | import io.numaproj.numaflow.reducestreamer.model.ReduceStreamerFactory; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | /** 8 | * SumFactory extends ReduceStreamerFactory to support creating instances of SumFunction. 9 | * It also provides a main function to start a server for handling the reduce stream. 10 | */ 11 | @Slf4j 12 | public class SumFactory extends ReduceStreamerFactory { 13 | 14 | public static void main(String[] args) throws Exception { 15 | log.info("sum udf was invoked"); 16 | Server server = new Server(new SumFactory()); 17 | 18 | // Start the server 19 | server.start(); 20 | 21 | // wait for the server to shut down 22 | server.awaitTermination(); 23 | } 24 | 25 | @Override 26 | public SumFunction createReduceStreamer() { 27 | return new SumFunction(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/reducestreamer/sum/SumFunction.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.reducestreamer.sum; 2 | 3 | import io.numaproj.numaflow.reducestreamer.model.Datum; 4 | import io.numaproj.numaflow.reducestreamer.model.Message; 5 | import io.numaproj.numaflow.reducestreamer.model.Metadata; 6 | import io.numaproj.numaflow.reducestreamer.model.OutputStreamObserver; 7 | import io.numaproj.numaflow.reducestreamer.model.ReduceStreamer; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | /** 11 | * SumFunction is a User Defined Reduce Stream Function example which sums up the values for the given keys 12 | * and outputs the sum when the sum is greater than 100. 13 | * When the input stream closes, the function outputs the sum no matter what value it holds. 14 | */ 15 | @Slf4j 16 | public class SumFunction extends ReduceStreamer { 17 | 18 | private int sum = 0; 19 | 20 | @Override 21 | public void processMessage( 22 | String[] keys, 23 | Datum datum, 24 | OutputStreamObserver outputStreamObserver, 25 | Metadata md) { 26 | try { 27 | sum += Integer.parseInt(new String(datum.getValue())); 28 | } catch (NumberFormatException e) { 29 | log.info("error while parsing integer - {}", e.getMessage()); 30 | } 31 | if (sum >= 100) { 32 | outputStreamObserver.send(new Message(String.valueOf(sum).getBytes(), keys)); 33 | sum = 0; 34 | } 35 | } 36 | 37 | @Override 38 | public void handleEndOfStream( 39 | String[] keys, 40 | OutputStreamObserver outputStreamObserver, 41 | Metadata md) { 42 | outputStreamObserver.send(new Message(String.valueOf(sum).getBytes())); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/servingstore/memory/ServingInMemoryStore.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.servingstore.memory; 2 | 3 | import io.numaproj.numaflow.servingstore.GetDatum; 4 | import io.numaproj.numaflow.servingstore.Payload; 5 | import io.numaproj.numaflow.servingstore.PutDatum; 6 | import io.numaproj.numaflow.servingstore.Server; 7 | import io.numaproj.numaflow.servingstore.ServingStorer; 8 | import io.numaproj.numaflow.servingstore.StoredResult; 9 | import lombok.extern.slf4j.Slf4j; 10 | 11 | import java.util.Collections; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | @Slf4j 17 | public class ServingInMemoryStore extends ServingStorer { 18 | private final Map> store = new HashMap<>(); 19 | 20 | public void put(PutDatum putDatum) { 21 | log.info("Putting data into the store with ID: {}", putDatum.ID()); 22 | store.put(putDatum.ID(), putDatum.Payloads()); 23 | } 24 | 25 | public StoredResult get(GetDatum getDatum) { 26 | log.info("Getting data from the store with ID: {}", getDatum.ID()); 27 | List payloads = store.getOrDefault(getDatum.ID(), Collections.emptyList()); 28 | return new StoredResult(getDatum.ID(), payloads); 29 | } 30 | 31 | public static void main(String[] args) throws Exception { 32 | Server server = new Server(new ServingInMemoryStore()); 33 | 34 | // Start the server 35 | server.start(); 36 | 37 | // Wait for the server to shutdown 38 | server.awaitTermination(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/sideinput/Config.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.sideinput; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | import lombok.Setter; 6 | import lombok.ToString; 7 | 8 | /** 9 | * A simple config class to hold the source and sampling rate. 10 | * This config will be serialized and used as a side input message. 11 | */ 12 | @Getter 13 | @Setter 14 | @AllArgsConstructor 15 | @ToString 16 | public class Config { 17 | private String source; 18 | private float sampling; 19 | } 20 | -------------------------------------------------------------------------------- /examples/src/main/java/io/numaproj/numaflow/examples/sink/simple/SimpleSink.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.sink.simple; 2 | 3 | import io.numaproj.numaflow.sinker.Datum; 4 | import io.numaproj.numaflow.sinker.DatumIterator; 5 | import io.numaproj.numaflow.sinker.Response; 6 | import io.numaproj.numaflow.sinker.ResponseList; 7 | import io.numaproj.numaflow.sinker.Server; 8 | import io.numaproj.numaflow.sinker.Sinker; 9 | import lombok.extern.slf4j.Slf4j; 10 | 11 | 12 | /** 13 | * This is a simple User Defined Sink example which logs the input message 14 | */ 15 | 16 | @Slf4j 17 | public class SimpleSink extends Sinker { 18 | 19 | public static void main(String[] args) throws Exception { 20 | Server server = new Server(new SimpleSink()); 21 | 22 | // Start the server 23 | server.start(); 24 | 25 | // wait for the server to shut down 26 | server.awaitTermination(); 27 | } 28 | 29 | @Override 30 | public ResponseList processMessages(DatumIterator datumIterator) { 31 | ResponseList.ResponseListBuilder responseListBuilder = ResponseList.newBuilder(); 32 | while (true) { 33 | Datum datum = null; 34 | try { 35 | datum = datumIterator.next(); 36 | } catch (InterruptedException e) { 37 | Thread.currentThread().interrupt(); 38 | continue; 39 | } 40 | // null means the iterator is closed, so we break the loop 41 | if (datum == null) { 42 | break; 43 | } 44 | try { 45 | String msg = new String(datum.getValue()); 46 | log.info("Received message: {}, headers - {}", msg, datum.getHeaders()); 47 | responseListBuilder.addResponse(Response.responseOK(datum.getId())); 48 | } catch (Exception e) { 49 | responseListBuilder.addResponse(Response.responseFailure( 50 | datum.getId(), 51 | e.getMessage())); 52 | } 53 | } 54 | return responseListBuilder.build(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /examples/src/test/java/io/numaproj/numaflow/examples/map/flatmap/FlatMapFunctionTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.map.flatmap; 2 | 3 | import io.numaproj.numaflow.mapper.MapperTestKit; 4 | import io.numaproj.numaflow.mapper.Message; 5 | import io.numaproj.numaflow.mapper.MessageList; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.junit.jupiter.api.Assertions; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.util.List; 11 | 12 | 13 | @Slf4j 14 | public class FlatMapFunctionTest { 15 | 16 | @Test 17 | public void testSingleString() { 18 | MapperTestKit.TestDatum datum = MapperTestKit.TestDatum 19 | .builder() 20 | .value("apple".getBytes()) 21 | .build(); 22 | 23 | FlatMapFunction flatMapFunction = new FlatMapFunction(); 24 | MessageList result = flatMapFunction.processMessage(new String[]{}, datum); 25 | 26 | List messages = result.getMessages(); 27 | Assertions.assertEquals(1, messages.size()); 28 | 29 | Assertions.assertEquals("apple", new String(messages.get(0).getValue())); 30 | } 31 | 32 | @Test 33 | public void testEmptyString() { 34 | MapperTestKit.TestDatum datum = MapperTestKit.TestDatum 35 | .builder() 36 | .value("".getBytes()) 37 | .build(); 38 | 39 | FlatMapFunction flatMapFunction = new FlatMapFunction(); 40 | MessageList result = flatMapFunction.processMessage(new String[]{}, datum); 41 | 42 | List messages = result.getMessages(); 43 | Assertions.assertEquals(1, messages.size()); 44 | 45 | Assertions.assertEquals("", new String(messages.get(0).getValue())); 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /examples/src/test/java/io/numaproj/numaflow/examples/sink/simple/SimpleSinkTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.examples.sink.simple; 2 | 3 | import io.numaproj.numaflow.sinker.Response; 4 | import io.numaproj.numaflow.sinker.ResponseList; 5 | import io.numaproj.numaflow.sinker.SinkerTestKit; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.junit.jupiter.api.Assertions; 8 | import org.junit.jupiter.api.Test; 9 | 10 | @Slf4j 11 | public class SimpleSinkTest { 12 | 13 | @Test 14 | public void testSimpleSink() { 15 | int datumCount = 10; 16 | SimpleSink simpleSink = new SimpleSink(); 17 | // Create a test datum iterator with 10 messages 18 | SinkerTestKit.TestListIterator testListIterator = new SinkerTestKit.TestListIterator(); 19 | for (int i = 0; i < datumCount; i++) { 20 | testListIterator.addDatum( 21 | SinkerTestKit.TestDatum 22 | .builder() 23 | .id("id-" + i) 24 | .value(("value-" + i).getBytes()) 25 | .build()); 26 | } 27 | ResponseList responseList = simpleSink.processMessages(testListIterator); 28 | Assertions.assertEquals(datumCount, responseList.getResponses().size()); 29 | for (Response response : responseList.getResponses()) { 30 | Assertions.assertEquals(true, response.getSuccess()); 31 | } 32 | // we can add the logic to verify if the messages were 33 | // successfully written to the sink(could be a file, database, etc.) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /lombok.config: -------------------------------------------------------------------------------- 1 | # This tells lombok this directory is the root, 2 | # no need to look somewhere else for java code. 3 | config.stopBubbling = true 4 | # This will add the @lombok.Generated annotation 5 | # to all the code generated by Lombok, 6 | # so it can be excluded from coverage by jacoco. 7 | lombok.addLombokGeneratedAnnotation = true 8 | -------------------------------------------------------------------------------- /settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | github 9 | ${env.JAVA_SDK_MAVEN_USER} 10 | ${env.JAVA_SDK_MAVEN_PASSWORD} 11 | 12 | 13 | central 14 | ${env.MAVEN_USERNAME} 15 | ${env.MAVEN_PASSWORD} 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator; 2 | 3 | class Constants { 4 | public static final String SUCCESS = "SUCCESS"; 5 | public static final String DELIMITER = ":"; 6 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 7 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/accumulator.sock"; 8 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = 9 | "/var/run/numaflow/accumulator-server-info"; 10 | public static final int DEFAULT_PORT = 50051; 11 | public static final String DEFAULT_HOST = "localhost"; 12 | public static final String EOF = "EOF"; 13 | 14 | // Private constructor to prevent instantiation 15 | private Constants() { 16 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for map gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running 36 | // using numaflow 37 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator; 2 | 3 | 4 | import io.numaproj.numaflow.accumulator.model.Datum; 5 | import lombok.AllArgsConstructor; 6 | 7 | import java.time.Instant; 8 | import java.util.Map; 9 | 10 | @AllArgsConstructor 11 | class HandlerDatum implements Datum { 12 | private String[] keys; 13 | private byte[] value; 14 | private Instant watermark; 15 | private Instant eventTime; 16 | private Map headers; 17 | private String id; 18 | 19 | 20 | @Override 21 | public Instant getWatermark() { 22 | return this.watermark; 23 | } 24 | 25 | @Override 26 | public byte[] getValue() { 27 | return this.value; 28 | } 29 | 30 | @Override 31 | public String[] getKeys() { 32 | return this.keys; 33 | } 34 | 35 | @Override 36 | public Instant getEventTime() { 37 | return this.eventTime; 38 | } 39 | 40 | @Override 41 | public Map getHeaders() { 42 | return this.headers; 43 | } 44 | 45 | @Override 46 | public String getID() { 47 | return this.id; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/OutputActor.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator; 2 | 3 | import akka.actor.AbstractActor; 4 | import akka.actor.Props; 5 | import io.grpc.stub.StreamObserver; 6 | import io.numaproj.numaflow.accumulator.v1.AccumulatorOuterClass; 7 | import lombok.AllArgsConstructor; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | /** 11 | * Output actor is a wrapper around the gRPC output stream. 12 | * It ensures synchronized calls to the responseObserver onNext() and invokes onComplete at the end of the stream. 13 | * ALL accumulator responses are sent to the output actor before getting forwarded to the output gRPC stream. 14 | *

15 | * More details about gRPC StreamObserver concurrency: https://grpc.github.io/grpc-java/javadoc/io/grpc/stub/StreamObserver.html 16 | */ 17 | @Slf4j 18 | @AllArgsConstructor 19 | class OutputActor extends AbstractActor { 20 | StreamObserver responseObserver; 21 | 22 | public static Props props( 23 | StreamObserver responseObserver) { 24 | return Props.create( 25 | OutputActor.class, 26 | responseObserver); 27 | } 28 | 29 | @Override 30 | public Receive createReceive() { 31 | return receiveBuilder() 32 | .match(AccumulatorOuterClass.AccumulatorResponse.class, this::handleResponse) 33 | .match(String.class, this::handleEOF) 34 | .build(); 35 | } 36 | 37 | private void handleResponse(AccumulatorOuterClass.AccumulatorResponse response) { 38 | responseObserver.onNext(response); 39 | } 40 | 41 | private void handleEOF(String eof) { 42 | responseObserver.onCompleted(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/model/Accumulator.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator.model; 2 | 3 | /** 4 | * Accumulator exposes methods for performing accumulation operations on the 5 | * stream of data. 6 | */ 7 | public abstract class Accumulator { 8 | /** 9 | * method which will be used for processing messages, gets invoked for every 10 | * message in the keyed accumulator stream. 11 | * 12 | * @param datum current message to be processed by the accumulator 13 | * @param outputStream observer for sending the output {@link Message} to the 14 | * output stream. 15 | */ 16 | public abstract void processMessage( 17 | Datum datum, 18 | OutputStreamObserver outputStream); 19 | 20 | /** 21 | * handleEndOfStream handles the closure of the keyed accumulator stream. 22 | * This method is invoked when the input accumulator stream is closed. 23 | * It provides the capability of constructing final responses based on the 24 | * messages processed so far. 25 | * 26 | * @param outputStreamObserver observer for sending the output {@link Message} 27 | * to the output stream. 28 | */ 29 | public abstract void handleEndOfStream( 30 | OutputStreamObserver outputStreamObserver); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/model/AccumulatorFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator.model; 2 | 3 | /** 4 | * AccumulatorFactory is the factory for Accumulator. 5 | */ 6 | public abstract class AccumulatorFactory { 7 | /** 8 | * Create a concrete instance of Accumulator, will be invoked for 9 | * every keyed stream. Separate accumulator instances for used for 10 | * processing keyed streams. 11 | * 12 | * @return a concrete instance of Accumulator 13 | */ 14 | public abstract AccumulatorT createAccumulator(); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/model/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator.model; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | 6 | /** 7 | * Datum contains methods to get the payload and metadata information. 8 | */ 9 | public interface Datum { 10 | /** 11 | * method to get the payload value 12 | * 13 | * @return returns the payload value in byte array 14 | */ 15 | byte[] getValue(); 16 | 17 | /** 18 | * method to get the keys 19 | * 20 | * @return returns the keys in the form of string array 21 | */ 22 | String[] getKeys(); 23 | 24 | /** 25 | * method to get the event time of the payload 26 | * 27 | * @return returns the event time of the payload 28 | */ 29 | Instant getEventTime(); 30 | 31 | /** 32 | * method to get the watermark information 33 | * 34 | * @return returns the watermark 35 | */ 36 | Instant getWatermark(); 37 | 38 | /** 39 | * method to get the headers information of the payload 40 | * 41 | * @return returns the headers in the form of key value pair 42 | */ 43 | Map getHeaders(); 44 | 45 | /** 46 | * method to get the ID of the payload, ID is used for deduplication 47 | * 48 | * @return returns the ID of the payload 49 | */ 50 | String getID(); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/model/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator.model; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | import lombok.Getter; 6 | 7 | /** Message is used to wrap the data returned by Accumulator functions. */ 8 | @Getter 9 | public class Message { 10 | private final Instant eventTime; 11 | private final Instant watermark; 12 | private final Map headers; 13 | private final String id; 14 | private String[] keys; 15 | private byte[] value; 16 | private String[] tags; 17 | 18 | /** 19 | * Constructor for constructing message from Datum, it is advised to use the incoming Datum to 20 | * construct the message, because event time, watermark, id and headers of the message are derived 21 | * from the Datum. Only use custom implementation of the Datum if you know what you are doing. 22 | * 23 | * @param datum {@link Datum} object 24 | */ 25 | public Message(Datum datum) { 26 | this.keys = datum.getKeys(); 27 | this.value = datum.getValue(); 28 | this.headers = datum.getHeaders(); 29 | this.eventTime = datum.getEventTime(); 30 | this.watermark = datum.getWatermark(); 31 | this.id = datum.getID(); 32 | this.tags = null; 33 | } 34 | 35 | /* 36 | * sets the value of the message 37 | * 38 | * @param value byte array of the value 39 | */ 40 | public void setValue(byte[] value) { 41 | this.value = value; 42 | } 43 | 44 | /* 45 | * sets the keys of the message 46 | * 47 | * @param keys string array of the keys 48 | */ 49 | public void setKeys(String[] keys) { 50 | this.keys = keys; 51 | } 52 | 53 | /* 54 | * sets the tags of the message, tags are used for conditional forwarding 55 | * 56 | * @param tags string array of the tags 57 | */ 58 | public void setTags(String[] tags) { 59 | this.tags = tags; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/accumulator/model/OutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator.model; 2 | 3 | /** 4 | * OutputStreamObserver can be used to send the output messages to the next 5 | * stage of the flow inside the {@link Accumulator}. 6 | */ 7 | public interface OutputStreamObserver { 8 | /** 9 | * method will be used for sending messages to the output stream. 10 | * 11 | * @param message the {@link Message} to be sent 12 | */ 13 | void send(Message message); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/BatchMapper.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | /** 4 | * BatchMapper exposes method for performing batch map operation. 5 | * Implementations should override the processMessage method 6 | * which will be used for processing the input messages 7 | */ 8 | 9 | public abstract class BatchMapper { 10 | /** 11 | * method which will be used for processing messages. Please implement the interface to ensure that each message generates a corresponding BatchResponse object with a matching ID. 12 | * 13 | * @param datumStream current message to be processed 14 | * 15 | * @return BatchResponses which contains output from batch map 16 | */ 17 | public abstract BatchResponses processMessage(DatumIterator datumStream); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/BatchResponse.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import lombok.Getter; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | /** 9 | * BatchResponse is used to collect and manage a batch of Message objects. 10 | */ 11 | public class BatchResponse { 12 | @Getter 13 | private final String id; 14 | private final List messages; 15 | 16 | /** 17 | * Constructs a BatchResponse with a specified ID. 18 | * 19 | * @param id the unique identifier for this batch response 20 | */ 21 | public BatchResponse(String id) { 22 | this.id = id; 23 | this.messages = new ArrayList<>(); 24 | } 25 | 26 | /** 27 | * Appends a Message to the batch. 28 | * 29 | * @param msg the Message to be added to the batch 30 | * 31 | * @return the current BatchResponse instance for method chaining 32 | */ 33 | public BatchResponse append(Message msg) { 34 | this.messages.add(msg); 35 | return this; 36 | } 37 | 38 | /** 39 | * Retrieves the list of Messages in the batch. 40 | * 41 | * @return the list of Messages 42 | */ 43 | public List getItems() { 44 | return messages; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/BatchResponses.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * BatchResponses is used to send a response from the batch map functions. 8 | * It contains a list of BatchResponse objects. 9 | */ 10 | public class BatchResponses { 11 | private final List batchResponses; 12 | 13 | /** 14 | * Constructs an empty BatchResponses object. 15 | */ 16 | public BatchResponses() { 17 | this.batchResponses = new ArrayList<>(); 18 | } 19 | 20 | /** 21 | * Appends a BatchResponse to the list of batchResponses. 22 | * 23 | * @param batchResponse the BatchResponse to be added 24 | * 25 | * @return the current BatchResponses object 26 | */ 27 | public BatchResponses append(BatchResponse batchResponse) { 28 | this.batchResponses.add(batchResponse); 29 | return this; 30 | } 31 | 32 | /** 33 | * Retrieves the list of BatchResponse objects. 34 | * 35 | * @return the list of BatchResponse objects 36 | */ 37 | public List getItems() { 38 | return batchResponses; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | class Constants { 4 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 5 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/batchmap.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = "/var/run/numaflow/mapper-server-info"; 7 | public static final int DEFAULT_PORT = 50051; 8 | public static final String DEFAULT_HOST = "localhost"; 9 | public static final String SUCCESS = "SUCCESS"; 10 | public static final String MAP_MODE_KEY = "MAP_MODE"; 11 | public static final String MAP_MODE = "batch-map"; 12 | 13 | // Private constructor to prevent instantiation 14 | private Constants() { 15 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | 6 | /** 7 | * Datum contains methods to get the payload information. 8 | */ 9 | public interface Datum { 10 | /** 11 | * method to get the payload keys 12 | * 13 | * @return returns the datum keys. 14 | */ 15 | String[] getKeys(); 16 | 17 | /** 18 | * method to get the payload value 19 | * 20 | * @return returns the payload value in byte array 21 | */ 22 | byte[] getValue(); 23 | 24 | /** 25 | * method to get the event time of the payload 26 | * 27 | * @return returns the event time of the payload 28 | */ 29 | Instant getEventTime(); 30 | 31 | /** 32 | * method to get the watermark information 33 | * 34 | * @return returns the watermark 35 | */ 36 | Instant getWatermark(); 37 | 38 | /** 39 | * method to get the ID for the Payload 40 | * 41 | * @return returns the ID 42 | */ 43 | String getId(); 44 | 45 | /** 46 | * method to get the headers information of the payload 47 | * 48 | * @return returns the headers in the form of key value pair 49 | */ 50 | Map getHeaders(); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/DatumIterator.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | 4 | /** 5 | * An iterator over a collection of {@link Datum} elements. 6 | * Passed to {@link BatchMapper#processMessage(DatumIterator)} method. 7 | */ 8 | public interface DatumIterator { 9 | 10 | /** 11 | * Returns the next element in the iterator 12 | * This method blocks until an element becomes available in the queue. 13 | * When EOF_DATUM is received, this method will return null and the iterator will be closed. 14 | * 15 | * @return the next element in the iterator, null if EOF_DATUM is received or the iterator is already closed 16 | * 17 | * @throws InterruptedException if the thread is interrupted while waiting for the next element 18 | */ 19 | Datum next() throws InterruptedException; 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/DatumIteratorImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | 5 | import java.util.concurrent.BlockingQueue; 6 | import java.util.concurrent.LinkedBlockingDeque; 7 | import java.util.concurrent.atomic.AtomicBoolean; 8 | import java.util.concurrent.atomic.AtomicInteger; 9 | 10 | /** 11 | * A thread-safe implementation of {@link DatumIterator}, backed by a blocking queue. 12 | */ 13 | @Slf4j 14 | class DatumIteratorImpl implements DatumIterator { 15 | private final BlockingQueue blockingQueue = new LinkedBlockingDeque<>(); 16 | private final AtomicBoolean closed = new AtomicBoolean(false); 17 | private final AtomicInteger counter = new AtomicInteger(0); // Keep Track of number of requests 18 | 19 | @Override 20 | public Datum next() throws InterruptedException { 21 | // if the iterator is closed, return null 22 | if (closed.get()) { 23 | return null; 24 | } 25 | Datum datum = blockingQueue.take(); 26 | // if EOF is received, close the iterator and return null 27 | if (datum == HandlerDatum.EOF_DATUM) { 28 | closed.set(true); 29 | return null; 30 | } 31 | return datum; 32 | } 33 | 34 | // blocking call, waits until the write operation is successful 35 | public void writeMessage(Datum datum) throws InterruptedException { 36 | blockingQueue.put(datum); 37 | counter.incrementAndGet(); 38 | } 39 | 40 | public int getCount() { 41 | return counter.get(); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for map gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | @AllArgsConstructor 10 | class HandlerDatum implements Datum { 11 | 12 | static final HandlerDatum EOF_DATUM = new HandlerDatum(null, null, null, null, null, null); 13 | private String[] keys; 14 | private byte[] value; 15 | private Instant watermark; 16 | private Instant eventTime; 17 | private String id; 18 | private Map headers; 19 | 20 | @Override 21 | public String[] getKeys() { 22 | return keys; 23 | } 24 | 25 | @Override 26 | public Instant getWatermark() { 27 | return this.watermark; 28 | } 29 | 30 | @Override 31 | public byte[] getValue() { 32 | return this.value; 33 | } 34 | 35 | @Override 36 | public Instant getEventTime() { 37 | return this.eventTime; 38 | } 39 | 40 | @Override 41 | public String getId() { 42 | return id; 43 | } 44 | 45 | @Override 46 | public Map getHeaders() { 47 | return this.headers; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/batchmapper/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import lombok.Getter; 4 | 5 | /** Message is used to wrap the data returned by Mapper. */ 6 | @Getter 7 | public class Message { 8 | private static final String[] DROP_TAGS = {"U+005C__DROP__"}; 9 | private final String[] keys; 10 | private final byte[] value; 11 | private final String[] tags; 12 | 13 | /** 14 | * used to create Message with value, keys and tags(used for conditional forwarding) 15 | * 16 | * @param value message value 17 | * @param keys message keys 18 | * @param tags message tags which will be used for conditional forwarding 19 | */ 20 | public Message(byte[] value, String[] keys, String[] tags) { 21 | // defensive copy - once the Message is created, the caller should not be able to modify it. 22 | this.keys = keys == null ? null : keys.clone(); 23 | this.value = value == null ? null : value.clone(); 24 | this.tags = tags == null ? null : tags.clone(); 25 | } 26 | 27 | /** 28 | * used to create Message with value. 29 | * 30 | * @param value message value 31 | */ 32 | public Message(byte[] value) { 33 | this(value, null, null); 34 | } 35 | 36 | /** 37 | * used to create Message with value and keys. 38 | * 39 | * @param value message value 40 | * @param keys message keys 41 | */ 42 | public Message(byte[] value, String[] keys) { 43 | this(value, keys, null); 44 | } 45 | 46 | /** 47 | * creates a Message which will be dropped 48 | * 49 | * @return returns the Message which will be dropped 50 | */ 51 | public static Message toDrop() { 52 | return new Message(new byte[0], null, DROP_TAGS); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/info/ContainerType.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.info; 2 | 3 | import com.fasterxml.jackson.annotation.JsonValue; 4 | 5 | public enum ContainerType { 6 | SOURCER("sourcer"), 7 | SOURCE_TRANSFORMER("sourcetransformer"), 8 | SINKER("sinker"), 9 | MAPPER("mapper"), 10 | REDUCER("reducer"), 11 | REDUCE_STREAMER("reducestreamer"), 12 | SESSION_REDUCER("sessionreducer"), 13 | SIDEINPUT("sideinput"), 14 | FBSINKER("fb-sinker"), 15 | SERVING("serving"), 16 | ACCUMULATOR("accumulator"); 17 | 18 | private final String name; 19 | 20 | ContainerType(String name) { 21 | this.name = name; 22 | } 23 | 24 | @JsonValue 25 | public String getName() { 26 | return name; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/info/Language.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.info; 2 | 3 | import com.fasterxml.jackson.annotation.JsonValue; 4 | 5 | /** 6 | * Please exercise cautions when updating the values below because the exact same values are defined in other Numaflow SDKs 7 | * to form a contract between server and clients. 8 | */ 9 | public enum Language { 10 | JAVA("java"); 11 | 12 | private final String name; 13 | 14 | Language(String name) { 15 | this.name = name; 16 | } 17 | 18 | @JsonValue 19 | public String getName() { 20 | return name; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/info/Protocol.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.info; 2 | 3 | import com.fasterxml.jackson.annotation.JsonValue; 4 | 5 | /** 6 | * Please exercise cautions when updating the values below because the exact same values are defined in other Numaflow SDKs 7 | * to form a contract between server and clients. 8 | */ 9 | public enum Protocol { 10 | UDS_PROTOCOL("uds"), 11 | TCP_PROTOCOL("tcp"); 12 | 13 | private final String name; 14 | 15 | Protocol(String name) { 16 | this.name = name; 17 | } 18 | 19 | @JsonValue 20 | public String getName() { 21 | return name; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/info/ServerInfoAccessor.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.info; 2 | 3 | public interface ServerInfoAccessor { 4 | /** 5 | * Get current runtime numaflow-java SDK version. 6 | */ 7 | String getSDKVersion(); 8 | 9 | /** 10 | * Delete filePath if it exists. 11 | * Write serverInfo to filePath in Json format. 12 | * 13 | * @param serverInfo server information POJO 14 | * @param filePath file path to write to 15 | * 16 | * @throws Exception any exceptions are thrown to the caller. 17 | */ 18 | void write(ServerInfo serverInfo, String filePath) throws Exception; 19 | 20 | /** 21 | * Read from filePath to retrieve server information POJO. 22 | * This API is only used for unit tests. 23 | * 24 | * @param filePath file path to read from 25 | * 26 | * @return server information POJO 27 | * 28 | * @throws Exception any exceptions are thrown to the caller. 29 | */ 30 | ServerInfo read(String filePath) throws Exception; 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapper/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | class Constants { 4 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 5 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/map.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = "/var/run/numaflow/mapper-server-info"; 7 | public static final int DEFAULT_PORT = 50051; 8 | public static final String DEFAULT_HOST = "localhost"; 9 | public static final String MAP_MODE_KEY = "MAP_MODE"; 10 | public static final String MAP_MODE = "unary-map"; 11 | public static final String EOF = "EOF"; 12 | 13 | // Private constructor to prevent instantiation 14 | private Constants() { 15 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapper/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | 4 | import java.time.Instant; 5 | import java.util.Map; 6 | 7 | /** 8 | * Datum contains methods to get the payload information. 9 | */ 10 | 11 | public interface Datum { 12 | /** 13 | * method to get the payload value 14 | * 15 | * @return returns the payload value in byte array 16 | */ 17 | byte[] getValue(); 18 | 19 | /** 20 | * method to get the event time of the payload 21 | * 22 | * @return returns the event time of the payload 23 | */ 24 | Instant getEventTime(); 25 | 26 | /** 27 | * method to get the watermark information 28 | * 29 | * @return returns the watermark 30 | */ 31 | Instant getWatermark(); 32 | 33 | /** 34 | * method to get the headers information of the payload 35 | * 36 | * @return returns the headers in the form of key value pair 37 | */ 38 | Map getHeaders(); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapper/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for map gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") == null) // if NUMAFLOW_POD is not set, then we are not running 35 | // using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapper/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | @AllArgsConstructor 10 | class HandlerDatum implements Datum { 11 | 12 | private byte[] value; 13 | private Instant watermark; 14 | private Instant eventTime; 15 | private Map headers; 16 | 17 | 18 | @Override 19 | public Instant getWatermark() { 20 | return this.watermark; 21 | } 22 | 23 | @Override 24 | public byte[] getValue() { 25 | return this.value; 26 | } 27 | 28 | @Override 29 | public Instant getEventTime() { 30 | return this.eventTime; 31 | } 32 | 33 | @Override 34 | public Map getHeaders() { 35 | return this.headers; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapper/Mapper.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | /** 4 | * Mapper exposes method for performing map operation. 5 | * Implementations should override the processMessage method 6 | * which will be used for processing the input messages 7 | */ 8 | 9 | public abstract class Mapper { 10 | /** 11 | * method which will be used for processing messages. 12 | * 13 | * @param keys message keys 14 | * @param datum current message to be processed 15 | * 16 | * @return MessageList which contains output from map 17 | */ 18 | public abstract MessageList processMessage(String[] keys, Datum datum); 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapper/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | import lombok.Getter; 4 | 5 | /** Message is used to wrap the data returned by Mapper. */ 6 | @Getter 7 | public class Message { 8 | private static final String[] DROP_TAGS = {"U+005C__DROP__"}; 9 | private final String[] keys; 10 | private final byte[] value; 11 | private final String[] tags; 12 | 13 | /** 14 | * used to create Message with value, keys and tags(used for conditional forwarding) 15 | * 16 | * @param value message value 17 | * @param keys message keys 18 | * @param tags message tags which will be used for conditional forwarding 19 | */ 20 | public Message(byte[] value, String[] keys, String[] tags) { 21 | // defensive copy - once the Message is created, the caller should not be able to modify it. 22 | this.keys = keys == null ? null : keys.clone(); 23 | this.value = value == null ? null : value.clone(); 24 | this.tags = tags == null ? null : tags.clone(); 25 | } 26 | 27 | /** 28 | * used to create Message with value. 29 | * 30 | * @param value message value 31 | */ 32 | public Message(byte[] value) { 33 | this(value, null, null); 34 | } 35 | 36 | /** 37 | * used to create Message with value and keys. 38 | * 39 | * @param value message value 40 | * @param keys message keys 41 | */ 42 | public Message(byte[] value, String[] keys) { 43 | this(value, keys, null); 44 | } 45 | 46 | /** 47 | * creates a Message which will be dropped 48 | * 49 | * @return returns the Message which will be dropped 50 | */ 51 | public static Message toDrop() { 52 | return new Message(new byte[0], null, DROP_TAGS); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapper/MessageList.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import lombok.Builder; 6 | import lombok.Getter; 7 | import lombok.Singular; 8 | 9 | /** MessageList is used to return the list of Messages returned from Map functions. */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class MessageList { 13 | 14 | @Singular("addMessage") 15 | private List messages; 16 | 17 | /** Builder to build MessageList */ 18 | public static class MessageListBuilder { 19 | /** 20 | * @param messages to append all the messages to MessageList 21 | * @return returns the builder 22 | */ 23 | public MessageListBuilder addMessages(Iterable messages) { 24 | if (this.messages == null) { 25 | this.messages = new ArrayList<>(); 26 | } 27 | 28 | for (Message message : messages) { 29 | this.messages.add(message); 30 | } 31 | return this; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapstreamer/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | class Constants { 4 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 5 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/mapstream.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = "/var/run/numaflow/mapper-server-info"; 7 | public static final int DEFAULT_PORT = 50051; 8 | public static final String DEFAULT_HOST = "localhost"; 9 | public static final String MAP_MODE_KEY = "MAP_MODE"; 10 | public static final String MAP_MODE = "stream-map"; 11 | 12 | // Private constructor to prevent instantiation 13 | private Constants() { 14 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapstreamer/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | 6 | /** 7 | * Datum contains methods to get the payload information. 8 | */ 9 | 10 | public interface Datum { 11 | /** 12 | * method to get the payload value 13 | * 14 | * @return returns the payload value in byte array 15 | */ 16 | byte[] getValue(); 17 | 18 | /** 19 | * method to get the event time of the payload 20 | * 21 | * @return returns the event time of the payload 22 | */ 23 | Instant getEventTime(); 24 | 25 | /** 26 | * method to get the watermark information 27 | * 28 | * @return returns the watermark 29 | */ 30 | Instant getWatermark(); 31 | 32 | /** 33 | * method to get the headers information of the payload 34 | * 35 | * @return returns the headers in the form of key value pair 36 | */ 37 | Map getHeaders(); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapstreamer/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapstreamer/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | @AllArgsConstructor 10 | class HandlerDatum implements Datum { 11 | 12 | private byte[] value; 13 | private Instant watermark; 14 | private Instant eventTime; 15 | private Map headers; 16 | 17 | 18 | @Override 19 | public Instant getWatermark() { 20 | return this.watermark; 21 | } 22 | 23 | @Override 24 | public byte[] getValue() { 25 | return this.value; 26 | } 27 | 28 | @Override 29 | public Instant getEventTime() { 30 | return this.eventTime; 31 | } 32 | 33 | @Override 34 | public Map getHeaders() { 35 | return this.headers; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapstreamer/MapStreamer.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | /** 4 | * MapStreamer exposes method for performing map streaming operation. 5 | * Implementations should override the processMessage method 6 | * which will be used for processing the input messages 7 | */ 8 | 9 | public abstract class MapStreamer { 10 | /** 11 | * method which will be used for processing streaming messages. 12 | * 13 | * @param keys message keys 14 | * @param datum current message to be processed 15 | * @param outputObserver observer of the response 16 | */ 17 | public abstract void processMessage(String[] keys, Datum datum, OutputObserver outputObserver); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapstreamer/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | import lombok.Getter; 4 | 5 | /** Message is used to wrap the data returned by MapStreamer. */ 6 | @Getter 7 | public class Message { 8 | private static final String[] DROP_TAGS = {"U+005C__DROP__"}; 9 | private final String[] keys; 10 | private final byte[] value; 11 | private final String[] tags; 12 | 13 | /** 14 | * used to create Message with value, keys and tags(used for conditional forwarding) 15 | * 16 | * @param value message value 17 | * @param keys message keys 18 | * @param tags message tags which will be used for conditional forwarding 19 | */ 20 | public Message(byte[] value, String[] keys, String[] tags) { 21 | // defensive copy - once the Message is created, the caller should not be able to modify it. 22 | this.keys = keys == null ? null : keys.clone(); 23 | this.value = value == null ? null : value.clone(); 24 | this.tags = tags == null ? null : tags.clone(); 25 | } 26 | 27 | /** 28 | * used to create Message with value. 29 | * 30 | * @param value message value 31 | */ 32 | public Message(byte[] value) { 33 | this(value, null, null); 34 | } 35 | 36 | /** 37 | * used to create Message with value and keys. 38 | * 39 | * @param value message value 40 | * @param keys message keys 41 | */ 42 | public Message(byte[] value, String[] keys) { 43 | this(value, keys, null); 44 | } 45 | 46 | /** 47 | * creates a Message which will be dropped 48 | * 49 | * @return returns the Message which will be dropped 50 | */ 51 | public static Message toDrop() { 52 | return new Message(new byte[0], null, DROP_TAGS); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/mapstreamer/OutputObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | 4 | /** 5 | * OutputObserver receives messages from the MapStreamer. 6 | */ 7 | public interface OutputObserver { 8 | /** 9 | * method will be used for sending messages to the output. 10 | * 11 | * @param message the message to be sent 12 | */ 13 | void send(Message message); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/ActorRequest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import io.numaproj.numaflow.reduce.v1.ReduceOuterClass; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | 7 | /** 8 | * ActorRequest is to store the request sent to ReduceActors. 9 | */ 10 | @Getter 11 | @AllArgsConstructor 12 | class ActorRequest { 13 | ReduceOuterClass.ReduceRequest request; 14 | 15 | // TODO - do we need to include window information in the id? 16 | // for aligned reducer, there is always single window. 17 | // but at the same time, would like to be consistent with GO SDK implementation. 18 | // we will revisit this one later. 19 | public String getUniqueIdentifier() { 20 | return String.join( 21 | Constants.DELIMITER, 22 | this.getRequest().getPayload().getKeysList().toArray(new String[0])); 23 | } 24 | 25 | public String[] getKeySet() { 26 | return this.getRequest().getPayload().getKeysList().toArray(new String[0]); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/ActorResponse.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import io.numaproj.numaflow.reduce.v1.ReduceOuterClass; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | 7 | /** 8 | * ActorResponse is to store the response from ReduceActors. 9 | */ 10 | @Getter 11 | @AllArgsConstructor 12 | class ActorResponse { 13 | ReduceOuterClass.ReduceResponse response; 14 | 15 | // TODO - do we need to include window information in the id? 16 | // for aligned reducer, there is always single window. 17 | // but at the same time, would like to be consistent with GO SDK implementation. 18 | // we will revisit this one later. 19 | public String getUniqueIdentifier() { 20 | return String.join( 21 | Constants.DELIMITER, 22 | this.getResponse().getResult().getKeysList().toArray(new String[0])); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | class Constants { 4 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 5 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/reduce.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = 7 | "/var/run/numaflow/reducer-server-info"; 8 | public static final int DEFAULT_PORT = 50051; 9 | public static final String DEFAULT_HOST = "localhost"; 10 | public static final String EOF = "EOF"; 11 | public static final String SUCCESS = "SUCCESS"; 12 | public static final String DELIMITER = ":"; 13 | 14 | // Private constructor to prevent instantiation 15 | private Constants() { 16 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | 6 | /** 7 | * Datum contains methods to get the payload information. 8 | */ 9 | 10 | public interface Datum { 11 | /** 12 | * method to get the payload value 13 | * 14 | * @return returns the payload value in byte array 15 | */ 16 | byte[] getValue(); 17 | 18 | /** 19 | * method to get the event time of the payload 20 | * 21 | * @return returns the event time of the payload 22 | */ 23 | Instant getEventTime(); 24 | 25 | /** 26 | * method to get the watermark information 27 | * 28 | * @return returns the watermark 29 | */ 30 | Instant getWatermark(); 31 | 32 | /** 33 | * method to get the headers information of the payload 34 | * 35 | * @return returns the headers in the form of key value pair 36 | */ 37 | Map getHeaders(); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | @AllArgsConstructor 10 | class HandlerDatum implements Datum { 11 | private byte[] value; 12 | private Instant watermark; 13 | private Instant eventTime; 14 | private Map headers; 15 | 16 | @Override 17 | public Instant getWatermark() { 18 | return this.watermark; 19 | } 20 | 21 | @Override 22 | public byte[] getValue() { 23 | return this.value; 24 | } 25 | 26 | @Override 27 | public Instant getEventTime() { 28 | return this.eventTime; 29 | } 30 | 31 | @Override 32 | public Map getHeaders() { 33 | return this.headers; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/IntervalWindow.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import java.time.Instant; 4 | 5 | /** 6 | * IntervalWindow contains methods to get the information for a given interval window. 7 | */ 8 | public interface IntervalWindow { 9 | /** 10 | * method to get the start time of the interval window 11 | * 12 | * @return start time of the window 13 | */ 14 | Instant getStartTime(); 15 | 16 | /** 17 | * method to get the end time of the interval window 18 | * 19 | * @return end time of the window 20 | */ 21 | Instant getEndTime(); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import lombok.Getter; 4 | 5 | /** Message is used to wrap the data returned by Reducer functions. */ 6 | @Getter 7 | public class Message { 8 | private static final String[] DROP_TAGS = {"U+005C__DROP__"}; 9 | private final String[] keys; 10 | private final byte[] value; 11 | private final String[] tags; 12 | 13 | /** 14 | * used to create Message with value, keys and tags(used for conditional forwarding) 15 | * 16 | * @param value message value 17 | * @param keys message keys 18 | * @param tags message tags which will be used for conditional forwarding 19 | */ 20 | public Message(byte[] value, String[] keys, String[] tags) { 21 | // defensive copy - once the Message is created, the caller should not be able to modify it. 22 | this.keys = keys == null ? null : keys.clone(); 23 | this.value = value == null ? null : value.clone(); 24 | this.tags = tags == null ? null : tags.clone(); 25 | } 26 | 27 | /** 28 | * used to create Message with value. 29 | * 30 | * @param value message value 31 | */ 32 | public Message(byte[] value) { 33 | this(value, null, null); 34 | } 35 | 36 | /** 37 | * used to create Message with value and keys. 38 | * 39 | * @param value message value 40 | * @param keys message keys 41 | */ 42 | public Message(byte[] value, String[] keys) { 43 | this(value, keys, null); 44 | } 45 | 46 | /** 47 | * creates a Message which will be dropped 48 | * 49 | * @return returns the Message which will be dropped 50 | */ 51 | public static Message toDrop() { 52 | return new Message(new byte[0], null, DROP_TAGS); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/MessageList.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import lombok.Builder; 6 | import lombok.Getter; 7 | import lombok.Singular; 8 | 9 | @Getter 10 | @Builder(builderMethodName = "newBuilder") 11 | public class MessageList { 12 | 13 | @Singular("addMessage") 14 | private List messages; 15 | 16 | /** Builder to build MessageList */ 17 | public static class MessageListBuilder { 18 | /** 19 | * @param messages to append all the messages to MessageList 20 | * @return returns the builder 21 | */ 22 | public MessageListBuilder addMessages(Iterable messages) { 23 | if (this.messages == null) { 24 | this.messages = new ArrayList<>(); 25 | } 26 | 27 | for (Message message : messages) { 28 | this.messages.add(message); 29 | } 30 | return this; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/Metadata.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | /** 4 | * Metadata contains methods to get the metadata for the reduce operation. 5 | */ 6 | public interface Metadata { 7 | /** 8 | * method to get the interval window. 9 | * 10 | * @return IntervalWindow which has the window information 11 | */ 12 | IntervalWindow getIntervalWindow(); 13 | } 14 | 15 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/Reducer.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | 4 | /** 5 | * Reducer exposes methods for performing reduce operation. 6 | */ 7 | 8 | 9 | public abstract class Reducer { 10 | /** 11 | * addMessage will be invoked for each input message. 12 | * It can be used to read the input data from datum and 13 | * update the result for given keys. 14 | * 15 | * @param keys message keys 16 | * @param datum current message to be processed 17 | * @param md metadata which contains window information 18 | */ 19 | public abstract void addMessage(String[] keys, Datum datum, Metadata md); 20 | 21 | /** 22 | * getOutput will be invoked at the end of input. 23 | * 24 | * @param keys message keys 25 | * @param md metadata which contains window information 26 | * 27 | * @return MessageList output value, aggregated result 28 | */ 29 | public abstract MessageList getOutput(String[] keys, Metadata md); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/ReducerFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | 4 | /** 5 | * ReducerFactory is the factory for Reducer. 6 | */ 7 | 8 | public abstract class ReducerFactory { 9 | public abstract ReducerT createReducer(); 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/metadata/IntervalWindowImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer.metadata; 2 | 3 | import io.numaproj.numaflow.reducer.IntervalWindow; 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | 8 | /** 9 | * IntervalWindowImpl implements IntervalWindow interface which will be passed as metadata to reduce 10 | * handlers 11 | */ 12 | @AllArgsConstructor 13 | public class IntervalWindowImpl implements IntervalWindow { 14 | private final Instant startTime; 15 | private final Instant endTime; 16 | 17 | @Override 18 | public Instant getStartTime() { 19 | return this.startTime; 20 | } 21 | 22 | @Override 23 | public Instant getEndTime() { 24 | return this.endTime; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducer/metadata/MetadataImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer.metadata; 2 | 3 | import io.numaproj.numaflow.reducer.IntervalWindow; 4 | import io.numaproj.numaflow.reducer.Metadata; 5 | import lombok.AllArgsConstructor; 6 | 7 | /** 8 | * MetadataImpl implements Metadata interface which will be passed to reduce handlers 9 | */ 10 | @AllArgsConstructor 11 | public class MetadataImpl implements Metadata { 12 | private final IntervalWindow intervalWindow; 13 | 14 | @Override 15 | public IntervalWindow getIntervalWindow() { 16 | return intervalWindow; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/ActorRequest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import io.numaproj.numaflow.reduce.v1.ReduceOuterClass; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | 7 | /** 8 | * ActorRequest is a wrapper of the gRpc input request. 9 | * It is constructed by the service when service receives an input request and then sent to 10 | * the supervisor actor, to be distributed to reduce streamer actors. 11 | */ 12 | @Getter 13 | @AllArgsConstructor 14 | class ActorRequest { 15 | ReduceOuterClass.ReduceRequest request; 16 | 17 | // TODO - do we need to include window information in the id? 18 | // for aligned reducer, there is always single window. 19 | // but at the same time, would like to be consistent with GO SDK implementation. 20 | // we will revisit this one later. 21 | public String getUniqueIdentifier() { 22 | return String.join( 23 | Constants.DELIMITER, 24 | this.getRequest().getPayload().getKeysList().toArray(new String[0])); 25 | } 26 | 27 | public String[] getKeySet() { 28 | return this.getRequest().getPayload().getKeysList().toArray(new String[0]); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/ActorResponse.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import io.numaproj.numaflow.reduce.v1.ReduceOuterClass; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | 8 | /** 9 | * The actor response holds the final EOF response for a particular key set. 10 | */ 11 | @Getter 12 | @Setter 13 | @AllArgsConstructor 14 | class ActorResponse { 15 | ReduceOuterClass.ReduceResponse response; 16 | 17 | // TODO - do we need to include window information in the id? 18 | // for aligned reducer, there is always single window. 19 | // but at the same time, would like to be consistent with GO SDK implementation. 20 | // we will revisit this one later. 21 | public String getActorUniqueIdentifier() { 22 | return String.join( 23 | Constants.DELIMITER, 24 | this.getResponse().getResult().getKeysList().toArray(new String[0])); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | class Constants { 4 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 5 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/reducestream.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = 7 | "/var/run/numaflow/reducestreamer-server-info"; 8 | public static final int DEFAULT_PORT = 50051; 9 | public static final String DEFAULT_HOST = "localhost"; 10 | public static final String EOF = "EOF"; 11 | public static final String SUCCESS = "SUCCESS"; 12 | public static final String DELIMITER = ":"; 13 | 14 | // Private constructor to prevent instantiation 15 | private Constants() { 16 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | @Builder.Default 16 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 17 | 18 | @Builder.Default 19 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 20 | 21 | @Builder.Default 22 | private int port = Constants.DEFAULT_PORT; 23 | 24 | private boolean isLocal; 25 | 26 | /** 27 | * Static method to create default GRPCConfig. 28 | */ 29 | static GRPCConfig defaultGrpcConfig() { 30 | return GRPCConfig.newBuilder() 31 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 32 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 33 | .isLocal(System.getenv("NUMAFLOW_POD") 34 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 35 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import io.numaproj.numaflow.reducestreamer.model.Datum; 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | @AllArgsConstructor 10 | class HandlerDatum implements Datum { 11 | private byte[] value; 12 | private Instant watermark; 13 | private Instant eventTime; 14 | private Map headers; 15 | 16 | @Override 17 | public Instant getWatermark() { 18 | return this.watermark; 19 | } 20 | 21 | @Override 22 | public byte[] getValue() { 23 | return this.value; 24 | } 25 | 26 | @Override 27 | public Instant getEventTime() { 28 | return this.eventTime; 29 | } 30 | 31 | @Override 32 | public Map getHeaders() { 33 | return this.headers; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/IntervalWindowImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import io.numaproj.numaflow.reducestreamer.model.IntervalWindow; 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | 8 | @AllArgsConstructor 9 | class IntervalWindowImpl implements IntervalWindow { 10 | private final Instant startTime; 11 | private final Instant endTime; 12 | 13 | @Override 14 | public Instant getStartTime() { 15 | return this.startTime; 16 | } 17 | 18 | @Override 19 | public Instant getEndTime() { 20 | return this.endTime; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/MetadataImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import io.numaproj.numaflow.reducestreamer.model.IntervalWindow; 4 | import io.numaproj.numaflow.reducestreamer.model.Metadata; 5 | import lombok.AllArgsConstructor; 6 | 7 | @AllArgsConstructor 8 | class MetadataImpl implements Metadata { 9 | private final IntervalWindow intervalWindow; 10 | 11 | @Override 12 | public IntervalWindow getIntervalWindow() { 13 | return intervalWindow; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/OutputActor.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import akka.actor.AbstractActor; 4 | import akka.actor.Props; 5 | import io.grpc.stub.StreamObserver; 6 | import io.numaproj.numaflow.reduce.v1.ReduceOuterClass; 7 | import lombok.AllArgsConstructor; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | /** 11 | * Output actor is a wrapper around the gRPC output stream. 12 | * It ensures synchronized calls to the responseObserver onNext() and invokes onComplete at the end of the stream. 13 | * ALL reduce responses are sent to the output actor before getting forwarded to the output gRPC stream. 14 | *

15 | * More details about gRPC StreamObserver concurrency: https://grpc.github.io/grpc-java/javadoc/io/grpc/stub/StreamObserver.html 16 | */ 17 | @Slf4j 18 | @AllArgsConstructor 19 | class OutputActor extends AbstractActor { 20 | StreamObserver responseObserver; 21 | 22 | public static Props props( 23 | StreamObserver responseObserver) { 24 | return Props.create(OutputActor.class, responseObserver); 25 | } 26 | 27 | @Override 28 | public Receive createReceive() { 29 | return receiveBuilder() 30 | .match(ActorResponse.class, this::handleResponse) 31 | .build(); 32 | } 33 | 34 | private void handleResponse(ActorResponse actorResponse) { 35 | if (actorResponse.getResponse().getEOF()) { 36 | // send the very last response. 37 | responseObserver.onNext(actorResponse.getResponse()); 38 | // close the output stream. 39 | responseObserver.onCompleted(); 40 | // stop the AKKA system right after we close the output stream. 41 | // note: could make more sense if the supervisor actor stops the system, 42 | // but it requires an extra tell. 43 | getContext().getSystem().stop(getSender()); 44 | } else { 45 | responseObserver.onNext(actorResponse.getResponse()); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/model/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer.model; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | 6 | /** 7 | * Datum contains methods to get the payload information. 8 | */ 9 | public interface Datum { 10 | /** 11 | * method to get the payload value 12 | * 13 | * @return returns the payload value in byte array 14 | */ 15 | byte[] getValue(); 16 | 17 | /** 18 | * method to get the event time of the payload 19 | * 20 | * @return returns the event time of the payload 21 | */ 22 | Instant getEventTime(); 23 | 24 | /** 25 | * method to get the watermark information 26 | * 27 | * @return returns the watermark 28 | */ 29 | Instant getWatermark(); 30 | 31 | /** 32 | * method to get the headers information of the payload 33 | * 34 | * @return returns the headers in the form of key value pair 35 | */ 36 | Map getHeaders(); 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/model/IntervalWindow.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer.model; 2 | 3 | import java.time.Instant; 4 | 5 | /** 6 | * IntervalWindow contains methods to get the information for a given interval window. 7 | */ 8 | public interface IntervalWindow { 9 | /** 10 | * method to get the start time of the interval window 11 | * 12 | * @return start time of the window 13 | */ 14 | Instant getStartTime(); 15 | 16 | /** 17 | * method to get the end time of the interval window 18 | * 19 | * @return end time of the window 20 | */ 21 | Instant getEndTime(); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/model/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer.model; 2 | 3 | import lombok.Getter; 4 | 5 | /** Message is used to wrap the data returned by Reduce Streamer functions. */ 6 | @Getter 7 | public class Message { 8 | private static final String[] DROP_TAGS = {"U+005C__DROP__"}; 9 | private final String[] keys; 10 | private final byte[] value; 11 | private final String[] tags; 12 | 13 | /** 14 | * used to create Message with value, keys and tags(used for conditional forwarding) 15 | * 16 | * @param value message value 17 | * @param keys message keys 18 | * @param tags message tags which will be used for conditional forwarding 19 | */ 20 | public Message(byte[] value, String[] keys, String[] tags) { 21 | // defensive copy - once the Message is created, the caller should not be able to modify it. 22 | this.keys = keys == null ? null : keys.clone(); 23 | this.value = value == null ? null : value.clone(); 24 | this.tags = tags == null ? null : tags.clone(); 25 | } 26 | 27 | /** 28 | * used to create Message with value. 29 | * 30 | * @param value message value 31 | */ 32 | public Message(byte[] value) { 33 | this(value, null, null); 34 | } 35 | 36 | /** 37 | * used to create Message with value and keys. 38 | * 39 | * @param value message value 40 | * @param keys message keys 41 | */ 42 | public Message(byte[] value, String[] keys) { 43 | this(value, keys, null); 44 | } 45 | 46 | /** 47 | * creates a Message which will be dropped 48 | * 49 | * @return returns the Message which will be dropped 50 | */ 51 | public static Message toDrop() { 52 | return new Message(new byte[0], null, DROP_TAGS); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/model/Metadata.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer.model; 2 | 3 | /** 4 | * Metadata contains methods to get the metadata for the reduce stream operation. 5 | */ 6 | public interface Metadata { 7 | /** 8 | * method to get the interval window. 9 | * 10 | * @return IntervalWindow which has the window information 11 | */ 12 | IntervalWindow getIntervalWindow(); 13 | } 14 | 15 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/model/OutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer.model; 2 | 3 | /** 4 | * OutputStreamObserver sends to the output stream, the messages generate by the reduce streamer. 5 | */ 6 | public interface OutputStreamObserver { 7 | /** 8 | * method will be used for sending messages to the output stream. 9 | * 10 | * @param message the message to be sent 11 | */ 12 | void send(Message message); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/model/ReduceStreamer.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer.model; 2 | 3 | /** 4 | * ReduceStreamer exposes methods for performing reduce streaming operations. 5 | */ 6 | public abstract class ReduceStreamer { 7 | /** 8 | * processMessage is invoked for each reduce input message. 9 | * It reads the input data from the datum and performs reduce operations for the given keys. 10 | * An output stream is provided for sending back the result to the reduce output stream. 11 | * 12 | * @param keys message keys 13 | * @param datum current message to be processed 14 | * @param outputStream observer of the reduce result, which is used to send back reduce responses 15 | * @param md metadata associated with the window 16 | */ 17 | public abstract void processMessage( 18 | String[] keys, 19 | Datum datum, 20 | OutputStreamObserver outputStream, 21 | Metadata md); 22 | 23 | /** 24 | * handleEndOfStream handles the closure of the reduce input stream. 25 | * This method is invoked when the input reduce stream is closed. 26 | * It provides the capability of constructing final responses based on the messages processed so far. 27 | * 28 | * @param keys message keys 29 | * @param outputStreamObserver observer of the reduce result, which is used to send back reduce responses 30 | * @param md metadata associated with the window 31 | */ 32 | public abstract void handleEndOfStream( 33 | String[] keys, 34 | OutputStreamObserver outputStreamObserver, 35 | Metadata md); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/reducestreamer/model/ReduceStreamerFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer.model; 2 | 3 | /** 4 | * ReduceStreamerFactory is the factory for ReduceStreamer. 5 | */ 6 | public abstract class ReduceStreamerFactory { 7 | /** 8 | * Helper function to create a reduce streamer. 9 | * 10 | * @return a concrete reduce streamer instance 11 | */ 12 | public abstract ReduceStreamerT createReduceStreamer(); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | class Constants { 4 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 5 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/serving.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = 7 | "/var/run/numaflow/serving-server-info"; 8 | public static final int DEFAULT_PORT = 50051; 9 | public static final String DEFAULT_HOST = "localhost"; 10 | 11 | // Private constructor to prevent instantiation 12 | private Constants() { 13 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/GetDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | /** 4 | * GetDatum is the interface to expose methods to retrieve data from the Get RPC. 5 | */ 6 | public interface GetDatum { 7 | /** 8 | * Returns the ID. 9 | * 10 | * @return the ID 11 | */ 12 | String ID(); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/GetDatumImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | @AllArgsConstructor 6 | public class GetDatumImpl implements GetDatum { 7 | private final String id; 8 | 9 | /** 10 | * Returns the ID. 11 | * 12 | * @return the ID 13 | */ 14 | @Override 15 | public String ID() { 16 | return id; 17 | } 18 | } -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/Payload.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | import lombok.Getter; 4 | import lombok.AllArgsConstructor; 5 | 6 | /** 7 | * Payload is each independent result stored in the Store per origin for a given ID. 8 | */ 9 | @Getter 10 | @AllArgsConstructor 11 | public class Payload { 12 | // origin is the name of the vertex that produced the payload. 13 | private String origin; 14 | // value is the byte array of the payload. 15 | private byte[] value; 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/PutDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * PutDatum interface exposes methods to retrieve data from the Put RPC. 7 | */ 8 | public interface PutDatum { 9 | /** 10 | * Returns the ID. 11 | * 12 | * @return the ID 13 | */ 14 | String ID(); 15 | 16 | /** 17 | * Returns the list of payloads. 18 | * 19 | * @return the list of payloads 20 | */ 21 | List Payloads(); 22 | } -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/PutDatumImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | import java.util.List; 4 | 5 | import lombok.AllArgsConstructor; 6 | 7 | @AllArgsConstructor 8 | public class PutDatumImpl implements PutDatum { 9 | private final String id; 10 | private final List payloads; 11 | 12 | /** 13 | * Returns the ID. 14 | * 15 | * @return the ID 16 | */ 17 | @Override 18 | public String ID() { 19 | return id; 20 | } 21 | 22 | /** 23 | * Returns the list of payloads. 24 | * 25 | * @return the list of payloads 26 | */ 27 | @Override 28 | public List Payloads() { 29 | return payloads; 30 | } 31 | } -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/ServingStorer.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | /** 4 | * ServingStorer is the interface for serving store to store and retrieve from a 5 | * custom store. 6 | */ 7 | public abstract class ServingStorer { 8 | /** 9 | * Puts data into the Serving Store. 10 | * 11 | * @param putDatum the data to be put into the store 12 | */ 13 | public abstract void put(PutDatum putDatum); 14 | 15 | /** 16 | * Retrieves data from the Serving Store. 17 | * 18 | * @param getDatum the data to be retrieved from the store 19 | * 20 | * @return the stored result 21 | */ 22 | public abstract StoredResult get(GetDatum getDatum); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/servingstore/StoredResult.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * StoredResult is the data stored in the store per origin. 11 | */ 12 | @Getter 13 | @Builder 14 | @AllArgsConstructor 15 | public class StoredResult { 16 | // ID is the unique identifier for the StoredResult. 17 | private String id; 18 | // Payloads is the list of Payloads stored in the Store for the given ID. 19 | private List payloads; 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/ActorRequestType.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | /** 4 | * ActorRequestType represents the purpose of an ActorRequest. 5 | */ 6 | public enum ActorRequestType { 7 | // open a brand-new session window 8 | OPEN, 9 | // append a message to an existing session window 10 | APPEND, 11 | // close a session window 12 | CLOSE, 13 | // expand a session window 14 | EXPAND, 15 | // ask a to-be-merged session window for it's accumulator 16 | GET_ACCUMULATOR, 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/ActorResponse.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import io.numaproj.numaflow.sessionreduce.v1.Sessionreduce; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | 8 | /** 9 | * The actor response holds the session reduce response from a particular session window. 10 | */ 11 | @Getter 12 | @Setter 13 | class ActorResponse { 14 | Sessionreduce.SessionReduceResponse response; 15 | // The isLast attribute indicates whether the response is globally the last one to be sent to 16 | // the output gRPC stream, if set to true, it means the response is the very last response among 17 | // all windows. When output actor receives an isLast response, it sends the response to and immediately 18 | // closes the output stream. 19 | boolean isLast; 20 | // The accumulator attribute holds the accumulator of the session. 21 | byte[] accumulator; 22 | // The mergeTaskId attribute holds the id of the merged window that this session is to be merged into. 23 | String mergeTaskId; 24 | 25 | @Builder 26 | private ActorResponse( 27 | Sessionreduce.SessionReduceResponse response, 28 | boolean isLast, 29 | byte[] accumulator, 30 | String mergeTaskId 31 | ) { 32 | this.response = response; 33 | this.isLast = isLast; 34 | this.accumulator = accumulator; 35 | this.mergeTaskId = mergeTaskId; 36 | } 37 | 38 | boolean isEOFResponse() { 39 | return this.accumulator == null && this.mergeTaskId == null; 40 | } 41 | 42 | static class ActorResponseBuilder { 43 | ActorResponse build() { 44 | if ((accumulator != null && mergeTaskId == null) || (accumulator == null 45 | && mergeTaskId != null)) { 46 | throw new IllegalStateException( 47 | "attributes accumulator and mergeTaskId should be either both null or both non-null."); 48 | } 49 | return new ActorResponse(response, isLast, accumulator, mergeTaskId); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | class Constants { 4 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 5 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/sessionreduce.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = 7 | "/var/run/numaflow/sessionreducer-server-info"; 8 | public static final int DEFAULT_PORT = 50051; 9 | public static final String EOF = "EOF"; 10 | public static final String SUCCESS = "SUCCESS"; 11 | public static final String DEFAULT_HOST = "localhost"; 12 | public static final String DELIMITER = ":"; 13 | 14 | // Private constructor to prevent instantiation 15 | private Constants() { 16 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/GetAccumulatorRequest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | 6 | /** 7 | * GetAccumulatorRequest is sent by supervisor actor to inform a session reducer actor that 8 | * the window is to be merged with other windows. 9 | *

10 | * "I am working on a merge task (mergeTaskId), 11 | * and you are one of the windows to be merged. 12 | * Please give me your accumulator." 13 | */ 14 | @AllArgsConstructor 15 | @Getter 16 | class GetAccumulatorRequest { 17 | private final String mergeTaskId; 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/GetAccumulatorResponse.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import io.numaproj.numaflow.sessionreduce.v1.Sessionreduce; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GetAccumulatorResponse is sent from a session reducer actor back to the supervisor actor, 9 | * containing the accumulator of the session. 10 | *

11 | * "Hey supervisor, I am the session window fromKeyedWindow, 12 | * I am returning my accumulator so that you can merge me." 13 | */ 14 | @AllArgsConstructor 15 | @Getter 16 | class GetAccumulatorResponse { 17 | private final Sessionreduce.KeyedWindow fromKeyedWindow; 18 | private final byte[] accumulator; 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import io.numaproj.numaflow.sessionreducer.model.Datum; 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | @AllArgsConstructor 10 | class HandlerDatum implements Datum { 11 | private byte[] value; 12 | private Instant watermark; 13 | private Instant eventTime; 14 | private Map headers; 15 | 16 | @Override 17 | public Instant getWatermark() { 18 | return this.watermark; 19 | } 20 | 21 | @Override 22 | public byte[] getValue() { 23 | return this.value; 24 | } 25 | 26 | @Override 27 | public Instant getEventTime() { 28 | return this.eventTime; 29 | } 30 | 31 | @Override 32 | public Map getHeaders() { 33 | return this.headers; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/MergeAccumulatorRequest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | 6 | /** 7 | * MergeAccumulatorRequest is sent from the supervisor actor to the merged session reducer actor to 8 | * ask the actor to merge an accumulator. 9 | *

10 | * "Hey, I received an accumulator from one of the sessions that are merging to you, please merge it with yourself." 11 | */ 12 | @AllArgsConstructor 13 | @Getter 14 | class MergeAccumulatorRequest { 15 | private final byte[] accumulator; 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/MergeDoneResponse.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | /** 4 | * MergeDoneResponse indicates the current MERGE request is done. 5 | */ 6 | public class MergeDoneResponse { 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/OutputActor.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import akka.actor.AbstractActor; 4 | import akka.actor.Props; 5 | import io.grpc.stub.StreamObserver; 6 | import io.numaproj.numaflow.sessionreduce.v1.Sessionreduce; 7 | import lombok.AllArgsConstructor; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | /** 11 | * Output actor is a wrapper around the gRPC output stream. 12 | * It ensures synchronized calls to the responseObserver onNext() and invokes onComplete at the end of the stream. 13 | * ALL session reduce responses are sent to the output actor before getting forwarded to the output gRPC stream. 14 | *

15 | * More details about gRPC StreamObserver concurrency: https://grpc.github.io/grpc-java/javadoc/io/grpc/stub/StreamObserver.html 16 | */ 17 | @Slf4j 18 | @AllArgsConstructor 19 | class OutputActor extends AbstractActor { 20 | StreamObserver responseObserver; 21 | 22 | public static Props props( 23 | StreamObserver responseObserver) { 24 | return Props.create(OutputActor.class, responseObserver); 25 | } 26 | 27 | @Override 28 | public Receive createReceive() { 29 | return receiveBuilder() 30 | .match(ActorResponse.class, this::handleResponse) 31 | .match(String.class, this::handleEOF) 32 | .build(); 33 | } 34 | 35 | private void handleResponse(ActorResponse actorResponse) { 36 | responseObserver.onNext(actorResponse.getResponse()); 37 | if (actorResponse.isLast()) { 38 | this.closeSystem(); 39 | } 40 | } 41 | 42 | private void handleEOF(String eof) { 43 | this.closeSystem(); 44 | } 45 | 46 | private void closeSystem() { 47 | // close the output stream. 48 | responseObserver.onCompleted(); 49 | // stop the AKKA system right after we close the output stream. 50 | // note: could make more sense if the supervisor actor stops the system, 51 | // but it requires an extra tell. 52 | getContext().getSystem().stop(getSender()); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/OutputStreamObserverImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import akka.actor.ActorRef; 4 | import com.google.protobuf.ByteString; 5 | import io.numaproj.numaflow.sessionreduce.v1.Sessionreduce; 6 | import io.numaproj.numaflow.sessionreducer.model.Message; 7 | import io.numaproj.numaflow.sessionreducer.model.OutputStreamObserver; 8 | import lombok.AllArgsConstructor; 9 | import lombok.Setter; 10 | 11 | import java.util.ArrayList; 12 | import java.util.Arrays; 13 | 14 | /** 15 | * OutputStreamObserverImpl transforms a message to an ActorResponse. 16 | * The send method sends the ActorResponse to the output actor to forward to output gRPC stream. 17 | */ 18 | @AllArgsConstructor 19 | class OutputStreamObserverImpl implements OutputStreamObserver { 20 | private final ActorRef outputActor; 21 | @Setter 22 | private Sessionreduce.KeyedWindow keyedWindow; 23 | 24 | @Override 25 | public void send(Message message) { 26 | this.outputActor.tell( 27 | buildResponse(message, this.keyedWindow), 28 | ActorRef.noSender()); 29 | } 30 | 31 | private ActorResponse buildResponse(Message message, Sessionreduce.KeyedWindow keyedWindow) { 32 | Sessionreduce.SessionReduceResponse.Builder responseBuilder = Sessionreduce.SessionReduceResponse.newBuilder(); 33 | // set the window 34 | responseBuilder.setKeyedWindow(keyedWindow); 35 | // set EOF to false 36 | responseBuilder.setEOF(false); 37 | // set the result. 38 | responseBuilder.setResult(Sessionreduce.SessionReduceResponse.Result 39 | .newBuilder() 40 | .setValue(ByteString.copyFrom(message.getValue())) 41 | .addAllKeys(message.getKeys() 42 | == null ? new ArrayList<>() : Arrays.asList(message.getKeys())) 43 | .addAllTags( 44 | message.getTags() == null ? new ArrayList<>() : Arrays.asList(message.getTags())) 45 | .build()); 46 | return ActorResponse.builder() 47 | .response(responseBuilder.build()) 48 | .build(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/UniqueIdGenerator.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import com.google.protobuf.Timestamp; 4 | import io.numaproj.numaflow.sessionreduce.v1.Sessionreduce; 5 | 6 | import java.time.Instant; 7 | 8 | /** UniqueIdGenerator is a utility class to generate a unique id for a keyed session window. */ 9 | public class UniqueIdGenerator { 10 | // private constructor to prevent instantiation 11 | private UniqueIdGenerator() { 12 | throw new AssertionError("utility class cannot be instantiated"); 13 | } 14 | 15 | public static String getUniqueIdentifier(Sessionreduce.KeyedWindow keyedWindow) { 16 | long startMillis = convertToEpochMilli(keyedWindow.getStart()); 17 | long endMillis = convertToEpochMilli(keyedWindow.getEnd()); 18 | return String.format( 19 | "%d:%d:%s", 20 | startMillis, endMillis, String.join(Constants.DELIMITER, keyedWindow.getKeysList())); 21 | } 22 | 23 | private static long convertToEpochMilli(Timestamp timestamp) { 24 | return Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()).toEpochMilli(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/model/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer.model; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | 6 | /** 7 | * Datum contains methods to get the payload information. 8 | */ 9 | public interface Datum { 10 | /** 11 | * method to get the payload value 12 | * 13 | * @return returns the payload value in byte array 14 | */ 15 | byte[] getValue(); 16 | 17 | /** 18 | * method to get the event time of the payload 19 | * 20 | * @return returns the event time of the payload 21 | */ 22 | Instant getEventTime(); 23 | 24 | /** 25 | * method to get the watermark information 26 | * 27 | * @return returns the watermark 28 | */ 29 | Instant getWatermark(); 30 | 31 | /** 32 | * method to get the headers information of the payload 33 | * 34 | * @return returns the headers in the form of key value pair 35 | */ 36 | Map getHeaders(); 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/model/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer.model; 2 | 3 | import lombok.Getter; 4 | 5 | /** Message is used to wrap the data returned by Session Reducer functions. */ 6 | @Getter 7 | public class Message { 8 | private static final String[] DROP_TAGS = {"U+005C__DROP__"}; 9 | private final String[] keys; 10 | private final byte[] value; 11 | private final String[] tags; 12 | 13 | /** 14 | * used to create Message with value, keys and tags(used for conditional forwarding) 15 | * 16 | * @param value message value 17 | * @param keys message keys 18 | * @param tags message tags which will be used for conditional forwarding 19 | */ 20 | public Message(byte[] value, String[] keys, String[] tags) { 21 | // defensive copy - once the Message is created, the caller should not be able to modify it. 22 | this.keys = keys == null ? null : keys.clone(); 23 | this.value = value == null ? null : value.clone(); 24 | this.tags = tags == null ? null : tags.clone(); 25 | } 26 | 27 | /** 28 | * used to create Message with value. 29 | * 30 | * @param value message value 31 | */ 32 | public Message(byte[] value) { 33 | this(value, null, null); 34 | } 35 | 36 | /** 37 | * used to create Message with value and keys. 38 | * 39 | * @param value message value 40 | * @param keys message keys 41 | */ 42 | public Message(byte[] value, String[] keys) { 43 | this(value, keys, null); 44 | } 45 | 46 | /** 47 | * creates a Message which will be dropped 48 | * 49 | * @return returns the Message which will be dropped 50 | */ 51 | public static Message toDrop() { 52 | return new Message(new byte[0], null, DROP_TAGS); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/model/OutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer.model; 2 | 3 | /** 4 | * OutputStreamObserver sends to the output stream, the messages generate by the session reducer. 5 | */ 6 | public interface OutputStreamObserver { 7 | /** 8 | * method will be used for sending messages to the output stream. 9 | * 10 | * @param message the message to be sent 11 | */ 12 | void send(Message message); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sessionreducer/model/SessionReducerFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer.model; 2 | 3 | /** 4 | * ReduceStreamerFactory is the factory for SessionReducer. 5 | */ 6 | public abstract class SessionReducerFactory { 7 | /** 8 | * Helper function to create a session reducer. 9 | * 10 | * @return a concrete session reducer instance 11 | */ 12 | public abstract SessionReducerT createSessionReducer(); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/shared/GrpcConfigRetriever.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.shared; 2 | 3 | // Currently each of the UDFs (Mapper, BatchMapper, Sourcer) has its own GrpcConfig class. 4 | // To start a gRPC server, we need to pass gRPC configurations to the GrpcServerWrapper. 5 | // In order to make the GrpcServerWrapper more generic, we create this GrpcConfigRetriever interface, 6 | // which is implemented by the UDFs' GrpcConfig classes. 7 | public interface GrpcConfigRetriever { 8 | String getSocketPath(); 9 | int getMaxMessageSize(); 10 | String getInfoFilePath(); 11 | int getPort(); 12 | boolean isLocal(); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/shared/ThreadUtils.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.shared; 2 | 3 | import lombok.AccessLevel; 4 | import lombok.NoArgsConstructor; 5 | 6 | import java.util.concurrent.ThreadFactory; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | 9 | /** 10 | * ThreadUtils is a utility class for thread related operations. 11 | */ 12 | @NoArgsConstructor(access = AccessLevel.PACKAGE) 13 | public class ThreadUtils { 14 | public static final ThreadUtils INSTANCE = new ThreadUtils(); 15 | 16 | 17 | /** 18 | * availableProcessors returns the number available processors. 19 | * 20 | * @return the number of available processors 21 | */ 22 | public int availableProcessors() { 23 | return Runtime.getRuntime().availableProcessors(); 24 | } 25 | 26 | /** 27 | * Creates a new thread factory with the given name. 28 | * 29 | * @param name the name of the thread 30 | * 31 | * @return a new thread factory 32 | */ 33 | public ThreadFactory newThreadFactory(String name) { 34 | return new ThreadFactory() { 35 | private final AtomicInteger num = new AtomicInteger(); 36 | 37 | @Override 38 | public Thread newThread(Runnable r) { 39 | Thread t = new Thread(r); 40 | int s = num.getAndIncrement(); 41 | t.setName(s > 0 ? name + "-" + s : name); 42 | return t; 43 | } 44 | }; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sideinput/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sideinput; 2 | 3 | public class Constants { 4 | public static final String SIDE_INPUT_DIR = "/var/numaflow/side-inputs"; 5 | static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/sideinput.sock"; 6 | static final String DEFAULT_SERVER_INFO_FILE_PATH = "/var/run/numaflow/sideinput-server-info"; 7 | static final String DEFAULT_HOST = "localhost"; 8 | static int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 9 | static int DEFAULT_PORT = 50051; 10 | 11 | // Private constructor to prevent instantiation 12 | private Constants() { 13 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sideinput/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sideinput; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sideinput/Message.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sideinput; 2 | 3 | import lombok.Getter; 4 | 5 | /** Message is used to wrap the data returned by Side Input Retriever. */ 6 | @Getter 7 | public class Message { 8 | private final byte[] value; 9 | private final boolean noBroadcast; 10 | 11 | /** used to create Message with value and noBroadcast flag. */ 12 | private Message(byte[] value, boolean noBroadcast) { 13 | // defensive copy - once the Message is created, the caller should not be able to modify it. 14 | this.value = value == null ? null : value.clone(); 15 | this.noBroadcast = noBroadcast; 16 | } 17 | 18 | /** 19 | * createBroadcastMessage creates a new Message with the given value This is used to broadcast the 20 | * message to other side input vertices. 21 | * 22 | * @param value message value 23 | * @return returns the Message with noBroadcast flag set to false 24 | */ 25 | public static Message createBroadcastMessage(byte[] value) { 26 | return new Message(value, false); 27 | } 28 | 29 | /** 30 | * createNoBroadcastMessage creates a new Message with noBroadcast flag set to true This is used 31 | * to drop the message and not to broadcast it to other side input vertices. 32 | * 33 | * @return returns the Message with noBroadcast flag set to true 34 | */ 35 | public static Message createNoBroadcastMessage() { 36 | return new Message(new byte[0], true); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sideinput/Service.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sideinput; 2 | 3 | import com.google.protobuf.Empty; 4 | import io.grpc.stub.StreamObserver; 5 | import com.google.protobuf.ByteString; 6 | import io.numaproj.numaflow.sideinput.v1.SideInputGrpc; 7 | import io.numaproj.numaflow.sideinput.v1.Sideinput; 8 | import lombok.AllArgsConstructor; 9 | import lombok.extern.slf4j.Slf4j; 10 | 11 | @Slf4j 12 | @AllArgsConstructor 13 | class Service extends SideInputGrpc.SideInputImplBase { 14 | 15 | private final SideInputRetriever sideInputRetriever; 16 | 17 | /** 18 | * Invokes the side input retriever to retrieve side input. 19 | */ 20 | @Override 21 | public void retrieveSideInput( 22 | Empty request, 23 | StreamObserver responseObserver) { 24 | 25 | if (this.sideInputRetriever == null) { 26 | io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall( 27 | SideInputGrpc.getRetrieveSideInputMethod(), 28 | responseObserver); 29 | return; 30 | } 31 | // process request 32 | Message message = sideInputRetriever.retrieveSideInput(); 33 | // set response 34 | responseObserver.onNext(buildResponse(message)); 35 | responseObserver.onCompleted(); 36 | } 37 | 38 | /** 39 | * IsReady is the heartbeat endpoint for gRPC. 40 | */ 41 | @Override 42 | public void isReady(Empty request, StreamObserver responseObserver) { 43 | responseObserver.onNext(Sideinput.ReadyResponse.newBuilder().setReady(true).build()); 44 | responseObserver.onCompleted(); 45 | } 46 | 47 | private Sideinput.SideInputResponse buildResponse(Message message) { 48 | return Sideinput.SideInputResponse.newBuilder() 49 | .setValue(message.getValue() == null ? ByteString.EMPTY 50 | : ByteString.copyFrom( 51 | message.getValue())) 52 | .setNoBroadcast(message.isNoBroadcast()) 53 | .build(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sideinput/SideInputRetriever.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sideinput; 2 | 3 | 4 | /** 5 | * SideInputRetriever exposes method for retrieving side input. 6 | * Implementations should override the retrieveSideInput method 7 | * which will be used for updating the side input. 8 | */ 9 | public abstract class SideInputRetriever { 10 | /** 11 | * method which will be used for retrieving side input. 12 | * In case of failure, user can return a Message with noBroadcast flag set to true. 13 | * This will drop the message and will not broadcast it. 14 | * 15 | * @return Message which contains side input 16 | */ 17 | public abstract Message retrieveSideInput(); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | class Constants { 4 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/sink.sock"; 5 | public static final String DEFAULT_FB_SINK_SOCKET_PATH = "/var/run/numaflow/fb-sink.sock"; 6 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = "/var/run/numaflow/sinker-server-info"; 7 | public static final String DEFAULT_FB_SERVER_INFO_FILE_PATH = 8 | "/var/run/numaflow/fb-sinker-server-info"; 9 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 10 | public static final int DEFAULT_PORT = 50051; 11 | public static final String DEFAULT_HOST = "localhost"; 12 | public static final String ENV_UD_CONTAINER_TYPE = "NUMAFLOW_UD_CONTAINER_TYPE"; 13 | public static final String UD_CONTAINER_FALLBACK_SINK = "fb-udsink"; 14 | 15 | // Private constructor to prevent instantiation 16 | private Constants() { 17 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import java.time.Instant; 4 | import java.util.Map; 5 | 6 | /** 7 | * Datum contains methods to get the payload information. 8 | */ 9 | public interface Datum { 10 | /** 11 | * method to get the payload keys 12 | * 13 | * @return returns the datum keys. 14 | */ 15 | String[] getKeys(); 16 | 17 | /** 18 | * method to get the payload value 19 | * 20 | * @return returns the payload value in byte array 21 | */ 22 | byte[] getValue(); 23 | 24 | /** 25 | * method to get the event time of the payload 26 | * 27 | * @return returns the event time of the payload 28 | */ 29 | Instant getEventTime(); 30 | 31 | /** 32 | * method to get the watermark information 33 | * 34 | * @return returns the watermark 35 | */ 36 | Instant getWatermark(); 37 | 38 | /** 39 | * method to get the ID for the Payload 40 | * 41 | * @return returns the ID 42 | */ 43 | String getId(); 44 | 45 | /** 46 | * method to get the headers information of the payload 47 | * 48 | * @return returns the headers in the form of key value pair 49 | */ 50 | Map getHeaders(); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/DatumIterator.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | /** 4 | * An iterator over a collection of {@link Datum} elements. 5 | * Passed to {@link Sinker#processMessages(DatumIterator)} method. 6 | */ 7 | public interface DatumIterator { 8 | 9 | /** 10 | * Returns the next element in the iterator 11 | * This method blocks until an element becomes available in the queue. 12 | * When EOF_DATUM is received, this method will return null and the iterator will be closed. 13 | * 14 | * @return the next element in the iterator, null if EOF_DATUM is received or the iterator is already closed 15 | * 16 | * @throws InterruptedException if the thread is interrupted while waiting for the next element 17 | */ 18 | Datum next() throws InterruptedException; 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/DatumIteratorImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | 5 | import java.util.concurrent.BlockingQueue; 6 | import java.util.concurrent.LinkedBlockingDeque; 7 | import java.util.concurrent.atomic.AtomicBoolean; 8 | 9 | /** 10 | * A thread-safe implementation of {@link DatumIterator}, backed by a blocking queue. 11 | */ 12 | @Slf4j 13 | class DatumIteratorImpl implements DatumIterator { 14 | private final BlockingQueue blockingQueue = new LinkedBlockingDeque<>(); 15 | private final AtomicBoolean closed = new AtomicBoolean(false); 16 | 17 | @Override 18 | public Datum next() throws InterruptedException { 19 | // if the iterator is closed, return null 20 | if (closed.get()) { 21 | return null; 22 | } 23 | Datum datum = blockingQueue.take(); 24 | // if EOF is received, close the iterator and return null 25 | if (datum == HandlerDatum.EOF_DATUM) { 26 | closed.set(true); 27 | return null; 28 | } 29 | return datum; 30 | } 31 | 32 | // blocking call, waits until the write operation is successful 33 | public void writeMessage(Datum datum) throws InterruptedException { 34 | blockingQueue.put(datum); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | String containerType = System.getenv(Constants.ENV_UD_CONTAINER_TYPE); 32 | String socketPath = Constants.DEFAULT_SOCKET_PATH; 33 | String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 34 | 35 | // if containerType is fb-udsink then we need to use fb sink socket path and info file path 36 | if (Constants.UD_CONTAINER_FALLBACK_SINK.equals(containerType)) { 37 | socketPath = Constants.DEFAULT_FB_SINK_SOCKET_PATH; 38 | infoFilePath = Constants.DEFAULT_FB_SERVER_INFO_FILE_PATH; 39 | } 40 | return GRPCConfig.newBuilder() 41 | .infoFilePath(infoFilePath) 42 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 43 | .isLocal(containerType 44 | == null) // if ENV_UD_CONTAINER_TYPE is not set, then we are not running using numaflow 45 | .socketPath(socketPath) 46 | .build(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | import java.time.Instant; 6 | import java.util.Map; 7 | 8 | @AllArgsConstructor 9 | class HandlerDatum implements Datum { 10 | 11 | // EOF_DATUM is used to indicate the end of the stream. 12 | static final HandlerDatum EOF_DATUM = new HandlerDatum(null, null, null, null, null, null); 13 | private String[] keys; 14 | private byte[] value; 15 | private Instant watermark; 16 | private Instant eventTime; 17 | private String id; 18 | private Map headers; 19 | 20 | @Override 21 | public String[] getKeys() { 22 | return keys; 23 | } 24 | 25 | @Override 26 | public Instant getWatermark() { 27 | return this.watermark; 28 | } 29 | 30 | @Override 31 | public byte[] getValue() { 32 | return this.value; 33 | } 34 | 35 | @Override 36 | public Instant getEventTime() { 37 | return this.eventTime; 38 | } 39 | 40 | @Override 41 | public String getId() { 42 | return id; 43 | } 44 | 45 | @Override 46 | public Map getHeaders() { 47 | return this.headers; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/Response.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import lombok.AccessLevel; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | 7 | /** 8 | * Response is used to send response from the user defined sinker. It contains the id of the 9 | * message, success status, an optional error message and a fallback status. Various static factory 10 | * methods are available to create a Response instance based on the processing outcome. 11 | */ 12 | @Getter 13 | @AllArgsConstructor(access = AccessLevel.PRIVATE) 14 | public class Response { 15 | private final String id; 16 | private final Boolean success; 17 | private final String err; 18 | private final Boolean fallback; 19 | 20 | /** 21 | * Static method to create response for successful message processing. 22 | * 23 | * @param id id of the message 24 | * @return Response object with success status 25 | */ 26 | public static Response responseOK(String id) { 27 | return new Response(id, true, null, false); 28 | } 29 | 30 | /** 31 | * Static method to create response for failed message processing. 32 | * 33 | * @param id id of the message 34 | * @param errMsg error message 35 | * @return Response object with failure status and error message 36 | */ 37 | public static Response responseFailure(String id, String errMsg) { 38 | return new Response(id, false, errMsg, false); 39 | } 40 | 41 | /** 42 | * Static method to create response for fallback message. This indicates that the message should 43 | * be sent to the fallback sink. 44 | * 45 | * @param id id of the message 46 | * @return Response object with fallback status 47 | */ 48 | public static Response responseFallback(String id) { 49 | return new Response(id, false, null, true); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/ResponseList.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import lombok.Builder; 4 | import lombok.Getter; 5 | import lombok.Singular; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | /** 11 | * ResponseList is used to return the list of responses from user defined sinker. 12 | */ 13 | 14 | @Getter 15 | @Builder(builderMethodName = "newBuilder") 16 | public class ResponseList { 17 | 18 | @Singular("addResponse") 19 | private List responses; 20 | 21 | /** 22 | * Builder to build ResponseList 23 | */ 24 | public static class ResponseListBuilder { 25 | /** 26 | * @param responses to append all the responses to ResponseList 27 | * 28 | * @return returns the builder 29 | */ 30 | public ResponseListBuilder addResponses(Iterable responses) { 31 | if (this.responses == null) { 32 | this.responses = new ArrayList<>(); 33 | } 34 | 35 | for (Response response : responses) { 36 | this.responses.add(response); 37 | } 38 | return this; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sinker/Sinker.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | /** 4 | * Sinker exposes method for publishing messages to sink. 5 | * Implementations should override the processMessage method 6 | * which will be used for processing the input messages 7 | */ 8 | 9 | public abstract class Sinker { 10 | /** 11 | * method will be used for processing messages. 12 | * response for the message should be added to the 13 | * response list using ResponseListBuilder and the 14 | * response list should be returned. 15 | * 16 | * @param datumStream stream of messages to be processed 17 | * 18 | * @return response list 19 | */ 20 | public abstract ResponseList processMessages(DatumIterator datumStream); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/AckRequest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | 4 | import java.util.List; 5 | 6 | /** 7 | * AckRequest request for acknowledging messages. 8 | */ 9 | public interface AckRequest { 10 | /** 11 | * @return the list of offsets to be acknowledged 12 | */ 13 | List getOffsets(); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/AckRequestImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * AckRequestImpl is the implementation of AckRequest. 9 | */ 10 | @AllArgsConstructor 11 | class AckRequestImpl implements AckRequest { 12 | private final List offsets; 13 | 14 | @Override 15 | public List getOffsets() { 16 | return this.offsets; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | class Constants { 4 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/source.sock"; 5 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = 6 | "/var/run/numaflow/sourcer-server-info"; 7 | public static final int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 8 | public static final int DEFAULT_PORT = 50051; 9 | public static final String DEFAULT_HOST = "localhost"; 10 | 11 | // Private constructor to prevent instantiation 12 | private Constants() { 13 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/NackRequest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | 4 | import java.util.List; 5 | 6 | /** 7 | * NackRequest request for negatively acknowledging messages. 8 | */ 9 | public interface NackRequest { 10 | /** 11 | * @return the list of offsets to be negatively acknowledged. 12 | */ 13 | List getOffsets(); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/NackRequestImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * NackRequestImpl is the implementation of NackRequest. 9 | */ 10 | @AllArgsConstructor 11 | class NackRequestImpl implements NackRequest { 12 | private final List offsets; 13 | 14 | @Override 15 | public List getOffsets() { 16 | return this.offsets; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/Offset.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | /** 7 | * Offset is the message offset. 8 | */ 9 | @Getter 10 | @Setter 11 | public class Offset { 12 | private final byte[] value; 13 | private final Integer partitionId; 14 | 15 | /** 16 | * used to create Offset with value and partitionId. 17 | * 18 | * @param value offset value 19 | * @param partitionId offset partitionId 20 | */ 21 | public Offset(byte[] value, Integer partitionId) { 22 | this.value = value; 23 | this.partitionId = partitionId; 24 | } 25 | 26 | /** 27 | * used to create Offset with value and default partitionId. 28 | * 29 | * @param value offset value 30 | */ 31 | public Offset(byte[] value) { 32 | this.value = value; 33 | this.partitionId = Sourcer.defaultPartitions().get(0); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/OutputObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | /** 4 | * OutputObserver receives messages from the sourcer. 5 | */ 6 | public interface OutputObserver { 7 | /** 8 | * method will be used for sending messages to the output. 9 | * 10 | * @param message the message to be sent 11 | */ 12 | void send(Message message); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/ReadRequest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | import java.time.Duration; 4 | 5 | /** 6 | * ReadRequest request for reading messages from source. 7 | */ 8 | public interface ReadRequest { 9 | /** 10 | * @return the number of messages to be read 11 | */ 12 | long getCount(); 13 | 14 | /** 15 | * @return the timeout for reading messages 16 | */ 17 | Duration getTimeout(); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcer/ReadRequestImpl.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | import java.time.Duration; 6 | 7 | /** 8 | * ReadRequest is used to wrap the request for reading messages from source. 9 | */ 10 | @AllArgsConstructor 11 | class ReadRequestImpl implements ReadRequest { 12 | long count; 13 | Duration timeout; 14 | 15 | @Override 16 | public long getCount() { 17 | return this.count; 18 | } 19 | 20 | @Override 21 | public Duration getTimeout() { 22 | return this.timeout; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcetransformer/Constants.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | class Constants { 4 | public static final String DEFAULT_SOCKET_PATH = "/var/run/numaflow/sourcetransform.sock"; 5 | public static final String DEFAULT_SERVER_INFO_FILE_PATH = 6 | "/var/run/numaflow/sourcetransformer-server-info"; 7 | public static final String DEFAULT_HOST = "localhost"; 8 | public static final String EOF = "EOF"; 9 | public static int DEFAULT_MESSAGE_SIZE = 1024 * 1024 * 64; 10 | public static int DEFAULT_PORT = 50051; 11 | 12 | // Private constructor to prevent instantiation 13 | private Constants() { 14 | throw new IllegalStateException("Utility class 'Constants' should not be instantiated"); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcetransformer/Datum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | 4 | import java.time.Instant; 5 | import java.util.Map; 6 | 7 | /** 8 | * Datum contains methods to get the payload information. 9 | */ 10 | 11 | public interface Datum { 12 | /** 13 | * method to get the payload value 14 | * 15 | * @return returns the payload value in byte array 16 | */ 17 | byte[] getValue(); 18 | 19 | /** 20 | * method to get the event time of the payload 21 | * 22 | * @return returns the event time of the payload 23 | */ 24 | Instant getEventTime(); 25 | 26 | /** 27 | * method to get the watermark information 28 | * 29 | * @return returns the watermark 30 | */ 31 | Instant getWatermark(); 32 | 33 | /** 34 | * method to get the headers information of the payload 35 | * 36 | * @return returns the headers in the form of key value pair 37 | */ 38 | Map getHeaders(); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcetransformer/GRPCConfig.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | import io.numaproj.numaflow.shared.GrpcConfigRetriever; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | /** 8 | * GRPCConfig is used to provide configurations for gRPC server. 9 | */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class GRPCConfig implements GrpcConfigRetriever { 13 | @Builder.Default 14 | private String socketPath = Constants.DEFAULT_SOCKET_PATH; 15 | 16 | @Builder.Default 17 | private int maxMessageSize = Constants.DEFAULT_MESSAGE_SIZE; 18 | 19 | @Builder.Default 20 | private String infoFilePath = Constants.DEFAULT_SERVER_INFO_FILE_PATH; 21 | 22 | @Builder.Default 23 | private int port = Constants.DEFAULT_PORT; 24 | 25 | private boolean isLocal; 26 | 27 | /** 28 | * Static method to create default GRPCConfig. 29 | */ 30 | static GRPCConfig defaultGrpcConfig() { 31 | return GRPCConfig.newBuilder() 32 | .infoFilePath(Constants.DEFAULT_SERVER_INFO_FILE_PATH) 33 | .maxMessageSize(Constants.DEFAULT_MESSAGE_SIZE) 34 | .isLocal(System.getenv("NUMAFLOW_POD") 35 | == null) // if NUMAFLOW_POD is not set, then we are not running using numaflow 36 | .socketPath(Constants.DEFAULT_SOCKET_PATH).build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcetransformer/HandlerDatum.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | @AllArgsConstructor 10 | class HandlerDatum implements Datum { 11 | 12 | private byte[] value; 13 | private Instant watermark; 14 | private Instant eventTime; 15 | private Map headers; 16 | 17 | 18 | @Override 19 | public Instant getWatermark() { 20 | return this.watermark; 21 | } 22 | 23 | @Override 24 | public byte[] getValue() { 25 | return this.value; 26 | } 27 | 28 | @Override 29 | public Instant getEventTime() { 30 | return this.eventTime; 31 | } 32 | 33 | @Override 34 | public Map getHeaders() { 35 | return this.headers; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcetransformer/MessageList.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import lombok.Builder; 6 | import lombok.Getter; 7 | import lombok.Singular; 8 | 9 | /** MessageList is used to return the list of Messages from SourceTransformer functions. */ 10 | @Getter 11 | @Builder(builderMethodName = "newBuilder") 12 | public class MessageList { 13 | 14 | @Singular("addMessage") 15 | private List messages; 16 | 17 | /** Builder to build MessageList */ 18 | public static class MessageListBuilder { 19 | /** 20 | * @param messages to append all the messages to MessageList 21 | * @return returns the builder 22 | */ 23 | public MessageListBuilder addAllMessages(Iterable messages) { 24 | if (this.messages == null) { 25 | this.messages = new ArrayList<>(); 26 | } 27 | 28 | for (Message message : messages) { 29 | this.messages.add(message); 30 | } 31 | return this; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/numaproj/numaflow/sourcetransformer/SourceTransformer.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | 4 | /** 5 | * SourceTransformer exposes method for performing transform operation 6 | * in the source. Implementations should override the processMessage 7 | * method which will be used for transforming and assigning event time 8 | * to input messages 9 | */ 10 | 11 | public abstract class SourceTransformer { 12 | /** 13 | * method which will be used for processing messages. 14 | * 15 | * @param keys message keys 16 | * @param datum current message to be processed 17 | * 18 | * @return MessageList which contains output from the transform function 19 | */ 20 | public abstract MessageList processMessage(String[] keys, Datum datum); 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/proto/map/v1/map.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option java_package = "io.numaproj.numaflow.map.v1"; 4 | 5 | import "google/protobuf/timestamp.proto"; 6 | import "google/protobuf/empty.proto"; 7 | 8 | package map.v1; 9 | 10 | service Map { 11 | // MapFn applies a function to each map request element. 12 | rpc MapFn(stream MapRequest) returns (stream MapResponse); 13 | 14 | // IsReady is the heartbeat endpoint for gRPC. 15 | rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); 16 | } 17 | 18 | /** 19 | * MapRequest represents a request element. 20 | */ 21 | message MapRequest { 22 | message Request { 23 | repeated string keys = 1; 24 | bytes value = 2; 25 | google.protobuf.Timestamp event_time = 3; 26 | google.protobuf.Timestamp watermark = 4; 27 | map headers = 5; 28 | } 29 | Request request = 1; 30 | // This ID is used to uniquely identify a map request 31 | string id = 2; 32 | optional Handshake handshake = 3; 33 | optional TransmissionStatus status = 4; 34 | } 35 | 36 | /* 37 | * Handshake message between client and server to indicate the start of transmission. 38 | */ 39 | message Handshake { 40 | // Required field indicating the start of transmission. 41 | bool sot = 1; 42 | } 43 | 44 | /* 45 | * Status message to indicate the status of the message. 46 | */ 47 | message TransmissionStatus { 48 | bool eot = 1; 49 | } 50 | 51 | /** 52 | * MapResponse represents a response element. 53 | */ 54 | message MapResponse { 55 | message Result { 56 | repeated string keys = 1; 57 | bytes value = 2; 58 | repeated string tags = 3; 59 | } 60 | repeated Result results = 1; 61 | // This ID is used to refer the responses to the request it corresponds to. 62 | string id = 2; 63 | optional Handshake handshake = 3; 64 | optional TransmissionStatus status = 4; 65 | } 66 | 67 | /** 68 | * ReadyResponse is the health check result. 69 | */ 70 | message ReadyResponse { 71 | bool ready = 1; 72 | } 73 | -------------------------------------------------------------------------------- /src/main/proto/sideinput/v1/sideinput.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option java_package = "io.numaproj.numaflow.sideinput.v1"; 4 | 5 | import "google/protobuf/empty.proto"; 6 | 7 | package sideinput.v1; 8 | 9 | // SideInput is the gRPC service for Side Inputs. 10 | // It is used to propagate changes in the values of the provided Side Inputs 11 | // which allows access to slow updated data or configuration without needing to retrieve 12 | // it during each message processing. 13 | // Through this service we should should be able to:- 14 | // 1) Invoke retrieval request for a single Side Input parameter, which in turn should 15 | // check for updates and return its latest value. 16 | // 2) Provide a health check endpoint to indicate whether the service is ready to be used. 17 | service SideInput { 18 | // RetrieveSideInput is the endpoint to retrieve the latest value of a given Side Input. 19 | rpc RetrieveSideInput(google.protobuf.Empty) returns (SideInputResponse); 20 | 21 | // IsReady is the health check endpoint to indicate whether the service is ready to be used. 22 | rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); 23 | } 24 | 25 | /** 26 | * SideInputResponse represents a response to a given side input retrieval request. 27 | */ 28 | message SideInputResponse { 29 | // value represents the latest value of the side input payload 30 | bytes value = 1; 31 | // noBroadcast indicates whether the side input value should be broadcasted to all 32 | // True if value should not be broadcasted 33 | // False if value should be broadcasted 34 | bool no_broadcast = 2; 35 | } 36 | 37 | /** 38 | * ReadyResponse is the health check result. 39 | */ 40 | message ReadyResponse { 41 | bool ready = 1; 42 | } 43 | -------------------------------------------------------------------------------- /src/main/resources/numaflow-java-sdk-version.properties: -------------------------------------------------------------------------------- 1 | sdk.version=${project.version} 2 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/accumulator/AccumulatorStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator; 2 | 3 | import io.grpc.stub.StreamObserver; 4 | import io.numaproj.numaflow.accumulator.v1.AccumulatorOuterClass; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.concurrent.CompletableFuture; 9 | 10 | public class AccumulatorStreamObserver implements StreamObserver { 11 | List responses = new ArrayList<>(); 12 | CompletableFuture done = new CompletableFuture<>(); 13 | Integer responseCount; 14 | 15 | public AccumulatorStreamObserver(Integer responseCount) { 16 | this.responseCount = responseCount; 17 | } 18 | 19 | @Override 20 | public void onNext(AccumulatorOuterClass.AccumulatorResponse response) { 21 | responses.add(response); 22 | if (responses.size() == responseCount) { 23 | done.complete(null); 24 | } 25 | } 26 | 27 | @Override 28 | public void onError(Throwable throwable) { 29 | done.completeExceptionally(throwable); 30 | } 31 | 32 | @Override 33 | public void onCompleted() { 34 | done.complete(null); 35 | } 36 | 37 | public List getResponses() { 38 | return responses; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/accumulator/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.accumulator; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | int port = 8001; 24 | String socketPath = "test-socket-path"; 25 | int maxMessageSize = 2000; 26 | String infoFilePath = "test-info-file-path"; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/batchmapper/BatchMapOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import io.grpc.stub.StreamObserver; 4 | import io.numaproj.numaflow.map.v1.MapOuterClass; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.concurrent.CompletableFuture; 9 | 10 | public class BatchMapOutputStreamObserver implements StreamObserver { 11 | List mapResponses = new ArrayList<>(); 12 | CompletableFuture done = new CompletableFuture<>(); 13 | Integer responseCount; 14 | 15 | public BatchMapOutputStreamObserver(Integer responseCount) { 16 | this.responseCount = responseCount; 17 | } 18 | 19 | @Override 20 | public void onNext(MapOuterClass.MapResponse mapResponse) { 21 | mapResponses.add(mapResponse); 22 | if (mapResponses.size() == responseCount) { 23 | done.complete(null); 24 | } 25 | } 26 | 27 | @Override 28 | public void onError(Throwable throwable) { 29 | done.completeExceptionally(throwable); 30 | } 31 | 32 | @Override 33 | public void onCompleted() { 34 | done.complete(null); 35 | } 36 | 37 | public List getMapResponses() { 38 | return mapResponses; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/batchmapper/DatumStreamImplTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import lombok.AllArgsConstructor; 4 | import org.junit.Test; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | import static org.junit.Assert.assertNull; 11 | 12 | public class DatumStreamImplTest { 13 | @Test 14 | public void datumStreamNextTest() throws InterruptedException { 15 | Datum datum1 = new TestDatum("test1"); 16 | Datum datum2 = new TestDatum("test2"); 17 | Datum eofDatum = HandlerDatum.EOF_DATUM; 18 | 19 | DatumIteratorImpl datumIterator = new DatumIteratorImpl(); 20 | 21 | datumIterator.writeMessage(datum1); 22 | datumIterator.writeMessage(datum2); 23 | datumIterator.writeMessage(eofDatum); 24 | 25 | assertEquals(datum1, datumIterator.next()); 26 | assertEquals(datum2, datumIterator.next()); 27 | assertEquals(3, datumIterator.getCount()); 28 | assertNull(datumIterator.next()); 29 | } 30 | 31 | @AllArgsConstructor 32 | public static class TestDatum implements Datum { 33 | private String id; 34 | 35 | @Override 36 | public String[] getKeys() { 37 | return new String[0]; 38 | } 39 | 40 | @Override 41 | public byte[] getValue() { 42 | return new byte[0]; 43 | } 44 | 45 | @Override 46 | public Instant getEventTime() { 47 | return null; 48 | } 49 | 50 | @Override 51 | public Instant getWatermark() { 52 | return null; 53 | } 54 | 55 | @Override 56 | public String getId() { 57 | return null; 58 | } 59 | 60 | @Override 61 | public Map getHeaders() { 62 | return null; 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/batchmapper/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.batchmapper; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String socketPath = "test-socket-path"; 24 | int maxMessageSize = 2000; 25 | String infoFilePath = "test-info-file-path"; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/info/ServerInfoAccessorImplTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.info; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.junit.runners.JUnit4; 7 | 8 | import java.util.HashMap; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | import static org.junit.Assert.assertTrue; 12 | import static org.junit.Assert.fail; 13 | 14 | @RunWith(JUnit4.class) 15 | public class ServerInfoAccessorImplTest { 16 | private final ServerInfoAccessor underTest = new ServerInfoAccessorImpl(new ObjectMapper()); 17 | 18 | @Test 19 | public void given_localEnvironment_when_getNumaflowJavaSDKVersion_then_returnAValidVersion() { 20 | String got = this.underTest.getSDKVersion(); 21 | assertTrue(got.matches("^\\d+\\.\\d+\\.\\d+$")); 22 | } 23 | 24 | @Test 25 | public void given_writeServerInfo_when_read_then_returnExactSame() { 26 | ServerInfo testServerInfo = new ServerInfo( 27 | Protocol.TCP_PROTOCOL, 28 | Language.JAVA, 29 | "1.3.1-z", 30 | "0.4.3", 31 | new HashMap<>() {{ 32 | put("key1", "value1"); 33 | put("key2", "value2"); 34 | }} 35 | ); 36 | String testFilePath = "/var/tmp/test-path"; 37 | try { 38 | this.underTest.write(testServerInfo, testFilePath); 39 | ServerInfo got = this.underTest.read(testFilePath); 40 | assertEquals(testServerInfo.getLanguage(), got.getLanguage()); 41 | assertEquals(testServerInfo.getProtocol(), got.getProtocol()); 42 | assertEquals( 43 | testServerInfo.getMinimum_numaflow_version(), 44 | got.getMinimum_numaflow_version()); 45 | assertEquals(testServerInfo.getVersion(), got.getVersion()); 46 | assertEquals(testServerInfo.getMetadata(), got.getMetadata()); 47 | } catch (Exception e) { 48 | fail("Expected no exception."); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/mapper/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | int port = 8001; 24 | String socketPath = "test-socket-path"; 25 | int maxMessageSize = 2000; 26 | String infoFilePath = "test-info-file-path"; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/mapper/MapOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapper; 2 | 3 | import io.grpc.stub.StreamObserver; 4 | import io.numaproj.numaflow.map.v1.MapOuterClass; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.concurrent.CompletableFuture; 9 | 10 | public class MapOutputStreamObserver implements StreamObserver { 11 | List mapResponses = new ArrayList<>(); 12 | CompletableFuture done = new CompletableFuture<>(); 13 | Integer responseCount; 14 | 15 | public MapOutputStreamObserver(Integer responseCount) { 16 | this.responseCount = responseCount; 17 | } 18 | 19 | @Override 20 | public void onNext(MapOuterClass.MapResponse mapResponse) { 21 | mapResponses.add(mapResponse); 22 | if (mapResponses.size() == responseCount) { 23 | done.complete(null); 24 | } 25 | } 26 | 27 | @Override 28 | public void onError(Throwable throwable) { 29 | done.completeExceptionally(throwable); 30 | } 31 | 32 | @Override 33 | public void onCompleted() { 34 | done.complete(null); 35 | } 36 | 37 | public List getMapResponses() { 38 | return mapResponses; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/mapstreamer/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String socketPath = "test-socket-path"; 24 | int maxMessageSize = 2000; 25 | String infoFilePath = "test-info-file-path"; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/mapstreamer/MapStreamOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.mapstreamer; 2 | 3 | import io.grpc.stub.StreamObserver; 4 | import io.numaproj.numaflow.map.v1.MapOuterClass; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.concurrent.CompletableFuture; 9 | 10 | public class MapStreamOutputStreamObserver implements StreamObserver { 11 | List mapResponses = new ArrayList<>(); 12 | CompletableFuture done = new CompletableFuture<>(); 13 | Integer responseCount; 14 | 15 | public MapStreamOutputStreamObserver(Integer responseCount) { 16 | this.responseCount = responseCount; 17 | } 18 | 19 | @Override 20 | public void onNext(MapOuterClass.MapResponse mapResponse) { 21 | mapResponses.add(mapResponse); 22 | if (mapResponses.size() == responseCount) { 23 | done.complete(null); 24 | } 25 | } 26 | 27 | @Override 28 | public void onError(Throwable throwable) { 29 | done.completeExceptionally(throwable); 30 | } 31 | 32 | @Override 33 | public void onCompleted() { 34 | done.complete(null); 35 | } 36 | 37 | public List getMapResponses() { 38 | return mapResponses; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/reducer/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String socketPath = "test-socket-path"; 24 | int maxMessageSize = 2000; 25 | String infoFilePath = "test-info-file-path"; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/reducer/ReduceErrTestFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | 4 | import java.util.Arrays; 5 | 6 | public class ReduceErrTestFactory extends ReducerFactory { 7 | @Override 8 | public ReduceTestFn createReducer() { 9 | return new ReduceTestFn(); 10 | } 11 | 12 | public static class ReduceTestFn extends Reducer { 13 | private final int sum = 0; 14 | 15 | @Override 16 | public void addMessage(String[] keys, Datum datum, Metadata md) { 17 | throw new RuntimeException("unknown exception"); 18 | } 19 | 20 | @Override 21 | public MessageList getOutput(String[] keys, Metadata md) { 22 | String[] updatedKeys = Arrays 23 | .stream(keys) 24 | .map(c -> c + "-processed") 25 | .toArray(String[]::new); 26 | return MessageList 27 | .newBuilder() 28 | .addMessage(new Message(String.valueOf(sum).getBytes(), updatedKeys)) 29 | .build(); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/reducer/ReduceOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | import io.grpc.stub.StreamObserver; 4 | import io.numaproj.numaflow.reduce.v1.ReduceOuterClass; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.concurrent.atomic.AtomicReference; 10 | 11 | /** 12 | * This is a dummy implementation of reduce output stream observer for testing purpose. 13 | */ 14 | @Slf4j 15 | public class ReduceOutputStreamObserver implements StreamObserver { 16 | public AtomicReference completed = new AtomicReference<>(false); 17 | public AtomicReference> resultDatum = new AtomicReference<>( 18 | new ArrayList<>()); 19 | public Throwable t; 20 | 21 | @Override 22 | public void onNext(ReduceOuterClass.ReduceResponse response) { 23 | List receivedResponses = resultDatum.get(); 24 | receivedResponses.add(response); 25 | resultDatum.set(receivedResponses); 26 | } 27 | 28 | @Override 29 | public void onError(Throwable throwable) { 30 | t = throwable; 31 | } 32 | 33 | @Override 34 | public void onCompleted() { 35 | log.info("on completed executed"); 36 | this.completed.set(true); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/reducer/ReduceTestFactory.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducer; 2 | 3 | 4 | import java.util.Arrays; 5 | 6 | public class ReduceTestFactory extends ReducerFactory { 7 | @Override 8 | public ReduceTestFn createReducer() { 9 | return new ReduceTestFn(); 10 | } 11 | 12 | public static class ReduceTestFn extends Reducer { 13 | private int sum = 0; 14 | 15 | @Override 16 | public void addMessage(String[] keys, Datum datum, Metadata md) { 17 | sum += Integer.parseInt(new String(datum.getValue())); 18 | } 19 | 20 | @Override 21 | public MessageList getOutput(String[] keys, Metadata md) { 22 | String[] updatedKeys = Arrays 23 | .stream(keys) 24 | .map(c -> c + "-processed") 25 | .toArray(String[]::new); 26 | return MessageList 27 | .newBuilder() 28 | .addMessage(new Message(String.valueOf(sum).getBytes(), updatedKeys)) 29 | .build(); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/reducestreamer/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String socketPath = "test-socket-path"; 24 | int maxMessageSize = 2000; 25 | String infoFilePath = "test-info-file-path"; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/reducestreamer/ReduceOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.reducestreamer; 2 | 3 | import io.grpc.stub.StreamObserver; 4 | import io.numaproj.numaflow.reduce.v1.ReduceOuterClass; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.concurrent.atomic.AtomicReference; 10 | 11 | /** 12 | * This is a dummy implementation of reduce output stream observer for testing purpose. 13 | */ 14 | @Slf4j 15 | public class ReduceOutputStreamObserver implements StreamObserver { 16 | public AtomicReference completed = new AtomicReference<>(false); 17 | public AtomicReference> resultDatum = new AtomicReference<>( 18 | new ArrayList<>()); 19 | public Throwable t; 20 | 21 | @Override 22 | public void onNext(ReduceOuterClass.ReduceResponse response) { 23 | List receivedResponses = resultDatum.get(); 24 | receivedResponses.add(response); 25 | resultDatum.set(receivedResponses); 26 | } 27 | 28 | @Override 29 | public void onError(Throwable throwable) { 30 | t = throwable; 31 | } 32 | 33 | @Override 34 | public void onCompleted() { 35 | log.info("on completed executed"); 36 | this.completed.set(true); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/servingstore/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.servingstore; 2 | 3 | public class GRPCConfigTest { 4 | 5 | @org.junit.Test 6 | public void testDefaultGrpcConfig() { 7 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 8 | org.junit.Assert.assertNotNull(grpcConfig); 9 | org.junit.Assert.assertEquals( 10 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 11 | grpcConfig.getInfoFilePath()); 12 | org.junit.Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 13 | org.junit.Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 14 | org.junit.Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 15 | org.junit.Assert.assertTrue(grpcConfig.isLocal()); 16 | } 17 | 18 | @org.junit.Test 19 | public void testNewBuilder() { 20 | String socketPath = "test-socket-path"; 21 | int maxMessageSize = 2000; 22 | String infoFilePath = "test-info-file-path"; 23 | int port = 8001; 24 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 25 | .socketPath(socketPath) 26 | .maxMessageSize(maxMessageSize) 27 | .infoFilePath(infoFilePath) 28 | .port(port) 29 | .isLocal(false) 30 | .build(); 31 | org.junit.Assert.assertNotNull(grpcConfig); 32 | org.junit.Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 33 | org.junit.Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 34 | org.junit.Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 35 | org.junit.Assert.assertEquals(port, grpcConfig.getPort()); 36 | org.junit.Assert.assertFalse(grpcConfig.isLocal()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sessionreducer/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sessionreducer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals( 16 | Constants.DEFAULT_MESSAGE_SIZE, 17 | grpcConfig.getMaxMessageSize()); 18 | Assert.assertEquals( 19 | Constants.DEFAULT_SOCKET_PATH, 20 | grpcConfig.getSocketPath()); 21 | } 22 | 23 | @Test 24 | public void testNewBuilder() { 25 | String socketPath = "test-socket-path"; 26 | int maxMessageSize = 2000; 27 | String infoFilePath = "test-info-file-path"; 28 | int port = 8001; 29 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 30 | .socketPath(socketPath) 31 | .maxMessageSize(maxMessageSize) 32 | .infoFilePath(infoFilePath) 33 | .port(port) 34 | .isLocal(false) 35 | .build(); 36 | Assert.assertNotNull(grpcConfig); 37 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 38 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 39 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 40 | Assert.assertEquals(port, grpcConfig.getPort()); 41 | Assert.assertFalse(grpcConfig.isLocal()); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/shared/ExceptionUtilsTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.shared; 2 | 3 | import org.junit.Test; 4 | import static org.junit.Assert.assertTrue; 5 | import static org.junit.Assert.assertEquals; 6 | 7 | public class ExceptionUtilsTest { 8 | 9 | @Test 10 | public void testGetStackTrace_NullException() { 11 | String result = ExceptionUtils.getStackTrace(null); 12 | assertEquals("No exception provided.", result); 13 | } 14 | 15 | @Test 16 | public void testGetStackTrace_ValidException() { 17 | Exception exception = new Exception("Test exception"); 18 | String result = ExceptionUtils.getStackTrace(exception); 19 | assertTrue(result.contains("Test exception")); 20 | assertTrue(result.contains("ExceptionUtilsTest.testGetStackTrace_ValidException")); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/shared/GrpcServerUtilsTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.shared; 2 | 3 | import io.grpc.Context; 4 | import io.netty.channel.EventLoopGroup; 5 | import io.netty.channel.ServerChannel; 6 | import io.numaproj.numaflow.info.ContainerType; 7 | import io.numaproj.numaflow.info.ServerInfoAccessor; 8 | import org.junit.Assert; 9 | import org.junit.Test; 10 | import org.mockito.Mockito; 11 | 12 | public class GrpcServerUtilsTest { 13 | 14 | @Test 15 | public void testGetChannelTypeClass() { 16 | Class channelTypeClass = GrpcServerUtils.getChannelTypeClass(); 17 | Assert.assertNotNull(channelTypeClass); 18 | } 19 | 20 | @Test 21 | public void testCreateEventLoopGroup() { 22 | int threads = 4; 23 | String name = "test-group"; 24 | EventLoopGroup eventLoopGroup = GrpcServerUtils.createEventLoopGroup(threads, name); 25 | Assert.assertNotNull(eventLoopGroup); 26 | } 27 | 28 | @Test 29 | public void testWriteServerInfo() throws Exception { 30 | ServerInfoAccessor mockAccessor = Mockito.mock(ServerInfoAccessor.class); 31 | Mockito.when(mockAccessor.getSDKVersion()).thenReturn("1.0.0"); 32 | GrpcServerUtils.writeServerInfo(mockAccessor, null, "infoFilePath", ContainerType.MAPPER); 33 | Mockito 34 | .verify(mockAccessor, Mockito.times(1)) 35 | .write(Mockito.any(), Mockito.eq("infoFilePath")); 36 | } 37 | 38 | @Test 39 | public void testWindowStartTime() { 40 | Context.Key windowStartTime = GrpcServerUtils.WINDOW_START_TIME; 41 | Assert.assertNotNull(windowStartTime); 42 | } 43 | 44 | @Test 45 | public void testWindowEndTime() { 46 | Context.Key windowEndTime = GrpcServerUtils.WINDOW_END_TIME; 47 | Assert.assertNotNull(windowEndTime); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sideinput/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sideinput; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String serverInfoFile = "test-server-info-path"; 24 | String socketPath = "test-socket-path"; 25 | int maxMessageSize = 2000; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(serverInfoFile) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(serverInfoFile, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sinker/DatumStreamImplTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import lombok.AllArgsConstructor; 4 | import org.junit.Test; 5 | 6 | import java.time.Instant; 7 | import java.util.Map; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | import static org.junit.Assert.assertNull; 11 | 12 | public class DatumStreamImplTest { 13 | @Test 14 | public void datumStreamNextTest() throws InterruptedException { 15 | Datum datum1 = new TestDatum("test1"); 16 | Datum datum2 = new TestDatum("test2"); 17 | Datum eofDatum = HandlerDatum.EOF_DATUM; 18 | 19 | DatumIteratorImpl datumIterator = new DatumIteratorImpl(); 20 | 21 | datumIterator.writeMessage(datum1); 22 | datumIterator.writeMessage(datum2); 23 | datumIterator.writeMessage(eofDatum); 24 | 25 | assertEquals(datum1, datumIterator.next()); 26 | 27 | assertEquals(datum2, datumIterator.next()); 28 | 29 | 30 | assertNull(datumIterator.next()); 31 | } 32 | 33 | @AllArgsConstructor 34 | public static class TestDatum implements Datum { 35 | private String id; 36 | 37 | @Override 38 | public String[] getKeys() { 39 | return new String[0]; 40 | } 41 | 42 | @Override 43 | public byte[] getValue() { 44 | return new byte[0]; 45 | } 46 | 47 | @Override 48 | public Instant getEventTime() { 49 | return null; 50 | } 51 | 52 | @Override 53 | public Instant getWatermark() { 54 | return null; 55 | } 56 | 57 | @Override 58 | public String getId() { 59 | return null; 60 | } 61 | 62 | @Override 63 | public Map getHeaders() { 64 | return null; 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sinker/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String socketPath = "test-socket-path"; 24 | int maxMessageSize = 2000; 25 | String infoFilePath = "test-info-file-path"; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sinker/SinkOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sinker; 2 | 3 | 4 | import io.grpc.stub.StreamObserver; 5 | import io.numaproj.numaflow.sink.v1.SinkOuterClass; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.concurrent.atomic.AtomicReference; 10 | 11 | public class SinkOutputStreamObserver implements StreamObserver { 12 | private final List sinkResponses = new ArrayList<>(); 13 | public AtomicReference completed = new AtomicReference<>(false); 14 | public Throwable t; 15 | 16 | public List getSinkResponse() { 17 | return sinkResponses; 18 | } 19 | 20 | @Override 21 | public void onNext(SinkOuterClass.SinkResponse datum) { 22 | sinkResponses.add(datum); 23 | } 24 | 25 | @Override 26 | public void onError(Throwable throwable) { 27 | t = throwable; 28 | this.completed.set(true); 29 | } 30 | 31 | @Override 32 | public void onCompleted() { 33 | this.completed.set(true); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sourcer/AckOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | 4 | import io.grpc.stub.StreamObserver; 5 | import io.numaproj.numaflow.source.v1.SourceOuterClass; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.concurrent.atomic.AtomicReference; 10 | 11 | public class AckOutputStreamObserver implements StreamObserver { 12 | private final List ackResponses = new ArrayList<>(); 13 | public AtomicReference completed = new AtomicReference<>(false); 14 | public Throwable t; 15 | 16 | public List getSinkResponse() { 17 | return ackResponses; 18 | } 19 | 20 | @Override 21 | public void onNext(SourceOuterClass.AckResponse datum) { 22 | ackResponses.add(datum); 23 | } 24 | 25 | @Override 26 | public void onError(Throwable throwable) { 27 | t = throwable; 28 | this.completed.set(true); 29 | } 30 | 31 | @Override 32 | public void onCompleted() { 33 | this.completed.set(true); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sourcer/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String socketPath = "test-socket-path"; 24 | int maxMessageSize = 2000; 25 | String infoFilePath = "test-info-file-path"; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sourcer/NackOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | 4 | import io.grpc.stub.StreamObserver; 5 | import io.numaproj.numaflow.source.v1.SourceOuterClass; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.concurrent.atomic.AtomicReference; 10 | 11 | public class NackOutputStreamObserver implements StreamObserver { 12 | private final List nackResponses = new ArrayList<>(); 13 | public AtomicReference completed = new AtomicReference<>(false); 14 | public Throwable t; 15 | 16 | public List getNackResponse() { 17 | return nackResponses; 18 | } 19 | 20 | @Override 21 | public void onNext(SourceOuterClass.NackResponse datum) { 22 | nackResponses.add(datum); 23 | } 24 | 25 | @Override 26 | public void onError(Throwable throwable) { 27 | t = throwable; 28 | this.completed.set(true); 29 | } 30 | 31 | @Override 32 | public void onCompleted() { 33 | this.completed.set(true); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sourcer/ReadOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcer; 2 | 3 | 4 | import io.grpc.stub.StreamObserver; 5 | import io.numaproj.numaflow.source.v1.SourceOuterClass; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.concurrent.atomic.AtomicReference; 10 | 11 | public class ReadOutputStreamObserver implements StreamObserver { 12 | private final List readResponses = new ArrayList<>(); 13 | public AtomicReference completed = new AtomicReference<>(false); 14 | public Throwable t; 15 | 16 | public List getSinkResponse() { 17 | return readResponses; 18 | } 19 | 20 | @Override 21 | public void onNext(SourceOuterClass.ReadResponse datum) { 22 | readResponses.add(datum); 23 | } 24 | 25 | @Override 26 | public void onError(Throwable throwable) { 27 | t = throwable; 28 | this.completed.set(true); 29 | } 30 | 31 | @Override 32 | public void onCompleted() { 33 | this.completed.set(true); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sourcetransformer/GRPCConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class GRPCConfigTest { 7 | 8 | @Test 9 | public void testDefaultGrpcConfig() { 10 | GRPCConfig grpcConfig = GRPCConfig.defaultGrpcConfig(); 11 | Assert.assertNotNull(grpcConfig); 12 | Assert.assertEquals( 13 | Constants.DEFAULT_SERVER_INFO_FILE_PATH, 14 | grpcConfig.getInfoFilePath()); 15 | Assert.assertEquals(Constants.DEFAULT_MESSAGE_SIZE, grpcConfig.getMaxMessageSize()); 16 | Assert.assertEquals(Constants.DEFAULT_SOCKET_PATH, grpcConfig.getSocketPath()); 17 | Assert.assertEquals(Constants.DEFAULT_PORT, grpcConfig.getPort()); 18 | Assert.assertTrue(grpcConfig.isLocal()); 19 | } 20 | 21 | @Test 22 | public void testNewBuilder() { 23 | String socketPath = "test-socket-path"; 24 | int maxMessageSize = 2000; 25 | String infoFilePath = "test-info-file-path"; 26 | int port = 8001; 27 | GRPCConfig grpcConfig = GRPCConfig.newBuilder() 28 | .socketPath(socketPath) 29 | .maxMessageSize(maxMessageSize) 30 | .infoFilePath(infoFilePath) 31 | .port(port) 32 | .isLocal(false) 33 | .build(); 34 | Assert.assertNotNull(grpcConfig); 35 | Assert.assertEquals(socketPath, grpcConfig.getSocketPath()); 36 | Assert.assertEquals(maxMessageSize, grpcConfig.getMaxMessageSize()); 37 | Assert.assertEquals(infoFilePath, grpcConfig.getInfoFilePath()); 38 | Assert.assertEquals(port, grpcConfig.getPort()); 39 | Assert.assertFalse(grpcConfig.isLocal()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/io/numaproj/numaflow/sourcetransformer/TransformerOutputStreamObserver.java: -------------------------------------------------------------------------------- 1 | package io.numaproj.numaflow.sourcetransformer; 2 | 3 | import io.grpc.stub.StreamObserver; 4 | import io.numaproj.numaflow.sourcetransformer.v1.Sourcetransformer; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.concurrent.CompletableFuture; 9 | 10 | public class TransformerOutputStreamObserver implements StreamObserver { 11 | List responses = new ArrayList<>(); 12 | CompletableFuture done = new CompletableFuture<>(); 13 | Integer responseCount; 14 | 15 | public TransformerOutputStreamObserver(Integer responseCount) { 16 | this.responseCount = responseCount; 17 | } 18 | 19 | @Override 20 | public void onNext(Sourcetransformer.SourceTransformResponse mapResponse) { 21 | responses.add(mapResponse); 22 | if (responses.size() == responseCount) { 23 | done.complete(null); 24 | } 25 | } 26 | 27 | @Override 28 | public void onError(Throwable throwable) { 29 | done.completeExceptionally(throwable); 30 | } 31 | 32 | @Override 33 | public void onCompleted() { 34 | done.complete(null); 35 | } 36 | 37 | public List getResponses() { 38 | return responses; 39 | } 40 | } 41 | --------------------------------------------------------------------------------