├── pkg ├── errors │ └── doc.go ├── accumulator │ ├── doc.go │ ├── options_test.go │ ├── types.go │ ├── server_test.go │ └── options.go ├── apis │ └── proto │ │ ├── map │ │ └── v1 │ │ │ └── mockgen.go │ │ ├── serving │ │ └── v1 │ │ │ └── mockgen.go │ │ ├── sink │ │ └── v1 │ │ │ └── mockgen.go │ │ ├── sideinput │ │ └── v1 │ │ │ └── mockgen.go │ │ ├── reduce │ │ └── v1 │ │ │ └── mockgen.go │ │ ├── source │ │ └── v1 │ │ │ └── mockgen.go │ │ ├── sourcetransform │ │ └── v1 │ │ │ └── mockgen.go │ │ ├── accumulator │ │ └── v1 │ │ │ └── mockgen.go │ │ └── sessionreduce │ │ └── v1 │ │ └── mockgen.go ├── mapper │ ├── doc.go │ ├── options_test.go │ └── options.go ├── sinker │ ├── doc.go │ └── options_test.go ├── reducer │ ├── doc.go │ ├── options_test.go │ ├── options.go │ └── server_test.go ├── batchmapper │ ├── doc.go │ ├── options_test.go │ ├── options.go │ ├── types.go │ └── server_test.go ├── sourcer │ ├── doc.go │ ├── options_test.go │ └── options.go ├── mapstreamer │ ├── doc.go │ ├── options_test.go │ ├── types.go │ ├── server_test.go │ ├── options.go │ └── interface.go ├── sideinput │ ├── doc.go │ ├── options_test.go │ ├── interface.go │ ├── message.go │ ├── server_test.go │ └── options.go ├── reducestreamer │ ├── doc.go │ ├── options_test.go │ ├── options.go │ └── server_test.go ├── servingstore │ ├── doc.go │ └── message.go ├── sessionreducer │ ├── doc.go │ ├── options_test.go │ ├── server_test.go │ ├── types.go │ └── options.go ├── interface.go ├── info │ ├── options.go │ └── doc.go └── sourcetransformer │ ├── doc.go │ └── options.go ├── CHANGELOG.md ├── examples ├── sessionreducer │ ├── sum │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile │ └── counter │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile ├── reducer │ ├── counter │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ └── sum │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile ├── sinker │ ├── log │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── Dockerfile │ │ └── main.go │ ├── log_metadata │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── serve │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── fallback │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── on-success-log │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── Dockerfile │ │ └── main.go │ ├── failure_sink │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── on-success-sink │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile │ └── redis_sink │ │ ├── README.md │ │ ├── Makefile │ │ ├── go.mod │ │ └── Dockerfile ├── servingstore │ └── redis-store │ │ ├── README.md │ │ ├── Makefile │ │ ├── go.mod │ │ └── Dockerfile ├── mapper │ ├── flatmap │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── tickgen │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile │ ├── forward_message │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── even_odd │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── Dockerfile │ │ └── main.go │ ├── slow_cat │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile │ ├── cat │ │ ├── README.md │ │ ├── main.go │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile │ ├── cat_metadata │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── filter │ │ ├── README.md │ │ ├── Makefile │ │ ├── Dockerfile │ │ ├── main.go │ │ ├── go.mod │ │ └── util │ │ │ └── expand.go │ └── retry │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile ├── reducestreamer │ ├── counter │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── Dockerfile │ │ └── main.go │ └── sum │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ └── Dockerfile ├── mapstreamer │ └── flatmap_stream │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── Dockerfile │ │ └── main.go ├── sourcetransformer │ ├── assign_event_time │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── metadata_event_time │ │ ├── README.md │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── event_time_filter │ │ ├── README.md │ │ ├── main.go │ │ ├── Makefile │ │ ├── impl │ │ │ └── filter.go │ │ ├── go.mod │ │ └── Dockerfile │ ├── event_time_extractor │ │ ├── README.md │ │ ├── Makefile │ │ ├── Dockerfile │ │ ├── go.mod │ │ └── util │ │ │ └── expand.go │ ├── filter │ │ ├── README.md │ │ ├── Makefile │ │ ├── Dockerfile │ │ ├── go.mod │ │ └── util │ │ │ └── expand.go │ └── time_extraction_filter │ │ ├── Makefile │ │ ├── README.md │ │ ├── Dockerfile │ │ ├── go.mod │ │ └── util │ │ └── expand.go ├── sourcer │ ├── simple_source │ │ ├── main.go │ │ ├── README.md │ │ ├── Makefile │ │ ├── go.mod │ │ └── Dockerfile │ └── simple_source_with_metadata │ │ ├── README.md │ │ ├── main.go │ │ ├── Makefile │ │ ├── go.mod │ │ └── Dockerfile ├── sideinput │ ├── map_sideinput │ │ ├── go.mod │ │ ├── udf │ │ │ ├── go.mod │ │ │ ├── Makefile │ │ │ └── Dockerfile │ │ ├── Makefile │ │ └── Dockerfile │ ├── reduce_sideinput │ │ ├── go.mod │ │ ├── udf │ │ │ ├── go.mod │ │ │ ├── Makefile │ │ │ └── Dockerfile │ │ ├── Makefile │ │ └── Dockerfile │ ├── simple_sideinput │ │ ├── go.mod │ │ ├── udf │ │ │ ├── go.mod │ │ │ ├── Makefile │ │ │ └── Dockerfile │ │ ├── Makefile │ │ ├── Dockerfile │ │ └── pipeline.yaml │ ├── sideinput_function │ │ ├── go.mod │ │ ├── Makefile │ │ ├── main.go │ │ └── Dockerfile │ ├── sink_sideinput │ │ ├── Makefile │ │ ├── go.mod │ │ ├── README.md │ │ └── Dockerfile │ └── simple_source_with_sideinput │ │ ├── Makefile │ │ ├── main.go │ │ ├── go.mod │ │ ├── README.md │ │ └── Dockerfile ├── accumulator │ └── streamsorter │ │ ├── go.mod │ │ ├── Makefile │ │ ├── Dockerfile │ │ └── manifest │ │ └── stream-sorter-pipeline.yaml └── batchmapper │ └── batchmap_flatmap │ ├── go.mod │ ├── README.md │ ├── Makefile │ ├── pipeline.yaml │ ├── Dockerfile │ └── main.go ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md ├── .gitignore ├── Makefile ├── hack ├── tools.go └── protogen.sh ├── README.md └── go.mod /pkg/errors/doc.go: -------------------------------------------------------------------------------- 1 | // Package related to error handling and utilities 2 | package errors 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | -------------------------------------------------------------------------------- /examples/sessionreducer/sum/README.md: -------------------------------------------------------------------------------- 1 | # Sum 2 | 3 | An example User Defined Function that computes sum of events. 4 | -------------------------------------------------------------------------------- /examples/reducer/counter/README.md: -------------------------------------------------------------------------------- 1 | # Counter 2 | 3 | An example User Defined Function that counts the number of events. 4 | -------------------------------------------------------------------------------- /pkg/accumulator/doc.go: -------------------------------------------------------------------------------- 1 | // Package accumulator implements the server code for accumulate operation. 2 | package accumulator 3 | -------------------------------------------------------------------------------- /examples/sessionreducer/counter/README.md: -------------------------------------------------------------------------------- 1 | # Counter 2 | 3 | An example User Defined Function that counts the number of events. 4 | -------------------------------------------------------------------------------- /examples/sinker/log/README.md: -------------------------------------------------------------------------------- 1 | # Log 2 | 3 | An example User Defined Sink that prints the payload of each incoming datum element. 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Please refer to [Contributing](https://github.com/numaproj/numaproj/blob/main/CONTRIBUTING.md) 4 | -------------------------------------------------------------------------------- /examples/servingstore/redis-store/README.md: -------------------------------------------------------------------------------- 1 | # Redis Store 2 | 3 | This example demonstrates how to use Redis store and retrieve data. 4 | -------------------------------------------------------------------------------- /examples/sinker/log_metadata/README.md: -------------------------------------------------------------------------------- 1 | # Log Metadata 2 | 3 | An example User Defined Sink that prints the user metadata of datum. 4 | -------------------------------------------------------------------------------- /examples/mapper/flatmap/README.md: -------------------------------------------------------------------------------- 1 | # Flatmap 2 | 3 | An example User Defined Function that demonstrates how to write a `flatmap` User Defined Function. 4 | -------------------------------------------------------------------------------- /examples/mapper/tickgen/README.md: -------------------------------------------------------------------------------- 1 | # Flatmap 2 | 3 | An example User Defined Function that demonstrates how to write a `flatmap` User Defined Function. 4 | -------------------------------------------------------------------------------- /examples/reducer/sum/README.md: -------------------------------------------------------------------------------- 1 | # Sum 2 | 3 | This is a User Defined Function example which receives a stream of numbers and returns the sum of the numbers. 4 | -------------------------------------------------------------------------------- /examples/reducestreamer/counter/README.md: -------------------------------------------------------------------------------- 1 | # Counter 2 | 3 | An example User Defined Function that count the incoming events and output the count every 10 events. 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | Please refer to [Code of Conduct](https://github.com/numaproj/numaproj/blob/main/CODE_OF_CONDUCT.md) 4 | -------------------------------------------------------------------------------- /examples/mapper/forward_message/README.md: -------------------------------------------------------------------------------- 1 | # Forward Message 2 | 3 | This is a simple User Defined Function example which receives a message and returns the message as is. 4 | -------------------------------------------------------------------------------- /examples/sinker/serve/README.md: -------------------------------------------------------------------------------- 1 | # Serve 2 | 3 | An example User Defined Sink that returns payload of each incoming datum element to use as the result for a Serving source. 4 | -------------------------------------------------------------------------------- /examples/sinker/fallback/README.md: -------------------------------------------------------------------------------- 1 | # Fallback Sink Example 2 | 3 | An example User Defined Sink that returns a fallback response so that the payload will be sent to the fallback sink. -------------------------------------------------------------------------------- /examples/reducestreamer/sum/README.md: -------------------------------------------------------------------------------- 1 | # Sum 2 | 3 | This is a User Defined Function example which sum up the values for the given keys and output the sum when the sum is greater than 100 4 | -------------------------------------------------------------------------------- /pkg/apis/proto/map/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination mapmock/mapmock.go -package mapmock github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1 MapClient 4 | -------------------------------------------------------------------------------- /examples/mapstreamer/flatmap_stream/README.md: -------------------------------------------------------------------------------- 1 | # Flatmap 2 | 3 | An example User Defined Function that demonstrates how to write a 4 | `flatmap` User Defined Function in a streaming fashion. 5 | -------------------------------------------------------------------------------- /pkg/mapper/doc.go: -------------------------------------------------------------------------------- 1 | // Package mapper implements the server code for Map Operation. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/mapper 4 | 5 | package mapper 6 | -------------------------------------------------------------------------------- /pkg/sinker/doc.go: -------------------------------------------------------------------------------- 1 | // Package sinker implements the server code for user defined sink. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/sinker/ 4 | 5 | package sinker 6 | -------------------------------------------------------------------------------- /pkg/reducer/doc.go: -------------------------------------------------------------------------------- 1 | // Package reducer implements the server code for reduce operation. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/reducer/ 4 | 5 | package reducer 6 | -------------------------------------------------------------------------------- /pkg/apis/proto/serving/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination servingmock/serving.go -package servingmock github.com/numaproj/numaflow-go/pkg/apis/proto/serving/v1 ServingStoreClient 4 | -------------------------------------------------------------------------------- /pkg/apis/proto/sink/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination sinkmock/sinkmock.go -package sinkmock github.com/numaproj/numaflow-go/pkg/apis/proto/sink/v1 SinkClient,Sink_SinkFnClient 4 | -------------------------------------------------------------------------------- /examples/mapper/even_odd/README.md: -------------------------------------------------------------------------------- 1 | # Even Odd 2 | 3 | An example User Defined Function that returns the incoming message with `even` or `odd` key if the input message is an integer; otherwise, the message is dropped. 4 | -------------------------------------------------------------------------------- /pkg/batchmapper/doc.go: -------------------------------------------------------------------------------- 1 | // Package batchmapper implements the server code for BatchMap Operation. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/batchmapper/ 4 | 5 | package batchmapper 6 | -------------------------------------------------------------------------------- /pkg/sourcer/doc.go: -------------------------------------------------------------------------------- 1 | // Package sourcer implements the server code for User Defined Sourcer in golang. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/sourcer/ 4 | 5 | package sourcer 6 | -------------------------------------------------------------------------------- /pkg/apis/proto/sideinput/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination sideinputmock/sideinputmock.go -package sideinputmock github.com/numaproj/numaflow-go/pkg/apis/proto/sideinput/v1 SideInputClient 4 | -------------------------------------------------------------------------------- /pkg/mapstreamer/doc.go: -------------------------------------------------------------------------------- 1 | // Package mapstreamer implements the server code for Map Stream Operation. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/mapstreamer 4 | 5 | package mapstreamer 6 | -------------------------------------------------------------------------------- /examples/mapper/slow_cat/README.md: -------------------------------------------------------------------------------- 1 | # Slow Cat 2 | 3 | An example User Defined Function that outputs the same value as the input, but first does a sleep. This example is for the purpose of performing testing with a slow vertex. 4 | -------------------------------------------------------------------------------- /examples/sourcetransformer/assign_event_time/README.md: -------------------------------------------------------------------------------- 1 | # Assign Event Time 2 | 3 | This is a simple User Defined Function example which receives a message, changes the message event time to a time.Now(), and returns the message. 4 | -------------------------------------------------------------------------------- /pkg/apis/proto/reduce/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination reducemock/reducemock.go -package reducemock github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1 ReduceClient,Reduce_ReduceFnClient 4 | -------------------------------------------------------------------------------- /pkg/sideinput/doc.go: -------------------------------------------------------------------------------- 1 | // package sideinput implements the server code for user-defined SideInputs in golang. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/sideinput/ 4 | 5 | package sideinput 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | .DS_Store 3 | vendor/ 4 | dist/ 5 | # delve debug binaries 6 | cmd/**/debug 7 | hack/**/debug 8 | debug.test 9 | *.iml 10 | .coverage 11 | *.out 12 | test/*.cov 13 | site/ 14 | /go-diagrams/ 15 | *.idea/ 16 | -------------------------------------------------------------------------------- /pkg/reducestreamer/doc.go: -------------------------------------------------------------------------------- 1 | // Package reduceStreamer implements the server code for reduceStream operation. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/reducestreamer/ 4 | 5 | package reducestreamer 6 | -------------------------------------------------------------------------------- /pkg/servingstore/doc.go: -------------------------------------------------------------------------------- 1 | // Package servingstore implements the server code for writing user-defined serving store. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/servingstore/ 4 | 5 | package servingstore 6 | -------------------------------------------------------------------------------- /pkg/sessionreducer/doc.go: -------------------------------------------------------------------------------- 1 | // Package sessionreducer implements the server code for sessionReduce operation. 2 | 3 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/sessionreducer/ 4 | 5 | package sessionreducer 6 | -------------------------------------------------------------------------------- /pkg/apis/proto/source/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination sourcemock/sourcemock.go -package sourcemock github.com/numaproj/numaflow-go/pkg/apis/proto/source/v1 SourceClient,Source_ReadFnClient,Source_AckFnClient 4 | -------------------------------------------------------------------------------- /pkg/apis/proto/sourcetransform/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination transformmock/transformmock.go -package transformermock github.com/numaproj/numaflow-go/pkg/apis/proto/sourcetransform/v1 SourceTransformClient 4 | -------------------------------------------------------------------------------- /pkg/interface.go: -------------------------------------------------------------------------------- 1 | package numaflow 2 | 3 | import "context" 4 | 5 | // Server is the interface for the all the numaflow servers. 6 | type Server interface { 7 | // Start starts the server. 8 | Start(ctx context.Context) error 9 | } 10 | -------------------------------------------------------------------------------- /examples/sinker/on-success-log/README.md: -------------------------------------------------------------------------------- 1 | # OnSuccess Sink Logger Example 2 | 3 | An example User Defined Sink that simulates primary sink write successes and returns a 4 | * OnSuccess response on write successes so that the payload will be sent to the onSuccess sink. -------------------------------------------------------------------------------- /pkg/apis/proto/accumulator/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination accumulatoremock/accumulatormock.go -package accumulatormock github.com/numaproj/numaflow-go/pkg/apis/proto/accumulator/v1 AccumulatorClient,Accumulator_AccumulateFnClient 4 | -------------------------------------------------------------------------------- /examples/mapper/cat/README.md: -------------------------------------------------------------------------------- 1 | # Cat 2 | 3 | An example User Defined Function that echoes the messages. 4 | 5 | > Note: We already have a builtin for [cat](https://numaflow.numaproj.io/user-guide/user-defined-functions/map/builtin-functions/cat/) in numaflow Go, but not in Rust. -------------------------------------------------------------------------------- /pkg/apis/proto/sessionreduce/v1/mockgen.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | //go:generate mockgen -destination sessionreducemock/sessionreducemock.go -package sessionreducemock github.com/numaproj/numaflow-go/pkg/apis/proto/sessionreduce/v1 SessionReduceClient,SessionReduce_SessionReduceFnClient 4 | -------------------------------------------------------------------------------- /examples/sinker/failure_sink/README.md: -------------------------------------------------------------------------------- 1 | # Failure Sink Example 2 | 3 | For each message received, sink will create a `failure` response, containing id in the message. 4 | The [Retry Strategy](https://numaflow.numaproj.io/user-guide/sinks/retry-strategy/) decides how Numaflow responds to the failed messages. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all 2 | all: proto generate test 3 | 4 | .PHONY: test 5 | test: 6 | go test -race -v ./... 7 | 8 | .PHONY: proto 9 | proto: 10 | go mod vendor 11 | ./hack/protogen.sh 12 | rm -rf ./vendor 13 | go mod tidy 14 | 15 | .PHONY: generate 16 | generate: 17 | go generate ./... 18 | -------------------------------------------------------------------------------- /examples/sinker/on-success-sink/README.md: -------------------------------------------------------------------------------- 1 | # OnSuccess Sink Example 2 | 3 | An example User Defined Sink that simulates primary sink write failures/successes and returns a 4 | * Fallback response on write failures so that the payload will be sent to the fallback sink. 5 | * OnSuccess response on write successes so that the payload will be sent to the onSuccess sink. -------------------------------------------------------------------------------- /hack/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | 3 | // This package contains code generation utilities 4 | // This package imports things required by build scripts, to force `go mod` to see them as dependencies 5 | package tools 6 | 7 | import ( 8 | _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" 9 | _ "google.golang.org/protobuf/cmd/protoc-gen-go" 10 | ) 11 | -------------------------------------------------------------------------------- /examples/mapper/cat_metadata/README.md: -------------------------------------------------------------------------------- 1 | # Forward Metadata 2 | 3 | This example demonstrates a User Defined Map Function that simply forwards the message payload (like `cat`) but adds metadata. 4 | 5 | ## Metadata Details 6 | 7 | The function adds the following user metadata to each message: 8 | - Group: `map-group` 9 | - Key: `map-key` 10 | - Value: `map-value` 11 | -------------------------------------------------------------------------------- /pkg/info/options.go: -------------------------------------------------------------------------------- 1 | package info 2 | 3 | type options struct { 4 | svrInfoFilePath string 5 | } 6 | 7 | func defaultOptions() *options { 8 | return &options{} 9 | } 10 | 11 | type Option func(*options) 12 | 13 | // WithServerInfoFilePath sets the server info file path 14 | func WithServerInfoFilePath(f string) Option { 15 | return func(o *options) { 16 | o.svrInfoFilePath = f 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pkg/mapper/options_test.go: -------------------------------------------------------------------------------- 1 | package mapper 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/reducer/options_test.go: -------------------------------------------------------------------------------- 1 | package reducer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/sinker/options_test.go: -------------------------------------------------------------------------------- 1 | package sinker 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "simple_source/impl" 8 | 9 | "github.com/numaproj/numaflow-go/pkg/sourcer" 10 | ) 11 | 12 | func main() { 13 | simpleSource := impl.NewSimpleSource() 14 | err := sourcer.NewServer(simpleSource).Start(context.Background()) 15 | if err != nil { 16 | log.Panic("Failed to start source server : ", err) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pkg/accumulator/options_test.go: -------------------------------------------------------------------------------- 1 | package accumulator 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/batchmapper/options_test.go: -------------------------------------------------------------------------------- 1 | package batchmapper 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/mapstreamer/options_test.go: -------------------------------------------------------------------------------- 1 | package mapstreamer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/reducestreamer/options_test.go: -------------------------------------------------------------------------------- 1 | package reducestreamer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/sessionreducer/options_test.go: -------------------------------------------------------------------------------- 1 | package sessionreducer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | size = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 17 | assert.Equal(t, size, opts.maxMessageSize) 18 | } 19 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source_with_metadata/README.md: -------------------------------------------------------------------------------- 1 | # Simple Source With Metadata 2 | 3 | This example demonstrates a User Defined Source that generates a limited stream of messages (100 total) and attaches metadata to each message. 4 | 5 | ## Metadata Details 6 | 7 | Each message includes the following user metadata: 8 | - Group: `simple-source` 9 | - Key: `txn-id` 10 | - Value: A random UUID (e.g., `550e8400-e29b-41d4-a716-446655440000`) 11 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source_with_metadata/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "simple_source/impl" 8 | 9 | "github.com/numaproj/numaflow-go/pkg/sourcer" 10 | ) 11 | 12 | func main() { 13 | simpleSource := impl.NewSimpleSourceWithMetadata() 14 | err := sourcer.NewServer(simpleSource).Start(context.Background()) 15 | if err != nil { 16 | log.Panic("Failed to start source server : ", err) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /examples/sinker/redis_sink/README.md: -------------------------------------------------------------------------------- 1 | # Redis E2E Test Sink 2 | A User Defined Sink using redis hashes to store messages. 3 | The hash key is set by an environment variable `SINK_HASH_KEY` under the sink container spec. 4 | 5 | For each message received, the sink will store the message in the hash with the key being the payload of the message 6 | and the value being the no. of occurrences of that payload so far. 7 | 8 | This sink is used by Numaflow E2E testing. 9 | 10 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source/README.md: -------------------------------------------------------------------------------- 1 | # Simple Source 2 | 3 | A simple example of a user-defined source. 4 | The source maintains an array of messages and implements the `Read()`, `Ack()`, and `Pending()` methods. 5 | 6 | The `Read(x)` method returns the next x number of messages in the array. 7 | The `Ack()` method acknowledges the last batch of messages returned by `Read()`. 8 | The `Pending()` method returns 0 to indicate that the simple source always has 0 pending messages. -------------------------------------------------------------------------------- /examples/sourcetransformer/metadata_event_time/README.md: -------------------------------------------------------------------------------- 1 | # Event Time Metadata 2 | 3 | This example demonstrates a Source Transformer that updates the event time of each message to the current time. It also adds this new timestamp to the message's metadata. 4 | 5 | ## Metadata Details 6 | 7 | Each message includes the following user metadata: 8 | - Group: `event-time-group` 9 | - Key: `event-time` 10 | - Value: The new event time in RFC3339 format (e.g., `2023-10-27T10:00:00Z`) 11 | -------------------------------------------------------------------------------- /examples/mapper/cat/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/numaproj/numaflow-go/pkg/mapper" 8 | ) 9 | 10 | type Cat struct { 11 | } 12 | 13 | func (c *Cat) Map(ctx context.Context, keys []string, d mapper.Datum) mapper.Messages { 14 | return mapper.MessagesBuilder().Append(mapper.NewMessage(d.Value()).WithKeys(keys)) 15 | } 16 | 17 | func main() { 18 | err := mapper.NewServer(&Cat{}).Start(context.Background()) 19 | if err != nil { 20 | log.Panic("Failed to start cat server: ", err) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /pkg/sideinput/options_test.go: -------------------------------------------------------------------------------- 1 | package sideinput 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | // TestWithMaxMessageSize tests the WithMaxMessageSize option. 10 | // It should set the max message size in the options struct. 11 | func TestWithMaxMessageSize(t *testing.T) { 12 | var ( 13 | size = 1024 * 1024 * 10 14 | opts = &options{ 15 | maxMessageSize: defaultMaxMessageSize, 16 | } 17 | ) 18 | WithMaxMessageSize(1024 * 1024 * 10)(opts) 19 | assert.Equal(t, size, opts.maxMessageSize) 20 | } 21 | -------------------------------------------------------------------------------- /examples/mapper/cat/go.mod: -------------------------------------------------------------------------------- 1 | module cat 2 | 3 | go 1.24.1 4 | 5 | replace github.com/numaproj/numaflow-go => ../../.. 6 | 7 | require github.com/numaproj/numaflow-go v0.10.1 8 | 9 | require ( 10 | golang.org/x/net v0.29.0 // indirect 11 | golang.org/x/sync v0.8.0 // indirect 12 | golang.org/x/sys v0.25.0 // indirect 13 | golang.org/x/text v0.18.0 // indirect 14 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 15 | google.golang.org/grpc v1.67.1 // indirect 16 | google.golang.org/protobuf v1.34.2 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /examples/mapper/cat_metadata/go.mod: -------------------------------------------------------------------------------- 1 | module cat 2 | 3 | go 1.24.1 4 | 5 | replace github.com/numaproj/numaflow-go => ../../.. 6 | 7 | require github.com/numaproj/numaflow-go v0.10.1 8 | 9 | require ( 10 | golang.org/x/net v0.29.0 // indirect 11 | golang.org/x/sync v0.8.0 // indirect 12 | golang.org/x/sys v0.25.0 // indirect 13 | golang.org/x/text v0.18.0 // indirect 14 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 15 | google.golang.org/grpc v1.67.1 // indirect 16 | google.golang.org/protobuf v1.34.2 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /examples/mapper/filter/README.md: -------------------------------------------------------------------------------- 1 | # Filter 2 | 3 | An example User Defined Function that filter the messages based on expression implemented with `expr` and `sprig` libraries. For more details, check [builtin filter.](https://numaflow.numaproj.io/user-guide/user-defined-functions/map/builtin-functions/filter/) 4 | 5 | In this example, the filter is designed to select json payloads where: 6 | 7 | - The `id` is less than 100. 8 | - The `msg` is 'hello'. 9 | - The `desc` contains 'good'. 10 | 11 | > Note: We already have a builtin for `filter` in numaflow Go, but not in Rust. -------------------------------------------------------------------------------- /examples/sideinput/map_sideinput/go.mod: -------------------------------------------------------------------------------- 1 | module map_sideinput 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sys v0.25.0 // indirect 14 | golang.org/x/text v0.18.0 // indirect 15 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 16 | google.golang.org/grpc v1.67.1 // indirect 17 | google.golang.org/protobuf v1.34.2 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_filter/README.md: -------------------------------------------------------------------------------- 1 | # Event Time Filter 2 | 3 | This is a simple User Defined Function example which receives a message, applies the following data transformation, and returns the message. 4 | 5 | ## Data Transformation 6 | * If the message event time is **before** year 2022, drop the message. 7 | * If it's **within** year 2022, update the tag to "within_year_2022" and update the message event time to Jan 1st 2022. 8 | * Otherwise(exclusively after year 2022), update the tag to "after_year_2022" and update the message event time to Jan 1st 2023. 9 | -------------------------------------------------------------------------------- /examples/mapper/retry/README.md: -------------------------------------------------------------------------------- 1 | # Retry 2 | 3 | This is a User Defined Function example which imitates a need to retry a certain message some of the time, due to failure 4 | or some other reason. 5 | For each unique message received, it retries the message twice and then succeeds on the third time. 6 | It does a retry by publishing the message with a "retry" tag (which by the definition of the Pipeline should cause the message to be cycled back to it). 7 | 8 | Note that this somewhat silly example would only be used with a single Vertex replica since it makes use of an in-memory map. 9 | -------------------------------------------------------------------------------- /examples/sideinput/reduce_sideinput/go.mod: -------------------------------------------------------------------------------- 1 | module reduce_sideinput 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sys v0.25.0 // indirect 14 | golang.org/x/text v0.18.0 // indirect 15 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 16 | google.golang.org/grpc v1.67.1 // indirect 17 | google.golang.org/protobuf v1.34.2 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /examples/sideinput/simple_sideinput/go.mod: -------------------------------------------------------------------------------- 1 | module simple_sideinput 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sys v0.25.0 // indirect 14 | golang.org/x/text v0.18.0 // indirect 15 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 16 | google.golang.org/grpc v1.67.1 // indirect 17 | google.golang.org/protobuf v1.34.2 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_extractor/README.md: -------------------------------------------------------------------------------- 1 | # Event Time Extractor 2 | 3 | 4 | A eventTimeExtractor transformer extracts event time from the payload of the message, based on a user-provided `expression` and an optional `format` specification. 5 | 6 | - `expression` is used to compile the payload to a string representation of the event time. 7 | - `format` is used to convert the event time in string format to a time.Time object. 8 | 9 | 10 | In this example, we have a `expression`, which is used to compile the time of second item in the json payload to a string. -------------------------------------------------------------------------------- /examples/sideinput/sideinput_function/go.mod: -------------------------------------------------------------------------------- 1 | module even_odd_sideinput 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sys v0.25.0 // indirect 14 | golang.org/x/text v0.18.0 // indirect 15 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 16 | google.golang.org/grpc v1.67.1 // indirect 17 | google.golang.org/protobuf v1.34.2 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /examples/sinker/failure_sink/go.mod: -------------------------------------------------------------------------------- 1 | module retry-e2e 2 | 3 | go 1.24.1 4 | 5 | replace github.com/numaproj/numaflow-go => ../../.. 6 | 7 | require github.com/numaproj/numaflow-go v0.10.1 8 | 9 | require ( 10 | golang.org/x/net v0.29.0 // indirect 11 | golang.org/x/sync v0.8.0 // indirect 12 | golang.org/x/sys v0.25.0 // indirect 13 | golang.org/x/text v0.18.0 // indirect 14 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 15 | google.golang.org/grpc v1.67.1 // indirect 16 | google.golang.org/protobuf v1.34.2 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /examples/reducer/sum/go.mod: -------------------------------------------------------------------------------- 1 | module sum 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/mapper/retry/go.mod: -------------------------------------------------------------------------------- 1 | module retry 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sinker/log/go.mod: -------------------------------------------------------------------------------- 1 | module log_sink 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sourcetransformer/filter/README.md: -------------------------------------------------------------------------------- 1 | # Filter 2 | 3 | An example User Defined Function that filter the messages based on expression implemented with `expr` and `sprig` libraries. For more details, check [builtin filter.](https://numaflow.numaproj.io/user-guide/sources/transformer/builtin-transformers/filter/) 4 | 5 | 6 | In this example, the filter is designed to select json payloads where: 7 | 8 | - The `id` is less than 100. 9 | - The `msg` is 'hello'. 10 | - The `desc` contains 'good'. 11 | 12 | > Note: We already have a builtin for transformer `filter` in numaflow Go, but not in Rust. -------------------------------------------------------------------------------- /examples/mapper/even_odd/go.mod: -------------------------------------------------------------------------------- 1 | module even_odd 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/mapper/flatmap/go.mod: -------------------------------------------------------------------------------- 1 | module flatmap 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/mapper/slow_cat/go.mod: -------------------------------------------------------------------------------- 1 | module even_odd 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/mapper/tickgen/go.mod: -------------------------------------------------------------------------------- 1 | module tickgen 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/reducer/counter/go.mod: -------------------------------------------------------------------------------- 1 | module counter 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/reducestreamer/sum/go.mod: -------------------------------------------------------------------------------- 1 | module sum 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sinker/fallback/go.mod: -------------------------------------------------------------------------------- 1 | module fallback 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sinker/serve/go.mod: -------------------------------------------------------------------------------- 1 | module serve_sink 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/reducestreamer/counter/go.mod: -------------------------------------------------------------------------------- 1 | module counter 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sinker/log_metadata/go.mod: -------------------------------------------------------------------------------- 1 | module log_sink 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.2 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/accumulator/streamsorter/go.mod: -------------------------------------------------------------------------------- 1 | module streamsorter 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/mapper/forward_message/go.mod: -------------------------------------------------------------------------------- 1 | module forward_message 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sinker/on-success-log/go.mod: -------------------------------------------------------------------------------- 1 | module on_success_log 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.4 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.46.0 // indirect 13 | golang.org/x/sync v0.17.0 // indirect 14 | golang.org/x/sys v0.37.0 // indirect 15 | golang.org/x/text v0.30.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect 17 | google.golang.org/grpc v1.76.0 // indirect 18 | google.golang.org/protobuf v1.36.10 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sinker/on-success-sink/go.mod: -------------------------------------------------------------------------------- 1 | module on_success 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.4 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.46.0 // indirect 13 | golang.org/x/sync v0.17.0 // indirect 14 | golang.org/x/sys v0.37.0 // indirect 15 | golang.org/x/text v0.30.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect 17 | google.golang.org/grpc v1.76.0 // indirect 18 | google.golang.org/protobuf v1.36.10 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/batchmapper/batchmap_flatmap/go.mod: -------------------------------------------------------------------------------- 1 | module batchmap-flatmap 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/mapstreamer/flatmap_stream/go.mod: -------------------------------------------------------------------------------- 1 | module flatmap_stream 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_filter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "event_time_filter/impl" 8 | 9 | "github.com/numaproj/numaflow-go/pkg/sourcetransformer" 10 | ) 11 | 12 | func transform(_ context.Context, keys []string, d sourcetransformer.Datum) sourcetransformer.Messages { 13 | return impl.FilterEventTime(keys, d) 14 | } 15 | 16 | func main() { 17 | err := sourcetransformer.NewServer(sourcetransformer.SourceTransformFunc(transform)).Start(context.Background()) 18 | if err != nil { 19 | log.Panic("Failed to start source transform server: ", err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /examples/sourcetransformer/assign_event_time/go.mod: -------------------------------------------------------------------------------- 1 | module assign_event_time 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sourcetransformer/metadata_event_time/go.mod: -------------------------------------------------------------------------------- 1 | module assign_event_time 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require github.com/numaproj/numaflow-go v0.10.1 10 | 11 | require ( 12 | golang.org/x/net v0.29.0 // indirect 13 | golang.org/x/sync v0.8.0 // indirect 14 | golang.org/x/sys v0.25.0 // indirect 15 | golang.org/x/text v0.18.0 // indirect 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 17 | google.golang.org/grpc v1.67.1 // indirect 18 | google.golang.org/protobuf v1.34.2 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /examples/sessionreducer/sum/go.mod: -------------------------------------------------------------------------------- 1 | module sum 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/numaproj/numaflow-go v0.10.1 11 | go.uber.org/atomic v1.11.0 12 | ) 13 | 14 | require ( 15 | golang.org/x/net v0.29.0 // indirect 16 | golang.org/x/sync v0.8.0 // indirect 17 | golang.org/x/sys v0.25.0 // indirect 18 | golang.org/x/text v0.18.0 // indirect 19 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 20 | google.golang.org/grpc v1.67.1 // indirect 21 | google.golang.org/protobuf v1.34.2 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /examples/sessionreducer/counter/go.mod: -------------------------------------------------------------------------------- 1 | module counter 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/numaproj/numaflow-go v0.10.1 11 | go.uber.org/atomic v1.11.0 12 | ) 13 | 14 | require ( 15 | golang.org/x/net v0.29.0 // indirect 16 | golang.org/x/sync v0.8.0 // indirect 17 | golang.org/x/sys v0.25.0 // indirect 18 | golang.org/x/text v0.18.0 // indirect 19 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 20 | google.golang.org/grpc v1.67.1 // indirect 21 | google.golang.org/protobuf v1.34.2 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /pkg/sourcetransformer/doc.go: -------------------------------------------------------------------------------- 1 | // Package sourcetransformer implements the server code for Source Transformer in golang. 2 | // 3 | // Example Transform (extracting event time from the datum payload) 4 | // Transform includes both Map and EventTime assignment functionalities. 5 | // Although the input datum already contains EventTime and Watermark, it's up to the Transform implementor to 6 | // decide on whether to use them for generating new EventTime. 7 | // Transform can be used only at source vertex by source data transformer. 8 | 9 | // Examples: https://github.com/numaproj/numaflow-go/tree/main/examples/sourcetransformer/ 10 | 11 | package sourcetransformer 12 | -------------------------------------------------------------------------------- /examples/sideinput/map_sideinput/udf/go.mod: -------------------------------------------------------------------------------- 1 | module udf 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../../.. 8 | 9 | require ( 10 | github.com/fsnotify/fsnotify v1.6.0 11 | github.com/numaproj/numaflow-go v0.10.1 12 | ) 13 | 14 | require ( 15 | golang.org/x/net v0.29.0 // indirect 16 | golang.org/x/sync v0.8.0 // indirect 17 | golang.org/x/sys v0.25.0 // indirect 18 | golang.org/x/text v0.18.0 // indirect 19 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 20 | google.golang.org/grpc v1.67.1 // indirect 21 | google.golang.org/protobuf v1.34.2 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /examples/sideinput/reduce_sideinput/udf/go.mod: -------------------------------------------------------------------------------- 1 | module udf 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../../.. 8 | 9 | require ( 10 | github.com/fsnotify/fsnotify v1.7.0 11 | github.com/numaproj/numaflow-go v0.10.1 12 | ) 13 | 14 | require ( 15 | golang.org/x/net v0.29.0 // indirect 16 | golang.org/x/sync v0.8.0 // indirect 17 | golang.org/x/sys v0.25.0 // indirect 18 | golang.org/x/text v0.18.0 // indirect 19 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 20 | google.golang.org/grpc v1.67.1 // indirect 21 | google.golang.org/protobuf v1.34.2 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /examples/sideinput/simple_sideinput/udf/go.mod: -------------------------------------------------------------------------------- 1 | module udf 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../../.. 8 | 9 | require ( 10 | github.com/fsnotify/fsnotify v1.6.0 11 | github.com/numaproj/numaflow-go v0.10.1 12 | ) 13 | 14 | require ( 15 | golang.org/x/net v0.29.0 // indirect 16 | golang.org/x/sync v0.8.0 // indirect 17 | golang.org/x/sys v0.25.0 // indirect 18 | golang.org/x/text v0.18.0 // indirect 19 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 20 | google.golang.org/grpc v1.67.1 // indirect 21 | google.golang.org/protobuf v1.34.2 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /examples/batchmapper/batchmap_flatmap/README.md: -------------------------------------------------------------------------------- 1 | # Batch Map Flatmap 2 | 3 | An example User Defined Function that demonstrates how to write a batch map based `flatmap` User Defined Function. 4 | 5 | Some important considerations for batch map are as follows 6 | 7 | - The user will have to ensure that the BatchResponse is tagged with the correct request ID as this will be used by Numaflow for populating information required for system correctness like MessageID for the ISB deduplication. 8 | 9 | 10 | - The user will have to ensure that all the length of the BatchResponses is equal to the number of requests received. This means that for **each request** there is a BatchResponse. -------------------------------------------------------------------------------- /examples/mapper/cat/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-cat:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/cat-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target cat . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target cat . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist -------------------------------------------------------------------------------- /examples/sinker/log/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/sink-log:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/log-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target log . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target log . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapper/retry/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-retry:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/retry-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target retry . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target map-retry . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/reducer/sum/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-sum:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/sum-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sum . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sum . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /pkg/sideinput/interface.go: -------------------------------------------------------------------------------- 1 | package sideinput 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // SideInputRetriever is the interface for side input retrieval implementation. 8 | type SideInputRetriever interface { 9 | // RetrieveSideInput is the function to process each side-input request. 10 | RetrieveSideInput(ctx context.Context) Message 11 | } 12 | 13 | // RetrieveFunc is a utility type used to convert a RetrieveSideInput function to a SideInputRetriever. 14 | type RetrieveFunc func(ctx context.Context) Message 15 | 16 | // RetrieveSideInput implements the function of RetrieveSideInput function. 17 | func (mf RetrieveFunc) RetrieveSideInput(ctx context.Context) Message { 18 | return mf(ctx) 19 | } 20 | -------------------------------------------------------------------------------- /examples/sinker/serve/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/sink-serve:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/serve-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target serve . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target serve . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /hack/protogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | if [ "`command -v protoc`" = "" ]; then 8 | echo "Please install protobuf with - brew install protobuf" 9 | exit 1 10 | fi 11 | 12 | export PATH="$PATH:$(go env GOPATH)/bin" 13 | 14 | if [ "`command -v protoc-gen-go`" = "" ]; then 15 | go install -mod=vendor ./vendor/google.golang.org/protobuf/cmd/protoc-gen-go 16 | fi 17 | 18 | if [ "`command -v protoc-gen-go-grpc`" = "" ]; then 19 | go install -mod=vendor ./vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc 20 | fi 21 | 22 | protoc --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. -I. $(find pkg/apis/proto -name '*.proto') 23 | -------------------------------------------------------------------------------- /examples/mapper/filter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-filter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/filter-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target filter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target filter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sessionreducer/sum/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/session-sum:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/sum-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sum . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sum . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sinker/failure_sink/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/sink-failure:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/failure-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target failure . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target failure . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist -------------------------------------------------------------------------------- /examples/sinker/redis_sink/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = "quay.io/numaio/numaflow-go/redis-sink:${TAG}" 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/redis-sink-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target redis . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target redis . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapper/flatmap/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-flatmap:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/flatmap-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target flatmap . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target flatmap . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapper/tickgen/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-tickgen:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/tickgen-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target tickgen . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target tickgen . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/reducestreamer/sum/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/sum-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sum . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sum . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapper/even_odd/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-even-odd:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/even-odd-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target even-odd . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target even-odd . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapper/slow_cat/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-slow-cat:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/slow-cat-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target slow-cat . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target slow-cat . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/reducer/counter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-counter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/counter-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target counter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target counter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sinker/fallback/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/fb-sink-log:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/fallback-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target fallback . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target fallback . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sourcetransformer/filter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/mapt-filter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/filter-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target filter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target filter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sessionreducer/counter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/session-counter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/counter-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target counter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target counter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/accumulator/streamsorter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/stream-sorter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/streamsorter-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target streamsorter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target streamsorter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/reducestreamer/counter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/counter-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target counter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target counter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/map_sideinput/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-sideinput:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/map-sideinput-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/simple_sideinput/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/sideinput-example:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/sideinput-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/servingstore/redis-store/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/serving-redis-store:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/serving-redis-store-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target redis-store . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target redis-store . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/reduce_sideinput/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-sideinput:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/reduce-sideinput-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/sideinput_function/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/sideinput-function:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/sideinput-function-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/source-simple-source:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/simple-source-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target simple-source . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target simple-source . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist -------------------------------------------------------------------------------- /examples/batchmapper/batchmap_flatmap/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/batch-map-flatmap:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/batchmap-flatmap-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target batch-flatmap . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target batch-flatmap . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/map_sideinput/udf/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/map-sideinput-udf-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput-udf . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput-udf . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapper/cat_metadata/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-metadata:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/metadata-cat-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target metadata-cat-example . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target metadata-cat-example . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapper/forward_message/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-forward-message:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/forward-message-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target forward-message . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target forward-message . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/mapstreamer/flatmap_stream/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/flatmap-stream-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target flatmap_stream . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target flatmap_stream . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/sink_sideinput/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/redis-sink-with-sideinput:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/redis-sink-with-sideinput-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sinker/log_metadata/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/sink-metadata:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/metadata-log-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target metadata-log-example . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target metadata-log-example . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/reduce_sideinput/udf/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/reduce-sideinput-udf-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput-udf . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput-udf . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/simple_sideinput/udf/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/udf-sideinput-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target udf-sideinput . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target udf-sideinput . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sideinput/simple_source_with_sideinput/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/simple-source-with-sideinput:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/simple-source-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target sideinput . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target sideinput . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sourcetransformer/assign_event_time/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/mapt-assign-event-time:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/assign-event-time-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target assign-event-time . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target assign-event-time . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist -------------------------------------------------------------------------------- /examples/sinker/serve/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" 8 | ) 9 | 10 | // serveSink is a sinker implementation that logs the input to stdout 11 | type serveSink struct { 12 | } 13 | 14 | func (l *serveSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { 15 | result := sinksdk.ResponsesBuilder() 16 | for d := range datumStreamCh { 17 | id := d.ID() 18 | result = result.Append(sinksdk.ResponseServe(id, d.Value())) 19 | } 20 | return result 21 | } 22 | 23 | func main() { 24 | err := sinksdk.NewServer(&serveSink{}).Start(context.Background()) 25 | if err != nil { 26 | log.Panic("Failed to start sink function server: ", err) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_filter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/mapt-event-time-filter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/event-time-filter-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target event-time-filter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target event-time-filter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sourcetransformer/metadata_event_time/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/transformer-metadata:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/metadata-event-time-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target metadata-event-time . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target metadata-event-time . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_extractor/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/mapt-event-time-extractor:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/event-time-extractor-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target event-time-extractor . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target event-time-extractor . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source_with_metadata/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/source-metadata:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/simple-source-with-metadata-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target simple-source-with-metadata . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target simple-source-with-metadata . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /examples/servingstore/redis-store/go.mod: -------------------------------------------------------------------------------- 1 | module memory_store 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/numaproj/numaflow-go v0.10.1 11 | github.com/redis/go-redis/v9 v9.7.1 12 | ) 13 | 14 | require ( 15 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 16 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 17 | golang.org/x/net v0.29.0 // indirect 18 | golang.org/x/sync v0.8.0 // indirect 19 | golang.org/x/sys v0.25.0 // indirect 20 | golang.org/x/text v0.18.0 // indirect 21 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 22 | google.golang.org/grpc v1.67.1 // indirect 23 | google.golang.org/protobuf v1.34.2 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /examples/sourcetransformer/time_extraction_filter/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/mapt-time-extraction-filter:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/time-extraction-filter-example-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target time-extraction-filter . --push 15 | 16 | .PHONY: image 17 | image: build 18 | docker build -t ${IMAGE_REGISTRY} --target time-extraction-filter . 19 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 20 | 21 | clean: 22 | -rm -rf ./dist 23 | -------------------------------------------------------------------------------- /pkg/sideinput/message.go: -------------------------------------------------------------------------------- 1 | package sideinput 2 | 3 | // Message is used to wrap the data return by UserSideInput function. 4 | // It contains the data value for the given side input parameter requested. 5 | type Message struct { 6 | value []byte 7 | noBroadcast bool 8 | } 9 | 10 | // BroadcastMessage creates a new Message with the given value 11 | // This is used to broadcast the message to other side input vertices. 12 | func BroadcastMessage(value []byte) Message { 13 | return Message{value: value, noBroadcast: false} 14 | } 15 | 16 | // NoBroadcastMessage creates a new Message with noBroadcast flag set to true 17 | // This is used to drop the message and not to broadcast it to other side input vertices. 18 | func NoBroadcastMessage() Message { 19 | return Message{value: []byte{}, noBroadcast: true} 20 | } 21 | -------------------------------------------------------------------------------- /examples/reducer/counter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "strconv" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/reducer" 9 | ) 10 | 11 | func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { 12 | // count the incoming events 13 | var resultKeys = keys 14 | var resultVal []byte 15 | var counter = 0 16 | for range inputCh { 17 | counter++ 18 | } 19 | resultVal = []byte(strconv.Itoa(counter)) 20 | return reducer.MessagesBuilder().Append(reducer.NewMessage(resultVal).WithKeys(resultKeys)) 21 | } 22 | 23 | func main() { 24 | err := reducer.NewServer(reducer.SimpleCreatorWithReduceFn(reduceCounter)).Start(context.Background()) 25 | if err != nil { 26 | log.Panic("Failed to start server: ", err) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Numaflow Golang SDK 2 | 3 | This SDK provides the interfaces to implement [Numaflow](https://github.com/numaproj/numaflow) User Defined Sources, 4 | Source Transformer, Functions, Sinks or SideInputs in Golang. 5 | - Implement [User Defined Sources](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sourcer) 6 | - Implement [User Defined Source Transformers](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sourcetransformer) 7 | - Implement User Defined Functions 8 | - [Map](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/mapper) 9 | - [Reduce](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/reducer) 10 | - Implement [User Defined Sinks](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sinker) 11 | - Implement [User Defined SideInputs](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sideinput) -------------------------------------------------------------------------------- /examples/mapper/flatmap/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "strings" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/mapper" 9 | ) 10 | 11 | func mapFn(_ context.Context, keys []string, d mapper.Datum) mapper.Messages { 12 | msg := d.Value() 13 | _ = d.EventTime() // Event time is available 14 | _ = d.Watermark() // Watermark is available 15 | // Split the msg into an array with comma. 16 | strs := strings.Split(string(msg), ",") 17 | results := mapper.MessagesBuilder() 18 | for _, s := range strs { 19 | results = results.Append(mapper.NewMessage([]byte(s))) 20 | } 21 | return results 22 | } 23 | 24 | func main() { 25 | err := mapper.NewServer(mapper.MapperFunc(mapFn)).Start(context.Background()) 26 | if err != nil { 27 | log.Panic("Failed to start map function server: ", err) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pkg/sessionreducer/server_test.go: -------------------------------------------------------------------------------- 1 | package sessionreducer 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestReduceServer_Start(t *testing.T) { 13 | socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") 14 | defer func() { 15 | _ = os.RemoveAll(socketFile.Name()) 16 | }() 17 | 18 | serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") 19 | defer func() { 20 | _ = os.RemoveAll(serverInfoFile.Name()) 21 | }() 22 | 23 | // note: using actual uds connection 24 | ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) 25 | defer cancel() 26 | err := NewServer(&SessionSumCreator{}, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) 27 | assert.NoError(t, err) 28 | } 29 | -------------------------------------------------------------------------------- /examples/sinker/failure_sink/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" 9 | ) 10 | 11 | // failureSink is a sinker implementation that creates a failed response 12 | type failureSink struct { 13 | } 14 | 15 | func (l *failureSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { 16 | result := sinksdk.ResponsesBuilder() 17 | for d := range datumStreamCh { 18 | id := d.ID() 19 | result = result.Append(sinksdk.ResponseFailure(id, fmt.Sprintf("test retry strategy for id: %s", id))) 20 | } 21 | return result 22 | } 23 | 24 | func main() { 25 | err := sinksdk.NewServer(&failureSink{}).Start(context.Background()) 26 | if err != nil { 27 | log.Panic("Failed to start failure sink server: ", err) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/sideinput/sink_sideinput/go.mod: -------------------------------------------------------------------------------- 1 | module sink_sideinput 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/fsnotify/fsnotify v1.5.1 11 | github.com/go-redis/redis/v8 v8.11.5 12 | github.com/numaproj/numaflow-go v0.10.1 13 | ) 14 | 15 | require ( 16 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 17 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 18 | golang.org/x/net v0.29.0 // indirect 19 | golang.org/x/sync v0.8.0 // indirect 20 | golang.org/x/sys v0.25.0 // indirect 21 | golang.org/x/text v0.18.0 // indirect 22 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 23 | google.golang.org/grpc v1.67.1 // indirect 24 | google.golang.org/protobuf v1.34.2 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /examples/sideinput/sink_sideinput/README.md: -------------------------------------------------------------------------------- 1 | # Redis E2E Test Sink 2 | 3 | This Redis UDSink was specifically created for Numaflow end-to-end tests. 4 | It interacts with a Redis instance and, upon receiving data, 5 | it sets the content of the side input as the key in Redis, 6 | and the number of occurrences of that key as its value. 7 | The name of a hash is pipelineName:sinkName. 8 | 9 | # How It Works: 10 | Uses the fsnotify package to watch for changes to the side input file. 11 | Reads and stores the content of the side input file when created. 12 | For each received message, the sink: 13 | 1. Uses the content of the side input as the Redis hash field (key) 14 | 2. Increments the value associated with that key (number of occurrences of that key) 15 | 3. The hash name in Redis is determined by the environment variables `SINK_HASH_KEY` 16 | 17 | 18 | -------------------------------------------------------------------------------- /examples/sinker/redis_sink/go.mod: -------------------------------------------------------------------------------- 1 | module redis-e2e-test-sink 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/go-redis/redis/v8 v8.11.5 11 | github.com/numaproj/numaflow-go v0.10.1 12 | ) 13 | 14 | require ( 15 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 16 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 17 | github.com/fsnotify/fsnotify v1.5.1 // indirect 18 | golang.org/x/net v0.29.0 // indirect 19 | golang.org/x/sync v0.8.0 // indirect 20 | golang.org/x/sys v0.25.0 // indirect 21 | golang.org/x/text v0.18.0 // indirect 22 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 23 | google.golang.org/grpc v1.67.1 // indirect 24 | google.golang.org/protobuf v1.34.2 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /examples/sourcetransformer/time_extraction_filter/README.md: -------------------------------------------------------------------------------- 1 | # Time Extraction Filter 2 | 3 | 4 | It evaluates a message on a pipeline and if valid, extracts event time from the payload of the messsage. 5 | - `filterExpr` is used to evaluate and drop invalid messages. 6 | - `eventTimeExpr` is used to compile the payload to a string representation of the event time. 7 | - `format` is used to convert the event time in string format to a time.Time object. 8 | 9 | For more details, check [builtin timeExtractionFilter.](https://numaflow.numaproj.io/user-guide/sources/transformer/builtin-transformers/time-extraction-filter/). 10 | 11 | In this example we have used `filterExpr` and `eventTimeExpr`. 12 | 13 | - Selects json payloads with id < 100 using `filterExpr` 14 | - Selects time field of json payload using `eventTimeExpr` for a string representation of the event time. -------------------------------------------------------------------------------- /examples/sinker/on-success-sink/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/on-success-sink:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/on-success-sink-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target on_success . --push 15 | 16 | .PHONY: image-load 17 | image-load: build 18 | docker build -t ${IMAGE_REGISTRY} --platform linux/arm64 --target on_success . --load 19 | 20 | .PHONY: image 21 | image: build 22 | docker build -t ${IMAGE_REGISTRY} --target on_success . 23 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 24 | 25 | clean: 26 | -rm -rf ./dist 27 | -------------------------------------------------------------------------------- /examples/sinker/log_metadata/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" 9 | ) 10 | 11 | // logMetadataSink is a sinker implementation that logs the user metadata of the input 12 | type logMetadataSink struct { 13 | } 14 | 15 | func (l *logMetadataSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { 16 | result := sinksdk.ResponsesBuilder() 17 | for d := range datumStreamCh { 18 | umd := d.UserMetadata() 19 | fmt.Println("User Metadata: ", umd) 20 | id := d.ID() 21 | result = result.Append(sinksdk.ResponseOK(id)) 22 | } 23 | return result 24 | } 25 | 26 | func main() { 27 | err := sinksdk.NewServer(&logMetadataSink{}).Start(context.Background()) 28 | if err != nil { 29 | log.Panic("Failed to start sink function server: ", err) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/numaproj/numaflow-go 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | require ( 8 | github.com/stretchr/testify v1.9.0 9 | go.uber.org/atomic v1.11.0 10 | go.uber.org/mock v0.5.0 11 | golang.org/x/net v0.29.0 12 | golang.org/x/sync v0.8.0 13 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 14 | google.golang.org/grpc v1.67.1 15 | google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.4.0 16 | google.golang.org/protobuf v1.34.2 17 | ) 18 | 19 | require ( 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/kr/pretty v0.3.1 // indirect 22 | github.com/pmezard/go-difflib v1.0.0 // indirect 23 | golang.org/x/sys v0.25.0 // indirect 24 | golang.org/x/text v0.18.0 // indirect 25 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 26 | gopkg.in/yaml.v3 v3.0.1 // indirect 27 | ) 28 | 29 | retract v0.4.1 30 | -------------------------------------------------------------------------------- /examples/mapper/forward_message/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/numaproj/numaflow-go/pkg/mapper" 8 | ) 9 | 10 | // Forward is a mapper that directly forward the input to the output 11 | type Forward struct { 12 | } 13 | 14 | func (f *Forward) Map(ctx context.Context, keys []string, d mapper.Datum) mapper.Messages { 15 | // directly forward the input to the output 16 | val := d.Value() 17 | eventTime := d.EventTime() 18 | _ = eventTime 19 | watermark := d.Watermark() 20 | _ = watermark 21 | 22 | var resultKeys = keys 23 | var resultVal = val 24 | return mapper.MessagesBuilder().Append(mapper.NewMessage(resultVal).WithKeys(resultKeys)) 25 | } 26 | 27 | func main() { 28 | err := mapper.NewServer(&Forward{}).Start(context.Background()) 29 | if err != nil { 30 | log.Panic("Failed to start map function server: ", err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /examples/sinker/on-success-log/Makefile: -------------------------------------------------------------------------------- 1 | TAG ?= stable 2 | PUSH ?= false 3 | IMAGE_REGISTRY = quay.io/numaio/numaflow-go/on-success-log:${TAG} 4 | ARCHITECTURES = amd64 arm64 5 | 6 | .PHONY: build 7 | build: 8 | for arch in $(ARCHITECTURES); do \ 9 | CGO_ENABLED=0 GOOS=linux GOARCH=$${arch} go build -v -o ./dist/on-success-log-$${arch} main.go; \ 10 | done 11 | 12 | .PHONY: image-push 13 | image-push: build 14 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target on_success_log . --push 15 | 16 | .PHONY: image-load 17 | image-load: build 18 | docker buildx build -t ${IMAGE_REGISTRY} --platform linux/arm64,linux/amd64 --target on_success_log . --load 19 | 20 | .PHONY: image 21 | image: build 22 | docker build -t ${IMAGE_REGISTRY} --target on_success_log . 23 | @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi 24 | 25 | clean: 26 | -rm -rf ./dist 27 | -------------------------------------------------------------------------------- /examples/mapper/cat_metadata/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/mapper" 9 | ) 10 | 11 | type CatMetadata struct { 12 | } 13 | 14 | func (c *CatMetadata) Map(ctx context.Context, keys []string, d mapper.Datum) mapper.Messages { 15 | um := d.UserMetadata() 16 | // Print the user metadata for e2e tests in numaflow core 17 | fmt.Println("Groups at mapper: ", um.Groups()) 18 | if um == nil { 19 | um = mapper.NewUserMetadata() 20 | } 21 | um.CreateGroup("map-group") 22 | um.AddKV("map-group", "map-key", []byte("map-value")) 23 | return mapper.MessagesBuilder().Append(mapper.NewMessage(d.Value()).WithKeys(keys).WithUserMetadata(um)) 24 | } 25 | 26 | func main() { 27 | err := mapper.NewServer(&CatMetadata{}).Start(context.Background()) 28 | if err != nil { 29 | log.Panic("Failed to start cat metadata server: ", err) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_filter/impl/filter.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/numaproj/numaflow-go/pkg/sourcetransformer" 7 | ) 8 | 9 | func FilterEventTime(_ []string, d sourcetransformer.Datum) sourcetransformer.Messages { 10 | janFirst2022 := time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC) 11 | janFirst2023 := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) 12 | if d.EventTime().Before(janFirst2022) { 13 | return sourcetransformer.MessagesBuilder().Append(sourcetransformer.MessageToDrop(d.EventTime())) 14 | } else if d.EventTime().Before(janFirst2023) { 15 | return sourcetransformer.MessagesBuilder().Append(sourcetransformer.NewMessage(d.Value(), janFirst2022).WithTags([]string{"within_year_2022"})) 16 | } else { 17 | return sourcetransformer.MessagesBuilder().Append(sourcetransformer.NewMessage(d.Value(), janFirst2023).WithTags([]string{"after_year_2022"})) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pkg/mapstreamer/types.go: -------------------------------------------------------------------------------- 1 | package mapstreamer 2 | 3 | import "time" 4 | 5 | // handlerDatum implements the Datum interface and is used in the map stream functions. 6 | type handlerDatum struct { 7 | value []byte 8 | eventTime time.Time 9 | watermark time.Time 10 | headers map[string]string 11 | } 12 | 13 | func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time, headers map[string]string) Datum { 14 | return &handlerDatum{ 15 | value: value, 16 | eventTime: eventTime, 17 | watermark: watermark, 18 | headers: headers, 19 | } 20 | } 21 | 22 | func (h *handlerDatum) Value() []byte { 23 | return h.value 24 | } 25 | 26 | func (h *handlerDatum) EventTime() time.Time { 27 | return h.eventTime 28 | } 29 | 30 | func (h *handlerDatum) Watermark() time.Time { 31 | return h.watermark 32 | } 33 | 34 | func (h *handlerDatum) Headers() map[string]string { 35 | return h.headers 36 | } 37 | -------------------------------------------------------------------------------- /examples/sideinput/simple_source_with_sideinput/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/fsnotify/fsnotify" 8 | 9 | "github.com/numaproj/numaflow-go/pkg/sideinput" 10 | "github.com/numaproj/numaflow-go/pkg/sourcer" 11 | "simple_source_with_sideinput/impl" 12 | ) 13 | 14 | var sideInputName = "myticker" 15 | 16 | func main() { 17 | // Create a new fsnotify watcher 18 | watcher, err := fsnotify.NewWatcher() 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | defer watcher.Close() 23 | // Add a path to the watcher 24 | err = watcher.Add(sideinput.DirPath) 25 | if err != nil { 26 | log.Fatal(err) 27 | } 28 | go impl.FileWatcher(watcher, sideInputName) 29 | 30 | simpleSource := impl.NewSimpleSource() 31 | err = sourcer.NewServer(simpleSource).Start(context.Background()) 32 | if err != nil { 33 | log.Panic("Failed to start source server : ", err) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_filter/go.mod: -------------------------------------------------------------------------------- 1 | module event_time_filter 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/numaproj/numaflow-go v0.10.1 11 | github.com/stretchr/testify v1.9.0 12 | ) 13 | 14 | require ( 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/kr/text v0.2.0 // indirect 17 | github.com/pmezard/go-difflib v1.0.0 // indirect 18 | github.com/rogpeppe/go-internal v1.11.0 // indirect 19 | golang.org/x/net v0.29.0 // indirect 20 | golang.org/x/sync v0.8.0 // indirect 21 | golang.org/x/sys v0.25.0 // indirect 22 | golang.org/x/text v0.18.0 // indirect 23 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 24 | google.golang.org/grpc v1.67.1 // indirect 25 | google.golang.org/protobuf v1.34.2 // indirect 26 | gopkg.in/yaml.v3 v3.0.1 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /examples/sideinput/sideinput_function/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | sideinputsdk "github.com/numaproj/numaflow-go/pkg/sideinput" 6 | "log" 7 | ) 8 | 9 | var counter = 0 10 | 11 | // handle is the side input handler function. 12 | func handle(_ context.Context) sideinputsdk.Message { 13 | // BroadcastMessage() is used to broadcast the message with the given value to other side input vertices. 14 | // val must be converted to []byte. 15 | counter = (counter + 1) % 10 16 | if counter%2 == 0 { 17 | return sideinputsdk.BroadcastMessage([]byte(`e2e-even`)) 18 | } 19 | 20 | return sideinputsdk.BroadcastMessage([]byte(`e2e-odd`)) 21 | } 22 | 23 | func main() { 24 | // Start the side input server. 25 | err := sideinputsdk.NewSideInputServer(sideinputsdk.RetrieveFunc(handle)).Start(context.Background()) 26 | if err != nil { 27 | log.Panic("Failed to start side input server: ", err) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pkg/sessionreducer/types.go: -------------------------------------------------------------------------------- 1 | package sessionreducer 2 | 3 | import "time" 4 | 5 | // handlerDatum implements the Datum interface and is used in the SessionReduce functions. 6 | type handlerDatum struct { 7 | value []byte 8 | eventTime time.Time 9 | watermark time.Time 10 | headers map[string]string 11 | } 12 | 13 | func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time, headers map[string]string) Datum { 14 | return &handlerDatum{ 15 | value: value, 16 | eventTime: eventTime, 17 | watermark: watermark, 18 | headers: headers, 19 | } 20 | } 21 | 22 | func (h *handlerDatum) Value() []byte { 23 | return h.value 24 | } 25 | 26 | func (h *handlerDatum) EventTime() time.Time { 27 | return h.eventTime 28 | } 29 | 30 | func (h *handlerDatum) Watermark() time.Time { 31 | return h.watermark 32 | } 33 | 34 | func (h *handlerDatum) Headers() map[string]string { 35 | return h.headers 36 | } 37 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source/go.mod: -------------------------------------------------------------------------------- 1 | module simple_source 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/google/uuid v1.6.0 11 | github.com/numaproj/numaflow-go v0.10.1 12 | github.com/stretchr/testify v1.9.0 13 | ) 14 | 15 | require ( 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/kr/text v0.2.0 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | github.com/rogpeppe/go-internal v1.12.0 // indirect 20 | golang.org/x/net v0.29.0 // indirect 21 | golang.org/x/sync v0.8.0 // indirect 22 | golang.org/x/sys v0.25.0 // indirect 23 | golang.org/x/text v0.18.0 // indirect 24 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 25 | google.golang.org/grpc v1.67.1 // indirect 26 | google.golang.org/protobuf v1.34.2 // indirect 27 | gopkg.in/yaml.v3 v3.0.1 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /examples/mapper/retry/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/retry-${TARGETARCH} /bin/retry 11 | RUN chmod +x /bin/retry 12 | 13 | #################################################################################################### 14 | # retry 15 | #################################################################################################### 16 | FROM scratch AS retry 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/retry /bin/retry 20 | ENTRYPOINT [ "/bin/retry" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/fallback/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" 9 | ) 10 | 11 | // fbLogSink is a sinker implementation that logs the input to stdout 12 | type fbLogSink struct { 13 | } 14 | 15 | func (l *fbLogSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { 16 | result := sinksdk.ResponsesBuilder() 17 | for d := range datumStreamCh { 18 | _ = d.EventTime() 19 | _ = d.Watermark() 20 | id := d.ID() 21 | fmt.Println("Primary sink under maintenance, writing to fallback sink - ", string(d.Value())) 22 | // write to fallback sink 23 | result = result.Append(sinksdk.ResponseFallback(id)) 24 | } 25 | return result 26 | } 27 | 28 | func main() { 29 | err := sinksdk.NewServer(&fbLogSink{}).Start(context.Background()) 30 | if err != nil { 31 | log.Panic("Failed to start sink function server: ", err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source_with_metadata/go.mod: -------------------------------------------------------------------------------- 1 | module simple_source 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/google/uuid v1.6.0 11 | github.com/numaproj/numaflow-go v0.10.1 12 | github.com/stretchr/testify v1.9.0 13 | ) 14 | 15 | require ( 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/kr/text v0.2.0 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | github.com/rogpeppe/go-internal v1.12.0 // indirect 20 | golang.org/x/net v0.29.0 // indirect 21 | golang.org/x/sync v0.8.0 // indirect 22 | golang.org/x/sys v0.25.0 // indirect 23 | golang.org/x/text v0.18.0 // indirect 24 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 25 | google.golang.org/grpc v1.67.1 // indirect 26 | google.golang.org/protobuf v1.34.2 // indirect 27 | gopkg.in/yaml.v3 v3.0.1 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /examples/sinker/failure_sink/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/failure-${TARGETARCH} /bin/failure 11 | RUN chmod +x /bin/failure 12 | 13 | #################################################################################################### 14 | # failure 15 | #################################################################################################### 16 | FROM scratch AS failure 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/failure /bin/failure 20 | ENTRYPOINT [ "/bin/failure" ] -------------------------------------------------------------------------------- /examples/mapper/cat/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/cat-example-${TARGETARCH} /bin/cat-example 11 | RUN chmod +x /bin/cat-example 12 | 13 | #################################################################################################### 14 | # cat 15 | #################################################################################################### 16 | FROM scratch AS cat 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/cat-example /bin/cat-example 20 | ENTRYPOINT [ "/bin/cat-example" ] -------------------------------------------------------------------------------- /examples/sideinput/simple_source_with_sideinput/go.mod: -------------------------------------------------------------------------------- 1 | module simple_source_with_sideinput 2 | 3 | go 1.22 4 | 5 | toolchain go1.23.1 6 | 7 | replace github.com/numaproj/numaflow-go => ../../.. 8 | 9 | require ( 10 | github.com/fsnotify/fsnotify v1.6.0 11 | github.com/numaproj/numaflow-go v0.10.1 12 | github.com/stretchr/testify v1.9.0 13 | ) 14 | 15 | require ( 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/kr/pretty v0.3.1 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | github.com/rogpeppe/go-internal v1.12.0 // indirect 20 | golang.org/x/net v0.29.0 // indirect 21 | golang.org/x/sync v0.8.0 // indirect 22 | golang.org/x/sys v0.25.0 // indirect 23 | golang.org/x/text v0.18.0 // indirect 24 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 25 | google.golang.org/grpc v1.67.1 // indirect 26 | google.golang.org/protobuf v1.34.2 // indirect 27 | gopkg.in/yaml.v3 v3.0.1 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /examples/reducer/sum/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/sum-example-${TARGETARCH} /bin/sum-example 11 | RUN chmod +x /bin/sum-example 12 | 13 | #################################################################################################### 14 | # sum 15 | #################################################################################################### 16 | FROM scratch AS sum 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/sum-example /bin/sum-example 20 | ENTRYPOINT [ "/bin/sum-example" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/log/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/log-example-${TARGETARCH} /bin/log-example 11 | RUN chmod +x /bin/log-example 12 | 13 | #################################################################################################### 14 | # log 15 | #################################################################################################### 16 | FROM scratch AS log 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/log-example /bin/log-example 20 | ENTRYPOINT [ "/bin/log-example" ] 21 | -------------------------------------------------------------------------------- /examples/reducestreamer/sum/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/sum-example-${TARGETARCH} /bin/sum-example 11 | RUN chmod +x /bin/sum-example 12 | 13 | #################################################################################################### 14 | # sum 15 | #################################################################################################### 16 | FROM scratch AS sum 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/sum-example /bin/sum-example 20 | ENTRYPOINT [ "/bin/sum-example" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/redis_sink/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/redis-sink-${TARGETARCH} /bin/redis-sink 11 | RUN chmod +x /bin/redis-sink 12 | 13 | #################################################################################################### 14 | # redis 15 | #################################################################################################### 16 | FROM scratch AS redis 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/redis-sink /bin/redis-sink 20 | ENTRYPOINT [ "/bin/redis-sink" ] 21 | -------------------------------------------------------------------------------- /pkg/servingstore/message.go: -------------------------------------------------------------------------------- 1 | package servingstore 2 | 3 | // StoredResult is the data stored in the store per origin. 4 | type StoredResult struct { 5 | id string 6 | payloads []Payload 7 | } 8 | 9 | // NewStoredResult creates a new StoreResult from the provided origin and payloads. 10 | func NewStoredResult(id string, payloads []Payload) StoredResult { 11 | return StoredResult{id: id, payloads: payloads} 12 | } 13 | 14 | // Payload is each independent result stored in the Store for the given ID. 15 | type Payload struct { 16 | origin string 17 | value []byte 18 | } 19 | 20 | // NewPayload creates a new Payload from the given value. 21 | func NewPayload(origin string, value []byte) Payload { 22 | return Payload{origin: origin, value: value} 23 | } 24 | 25 | // Value returns the value of the Payload. 26 | func (p *Payload) Value() []byte { 27 | return p.value 28 | } 29 | 30 | // Origin returns the origin name. 31 | func (p *Payload) Origin() string { 32 | return p.origin 33 | } 34 | -------------------------------------------------------------------------------- /examples/sessionreducer/sum/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/sum-example-${TARGETARCH} /bin/sum-example 11 | RUN chmod +x /bin/sum-example 12 | 13 | #################################################################################################### 14 | # counter 15 | #################################################################################################### 16 | FROM scratch AS sum 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/sum-example /bin/sum-example 20 | ENTRYPOINT [ "/bin/sum-example" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/serve/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/serve-example-${TARGETARCH} /bin/serve-example 11 | RUN chmod +x /bin/serve-example 12 | 13 | #################################################################################################### 14 | # serve 15 | #################################################################################################### 16 | FROM scratch AS serve 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/serve-example /bin/serve-example 20 | ENTRYPOINT [ "/bin/serve-example" ] 21 | -------------------------------------------------------------------------------- /examples/mapper/filter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/filter-example-${TARGETARCH} /bin/filter-example 11 | RUN chmod +x /bin/filter-example 12 | 13 | #################################################################################################### 14 | # filter 15 | #################################################################################################### 16 | FROM scratch AS filter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/filter-example /bin/filter-example 20 | ENTRYPOINT [ "/bin/filter-example" ] 21 | -------------------------------------------------------------------------------- /examples/sourcetransformer/metadata_event_time/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/sourcetransformer" 9 | ) 10 | 11 | // AssignEventTime is a source transformer that assigns event time to the message. 12 | type MetadataEventTime struct { 13 | } 14 | 15 | func (m *MetadataEventTime) Transform(ctx context.Context, keys []string, d sourcetransformer.Datum) sourcetransformer.Messages { 16 | eventTime := time.Now() 17 | umd := d.UserMetadata() 18 | if umd != nil { 19 | umd.AddKV("event-time-group", "event-time", []byte(eventTime.Format(time.RFC3339))) 20 | } 21 | return sourcetransformer.MessagesBuilder().Append(sourcetransformer.NewMessage(d.Value(), eventTime).WithKeys(keys).WithUserMetadata(umd)) 22 | } 23 | 24 | func main() { 25 | err := sourcetransformer.NewServer(&MetadataEventTime{}).Start(context.Background()) 26 | if err != nil { 27 | log.Panic("Failed to start metadata event time server: ", err) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/batchmapper/batchmap_flatmap/pipeline.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: numaflow.numaproj.io/v1alpha1 2 | kind: Pipeline 3 | metadata: 4 | name: flatmap 5 | spec: 6 | vertices: 7 | - name: in 8 | source: 9 | http: {} 10 | - name: go-split 11 | scale: 12 | min: 1 13 | udf: 14 | container: 15 | # Split input message into an array with comma, see https://github.com/numaproj/numaflow-go/tree/main/examples/batchmapper/batchmap_flatmap 16 | image: quay.io/numaio/numaflow-go/batch-map-flatmap:stable 17 | imagePullPolicy: Always 18 | - name: go-udsink 19 | scale: 20 | min: 1 21 | sink: 22 | udsink: 23 | container: 24 | # https://github.com/numaproj/numaflow-go/tree/main/examples/sinker/log 25 | image: quay.io/numaio/numaflow-go/sink-log:stable 26 | imagePullPolicy: Always 27 | edges: 28 | - from: in 29 | to: go-split 30 | - from: go-split 31 | to: go-udsink 32 | -------------------------------------------------------------------------------- /pkg/info/doc.go: -------------------------------------------------------------------------------- 1 | // Package info is used for the gRPC server to provide the information such as protocol, sdk version, language, etc, to the client. 2 | // 3 | // The server information can be used by the client to determine: 4 | // - what is the right protocol to use (UDS or TCP) 5 | // - what is the numaflow sdk version used by the server 6 | // - what is language used by the server 7 | // 8 | // The gRPC server (UDF, UDSink, etc.) is supposed to have a shared file system with the client (numa container). 9 | // 10 | // Write() 11 | // The gPRC server must use this function to write the correct ServerInfo when it starts. 12 | // 13 | // Read() 14 | // The client is supposed to call the function to read the server information, before it starts to communicate with the gRPC server. 15 | // 16 | // WaitUntilReady() 17 | // This function checks if the server info file is ready to read. 18 | // The client (numa container) is supposed to call the function before it starts to Read() the server info file. 19 | package info 20 | -------------------------------------------------------------------------------- /examples/accumulator/streamsorter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/streamsorter-${TARGETARCH} /bin/streamsorter 11 | RUN chmod +x /bin/streamsorter 12 | 13 | #################################################################################################### 14 | # accumulator 15 | #################################################################################################### 16 | FROM scratch AS streamsorter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/streamsorter /bin/streamsorter 20 | ENTRYPOINT [ "/bin/streamsorter" ] 21 | -------------------------------------------------------------------------------- /examples/mapper/flatmap/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/flatmap-example-${TARGETARCH} /bin/flatmap-example 11 | RUN chmod +x /bin/flatmap-example 12 | 13 | #################################################################################################### 14 | # flatmap 15 | #################################################################################################### 16 | FROM scratch AS flatmap 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/flatmap-example /bin/flatmap-example 20 | ENTRYPOINT [ "/bin/flatmap-example" ] 21 | -------------------------------------------------------------------------------- /examples/mapper/tickgen/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/tickgen-example-${TARGETARCH} /bin/tickgen-example 11 | RUN chmod +x /bin/tickgen-example 12 | 13 | #################################################################################################### 14 | # flatmap 15 | #################################################################################################### 16 | FROM scratch AS tickgen 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/tickgen-example /bin/tickgen-example 20 | ENTRYPOINT [ "/bin/tickgen-example" ] 21 | -------------------------------------------------------------------------------- /examples/reducer/counter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/counter-example-${TARGETARCH} /bin/counter-example 11 | RUN chmod +x /bin/counter-example 12 | 13 | #################################################################################################### 14 | # counter 15 | #################################################################################################### 16 | FROM scratch AS counter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/counter-example /bin/counter-example 20 | ENTRYPOINT [ "/bin/counter-example" ] 21 | -------------------------------------------------------------------------------- /examples/sourcetransformer/filter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/filter-example-${TARGETARCH} /bin/filter-example 11 | RUN chmod +x /bin/filter-example 12 | 13 | #################################################################################################### 14 | # filter 15 | #################################################################################################### 16 | FROM scratch AS filter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/filter-example /bin/filter-example 20 | ENTRYPOINT [ "/bin/filter-example" ] 21 | -------------------------------------------------------------------------------- /examples/sideinput/simple_source_with_sideinput/README.md: -------------------------------------------------------------------------------- 1 | # Simple Source 2 | 3 | A simple example of a user-defined source. 4 | The SimpleSource is a basic source implementation that listens for changes in a side 5 | input file using a file watcher. When the specified side input file is created, 6 | its content is read, stored in a global variable, and sent through a global channel. 7 | This content is then consumed by the source and sent to the associated message channel. 8 | 9 | The source maintains an array of messages and implements the `Read()`, `Ack()`, and `Pending()` methods. 10 | 11 | The `Read(x)` function of the SimpleSource struct is responsible for reading data from the source (in this case, from the globalChan channel, which receives file content when a side input file is created) and sending this data to a specified message channel. 12 | The `Ack()` method acknowledges the last batch of messages returned by `Read()`. 13 | The `Pending()` method returns 0 to indicate that the simple source always has 0 pending messages. -------------------------------------------------------------------------------- /examples/mapper/even_odd/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/even-odd-example-${TARGETARCH} /bin/even-odd-example 11 | RUN chmod +x /bin/even-odd-example 12 | 13 | #################################################################################################### 14 | # even-odd 15 | #################################################################################################### 16 | FROM scratch AS even-odd 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/even-odd-example /bin/even-odd-example 20 | ENTRYPOINT [ "/bin/even-odd-example" ] 21 | -------------------------------------------------------------------------------- /examples/mapper/slow_cat/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/slow-cat-example-${TARGETARCH} /bin/slow-cat-example 11 | RUN chmod +x /bin/slow-cat-example 12 | 13 | #################################################################################################### 14 | # slow-cat 15 | #################################################################################################### 16 | FROM scratch AS slow-cat 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/slow-cat-example /bin/slow-cat-example 20 | ENTRYPOINT [ "/bin/slow-cat-example" ] 21 | -------------------------------------------------------------------------------- /examples/reducestreamer/counter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/counter-example-${TARGETARCH} /bin/counter-example 11 | RUN chmod +x /bin/counter-example 12 | 13 | #################################################################################################### 14 | # counter 15 | #################################################################################################### 16 | FROM scratch AS counter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/counter-example /bin/counter-example 20 | ENTRYPOINT [ "/bin/counter-example" ] 21 | -------------------------------------------------------------------------------- /examples/sessionreducer/counter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/counter-example-${TARGETARCH} /bin/counter-example 11 | RUN chmod +x /bin/counter-example 12 | 13 | #################################################################################################### 14 | # counter 15 | #################################################################################################### 16 | FROM scratch AS counter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/counter-example /bin/counter-example 20 | ENTRYPOINT [ "/bin/counter-example" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/fallback/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/fallback-example-${TARGETARCH} /bin/fallback-example 11 | RUN chmod +x /bin/fallback-example 12 | 13 | #################################################################################################### 14 | # fallback 15 | #################################################################################################### 16 | FROM scratch AS fallback 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/fallback-example /bin/fallback-example 20 | ENTRYPOINT [ "/bin/fallback-example" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/on-success-log/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/on-success-log-${TARGETARCH} /bin/on-success-log 11 | RUN chmod +x /bin/on-success-log 12 | 13 | #################################################################################################### 14 | # on_success_log 15 | #################################################################################################### 16 | FROM scratch AS on_success_log 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/on-success-log /bin/on-success-log 20 | ENTRYPOINT [ "/bin/on-success-log" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/on-success-sink/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/on-success-sink-${TARGETARCH} /bin/on-success-sink 11 | RUN chmod +x /bin/on-success-sink 12 | 13 | #################################################################################################### 14 | # on_success 15 | #################################################################################################### 16 | FROM scratch AS on_success 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/on-success-sink /bin/on-success-sink 20 | ENTRYPOINT [ "/bin/on-success-sink" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/log/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" 9 | ) 10 | 11 | // logSink is a sinker implementation that logs the input to stdout 12 | type logSink struct { 13 | } 14 | 15 | func (l *logSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { 16 | result := sinksdk.ResponsesBuilder() 17 | for d := range datumStreamCh { 18 | _ = d.EventTime() 19 | _ = d.Watermark() 20 | fmt.Println("User Defined Sink:", string(d.Value())) 21 | id := d.ID() 22 | result = result.Append(sinksdk.ResponseOK(id)) 23 | // if we are not able to write to sink and if we have a fallback sink configured 24 | // we can use sinksdk.ResponseFallback(id)) to write the message to fallback sink 25 | } 26 | return result 27 | } 28 | 29 | func main() { 30 | err := sinksdk.NewServer(&logSink{}).Start(context.Background()) 31 | if err != nil { 32 | log.Panic("Failed to start sink function server: ", err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /examples/sinker/on-success-log/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" 9 | ) 10 | 11 | // onSuccessLogSink is a sinker implementation that logs the input to stdout 12 | type onSuccessLogSink struct { 13 | } 14 | 15 | func (l *onSuccessLogSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { 16 | result := sinksdk.ResponsesBuilder() 17 | for d := range datumStreamCh { 18 | _ = d.EventTime() 19 | _ = d.Watermark() 20 | id := d.ID() 21 | fmt.Println("Primary sink write succeeded, writing to onSuccess sink - ", string(d.Value())) 22 | // write to onSuccess sink 23 | result = result.Append(sinksdk.ResponseOnSuccess(id, sinksdk.NewMessage([]byte("on-success-message")))) 24 | } 25 | return result 26 | } 27 | 28 | func main() { 29 | err := sinksdk.NewServer(&onSuccessLogSink{}).Start(context.Background()) 30 | if err != nil { 31 | log.Panic("Failed to start sink function server: ", err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/sourcetransformer/assign_event_time/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/sourcetransformer" 9 | ) 10 | 11 | // AssignEventTime is a source transformer that assigns event time to the message. 12 | type AssignEventTime struct { 13 | } 14 | 15 | func (a *AssignEventTime) Transform(ctx context.Context, keys []string, d sourcetransformer.Datum) sourcetransformer.Messages { 16 | // Update message event time to time.Now() 17 | eventTime := time.Now() 18 | // Log the action so that we can use it for testing. 19 | log.Printf("AssignEventTime: Assigning event time %v to message %s", eventTime, string(d.Value())) 20 | return sourcetransformer.MessagesBuilder().Append(sourcetransformer.NewMessage(d.Value(), eventTime).WithKeys(keys)) 21 | } 22 | 23 | func main() { 24 | err := sourcetransformer.NewServer(&AssignEventTime{}).Start(context.Background()) 25 | if err != nil { 26 | log.Panic("Failed to start assign event time server: ", err) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/accumulator/types.go: -------------------------------------------------------------------------------- 1 | package accumulator 2 | 3 | import "time" 4 | 5 | // handlerDatum implements the Datum interface and is used in the accumulator functions. 6 | type handlerDatum struct { 7 | value []byte 8 | id string 9 | eventTime time.Time 10 | watermark time.Time 11 | keys []string 12 | tags []string 13 | headers map[string]string 14 | } 15 | 16 | func (h *handlerDatum) Value() []byte { 17 | return h.value 18 | } 19 | 20 | func (h *handlerDatum) EventTime() time.Time { 21 | return h.eventTime 22 | } 23 | 24 | func (h *handlerDatum) Watermark() time.Time { 25 | return h.watermark 26 | } 27 | 28 | func (h *handlerDatum) Keys() []string { 29 | return h.keys 30 | } 31 | 32 | func (h *handlerDatum) UpdateValue(value []byte) { 33 | h.value = value 34 | } 35 | 36 | func (h *handlerDatum) SetTags(tags []string) { 37 | h.tags = tags 38 | } 39 | 40 | func (h *handlerDatum) Headers() map[string]string { 41 | return h.headers 42 | } 43 | 44 | func (h *handlerDatum) ID() string { 45 | return h.id 46 | } 47 | -------------------------------------------------------------------------------- /examples/sideinput/map_sideinput/udf/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/map-sideinput-udf-${TARGETARCH} /bin/map-sideinput-udf 11 | RUN chmod +x /bin/map-sideinput-udf 12 | 13 | #################################################################################################### 14 | # udf-sideinput 15 | #################################################################################################### 16 | FROM scratch AS sideinput-udf 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/map-sideinput-udf /bin/map-sideinput-udf 20 | ENTRYPOINT [ "/bin/map-sideinput-udf" ] -------------------------------------------------------------------------------- /examples/sideinput/sideinput_function/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/sideinput-function-${TARGETARCH} /bin/sideinput-function 11 | RUN chmod +x /bin/sideinput-function 12 | 13 | #################################################################################################### 14 | # sideinput 15 | #################################################################################################### 16 | FROM scratch AS sideinput 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/sideinput-function /bin/sideinput-function 20 | ENTRYPOINT [ "/bin/sideinput-function" ] -------------------------------------------------------------------------------- /examples/sideinput/simple_sideinput/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/sideinput-example-${TARGETARCH} /bin/sideinput-example 11 | RUN chmod +x /bin/sideinput-example 12 | 13 | #################################################################################################### 14 | # sideinput 15 | #################################################################################################### 16 | FROM scratch AS sideinput 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/sideinput-example /bin/sideinput-example 20 | ENTRYPOINT [ "/bin/sideinput-example" ] 21 | -------------------------------------------------------------------------------- /pkg/accumulator/server_test.go: -------------------------------------------------------------------------------- 1 | package accumulator 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestAccumulatorServer_Start(t *testing.T) { 13 | socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") 14 | defer func() { 15 | _ = os.RemoveAll(socketFile.Name()) 16 | }() 17 | 18 | serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") 19 | defer func() { 20 | _ = os.RemoveAll(serverInfoFile.Name()) 21 | }() 22 | 23 | var accumulatorHandle = SimpleCreatorWithAccumulateFn(func(ctx context.Context, input <-chan Datum, output chan<- Message) { 24 | for datum := range input { 25 | output <- MessageFromDatum(datum) 26 | } 27 | }) 28 | // note: using actual uds connection 29 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 30 | defer cancel() 31 | err := NewServer(accumulatorHandle, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) 32 | assert.NoError(t, err) 33 | } 34 | -------------------------------------------------------------------------------- /examples/batchmapper/batchmap_flatmap/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/batchmap-flatmap-${TARGETARCH} /bin/batchmap-flatmap 11 | RUN chmod +x /bin/batchmap-flatmap 12 | 13 | #################################################################################################### 14 | # batchmap-flatmap 15 | #################################################################################################### 16 | FROM scratch AS batch-flatmap 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/batchmap-flatmap /bin/batchmap-flatmap 20 | ENTRYPOINT [ "/bin/batchmap-flatmap" ] 21 | -------------------------------------------------------------------------------- /pkg/sideinput/server_test.go: -------------------------------------------------------------------------------- 1 | package sideinput 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | // TestServer_Start tests the Start method to check whether the grpc server is correctly started. 13 | func TestServer_Start(t *testing.T) { 14 | socketFile, _ := os.CreateTemp("/tmp", "numaflow-sideinput") 15 | defer func() { 16 | _ = os.RemoveAll(socketFile.Name()) 17 | }() 18 | 19 | serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") 20 | defer func() { 21 | _ = os.RemoveAll(serverInfoFile.Name()) 22 | }() 23 | 24 | var retrieveHandler = RetrieveFunc(func(ctx context.Context) Message { 25 | return BroadcastMessage([]byte("test")) 26 | }) 27 | // note: using actual uds connection 28 | ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) 29 | defer cancel() 30 | err := NewSideInputServer(retrieveHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) 31 | assert.NoError(t, err) 32 | } 33 | -------------------------------------------------------------------------------- /examples/batchmapper/batchmap_flatmap/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "strings" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/batchmapper" 9 | ) 10 | 11 | func batchMapFn(_ context.Context, datums <-chan batchmapper.Datum) batchmapper.BatchResponses { 12 | batchResponses := batchmapper.BatchResponsesBuilder() 13 | for d := range datums { 14 | msg := d.Value() 15 | _ = d.EventTime() // Event time is available 16 | _ = d.Watermark() // Watermark is available 17 | batchResponse := batchmapper.NewBatchResponse(d.Id()) 18 | strs := strings.Split(string(msg), ",") 19 | for _, s := range strs { 20 | batchResponse = batchResponse.Append(batchmapper.NewMessage([]byte(s))) 21 | } 22 | 23 | batchResponses = batchResponses.Append(batchResponse) 24 | } 25 | return batchResponses 26 | } 27 | 28 | func main() { 29 | err := batchmapper.NewServer(batchmapper.BatchMapperFunc(batchMapFn)).Start(context.Background()) 30 | if err != nil { 31 | log.Panic("Failed to start batch map function server: ", err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/servingstore/redis-store/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/serving-redis-store-${TARGETARCH} /bin/serving-redis-store 11 | RUN chmod +x /bin/serving-redis-store 12 | 13 | #################################################################################################### 14 | # flatmap 15 | #################################################################################################### 16 | FROM scratch AS redis-store 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/serving-redis-store /bin/serving-redis-store 20 | ENTRYPOINT [ "/bin/serving-redis-store" ] 21 | -------------------------------------------------------------------------------- /pkg/sourcer/options_test.go: -------------------------------------------------------------------------------- 1 | package sourcer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithMaxMessageSize(t *testing.T) { 10 | var ( 11 | testSize = 1024 * 1024 * 10 12 | opts = &options{ 13 | maxMessageSize: defaultMaxMessageSize, 14 | } 15 | ) 16 | WithMaxMessageSize(testSize)(opts) 17 | assert.Equal(t, testSize, opts.maxMessageSize) 18 | } 19 | 20 | func TestWithSockAddr(t *testing.T) { 21 | var ( 22 | testSocketAddr = "test-socket-address" 23 | opts = &options{ 24 | sockAddr: address, 25 | } 26 | ) 27 | WithSockAddr(testSocketAddr)(opts) 28 | assert.Equal(t, testSocketAddr, opts.sockAddr) 29 | } 30 | 31 | func TestWithServerInfoFilePath(t *testing.T) { 32 | var ( 33 | testServerInfoFilePath = "test-server-info-file-path" 34 | opts = &options{ 35 | maxMessageSize: defaultMaxMessageSize, 36 | } 37 | ) 38 | WithServerInfoFilePath(testServerInfoFilePath)(opts) 39 | assert.Equal(t, testServerInfoFilePath, opts.serverInfoFilePath) 40 | } 41 | -------------------------------------------------------------------------------- /examples/sideinput/map_sideinput/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/map-sideinput-example-${TARGETARCH} /bin/map-sideinput-example 11 | RUN chmod +x /bin/map-sideinput-example 12 | 13 | #################################################################################################### 14 | # sideinput 15 | #################################################################################################### 16 | FROM scratch AS sideinput 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/map-sideinput-example /bin/map-sideinput-example 20 | ENTRYPOINT [ "/bin/map-sideinput-example" ] 21 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/simple-source-example-${TARGETARCH} /bin/simple-source-example 11 | RUN chmod +x /bin/simple-source-example 12 | 13 | #################################################################################################### 14 | # simple-source 15 | #################################################################################################### 16 | FROM scratch AS simple-source 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/simple-source-example /bin/simple-source-example 20 | ENTRYPOINT [ "/bin/simple-source-example" ] -------------------------------------------------------------------------------- /examples/mapper/cat_metadata/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/metadata-cat-example-${TARGETARCH} /bin/metadata-cat-example 11 | RUN chmod +x /bin/metadata-cat-example 12 | 13 | #################################################################################################### 14 | # metadata-cat-example 15 | #################################################################################################### 16 | FROM scratch AS metadata-cat-example 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/metadata-cat-example /bin/metadata-cat-example 20 | ENTRYPOINT [ "/bin/metadata-cat-example" ] 21 | -------------------------------------------------------------------------------- /examples/mapstreamer/flatmap_stream/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/flatmap-stream-example-${TARGETARCH} /bin/flatmap-stream-example 11 | RUN chmod +x /bin/flatmap-stream-example 12 | 13 | #################################################################################################### 14 | # flatmap 15 | #################################################################################################### 16 | FROM scratch AS flatmap_stream 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/flatmap-stream-example /bin/flatmap-stream-example 20 | ENTRYPOINT [ "/bin/flatmap-stream-example" ] 21 | -------------------------------------------------------------------------------- /examples/sinker/log_metadata/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/metadata-log-example-${TARGETARCH} /bin/metadata-log-example 11 | RUN chmod +x /bin/metadata-log-example 12 | 13 | #################################################################################################### 14 | # metadata-log-example 15 | #################################################################################################### 16 | FROM scratch AS metadata-log-example 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/metadata-log-example /bin/metadata-log-example 20 | ENTRYPOINT [ "/bin/metadata-log-example" ] 21 | -------------------------------------------------------------------------------- /examples/sideinput/reduce_sideinput/udf/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/reduce-sideinput-udf-${TARGETARCH} /bin/reduce-sideinput-udf 11 | RUN chmod +x /bin/reduce-sideinput-udf 12 | 13 | #################################################################################################### 14 | # udf-sideinput 15 | #################################################################################################### 16 | FROM scratch AS sideinput-udf 17 | ARG ARCH 18 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 19 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 20 | COPY --from=base /bin/reduce-sideinput-udf /bin/reduce-sideinput-udf 21 | ENTRYPOINT [ "/bin/reduce-sideinput-udf" ] -------------------------------------------------------------------------------- /pkg/mapstreamer/server_test.go: -------------------------------------------------------------------------------- 1 | package mapstreamer 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestMapStreamServer_Start(t *testing.T) { 13 | socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") 14 | defer func() { 15 | _ = os.RemoveAll(socketFile.Name()) 16 | }() 17 | 18 | serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") 19 | defer func() { 20 | _ = os.RemoveAll(serverInfoFile.Name()) 21 | }() 22 | 23 | var mapStreamHandler = MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { 24 | msg := datum.Value() 25 | messageCh <- NewMessage(msg).WithKeys([]string{keys[0] + "_test"}) 26 | close(messageCh) 27 | }) 28 | // note: using actual uds connection 29 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 30 | defer cancel() 31 | err := NewServer(mapStreamHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) 32 | assert.NoError(t, err) 33 | } 34 | -------------------------------------------------------------------------------- /examples/mapper/forward_message/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/forward-message-example-${TARGETARCH} /bin/forward-message-example 11 | RUN chmod +x /bin/forward-message-example 12 | 13 | #################################################################################################### 14 | # forward-message 15 | #################################################################################################### 16 | FROM scratch AS forward-message 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/forward-message-example /bin/forward-message-example 20 | ENTRYPOINT [ "/bin/forward-message-example" ] 21 | -------------------------------------------------------------------------------- /examples/sideinput/reduce_sideinput/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/reduce-sideinput-example-${TARGETARCH} /bin/reduce-sideinput-example 11 | RUN chmod +x /bin/reduce-sideinput-example 12 | 13 | #################################################################################################### 14 | # sideinput 15 | #################################################################################################### 16 | FROM scratch AS sideinput 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/reduce-sideinput-example /bin/reduce-sideinput-example 20 | ENTRYPOINT [ "/bin/reduce-sideinput-example" ] 21 | -------------------------------------------------------------------------------- /examples/sideinput/simple_source_with_sideinput/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/simple-source-example-${TARGETARCH} /bin/simple-source-example 11 | RUN chmod +x /bin/simple-source-example 12 | 13 | #################################################################################################### 14 | # simple-source 15 | #################################################################################################### 16 | FROM scratch AS sideinput 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/simple-source-example /bin/simple-source-example 20 | ENTRYPOINT [ "/bin/simple-source-example" ] 21 | -------------------------------------------------------------------------------- /examples/sideinput/sink_sideinput/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/redis-sink-with-sideinput-${TARGETARCH} /bin/redis-sink-with-sideinput 11 | RUN chmod +x /bin/redis-sink-with-sideinput 12 | 13 | #################################################################################################### 14 | # redis 15 | #################################################################################################### 16 | FROM scratch AS sideinput 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/redis-sink-with-sideinput /bin/redis-sink-with-sideinput 20 | ENTRYPOINT [ "/bin/redis-sink-with-sideinput" ] 21 | -------------------------------------------------------------------------------- /examples/mapstreamer/flatmap_stream/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "strings" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/mapstreamer" 9 | ) 10 | 11 | // FlatMap is a MapStreamer that split the input message into multiple messages and stream them. 12 | type FlatMap struct { 13 | } 14 | 15 | func (f *FlatMap) MapStream(ctx context.Context, keys []string, d mapstreamer.Datum, messageCh chan<- mapstreamer.Message) { 16 | // we have to close to indicate the end of the stream, otherwise the client will wait forever. 17 | defer close(messageCh) 18 | msg := d.Value() 19 | _ = d.EventTime() // Event time is available 20 | _ = d.Watermark() // Watermark is available 21 | // Split the msg into an array with comma. 22 | strs := strings.Split(string(msg), ",") 23 | for _, s := range strs { 24 | messageCh <- mapstreamer.NewMessage([]byte(s)) 25 | } 26 | } 27 | 28 | func main() { 29 | err := mapstreamer.NewServer(&FlatMap{}).Start(context.Background()) 30 | if err != nil { 31 | log.Panic("Failed to start map stream function server: ", err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/accumulator/streamsorter/manifest/stream-sorter-pipeline.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: numaflow.numaproj.io/v1alpha1 2 | kind: Pipeline 3 | metadata: 4 | name: stream-sorter 5 | spec: 6 | limits: 7 | readBatchSize: 1 8 | vertices: 9 | - name: http-one 10 | scale: 11 | min: 1 12 | max: 1 13 | source: 14 | http: {} 15 | - name: http-two 16 | scale: 17 | min: 1 18 | max: 1 19 | source: 20 | http: {} 21 | - name: accum 22 | udf: 23 | container: 24 | image: quay.io/numaio/numaflow-go/stream-sorter:stable 25 | groupBy: 26 | window: 27 | accumulator: 28 | timeout: 10s 29 | keyed: true 30 | storage: 31 | persistentVolumeClaim: 32 | volumeSize: 1Gi 33 | - name: sink 34 | scale: 35 | min: 1 36 | max: 1 37 | sink: 38 | log: {} 39 | edges: 40 | - from: http-one 41 | to: accum 42 | - from: http-two 43 | to: accum 44 | - from: accum 45 | to: sink 46 | -------------------------------------------------------------------------------- /examples/sideinput/simple_sideinput/udf/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/udf-sideinput-example-${TARGETARCH} /bin/udf-sideinput-example 11 | RUN chmod +x /bin/udf-sideinput-example 12 | 13 | #################################################################################################### 14 | # udf-sideinput 15 | #################################################################################################### 16 | FROM scratch AS udf-sideinput 17 | ARG ARCH 18 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 19 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 20 | COPY --from=base /bin/udf-sideinput-example /bin/udf-sideinput-example 21 | ENTRYPOINT [ "/bin/udf-sideinput-example" ] 22 | -------------------------------------------------------------------------------- /examples/reducestreamer/counter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | 7 | "github.com/numaproj/numaflow-go/pkg/reducestreamer" 8 | ) 9 | 10 | // reduceCounter is a ReduceStreamer that count the incoming events and output the count every 10 events. 11 | // The output message is the count of the events. 12 | func reduceCounter(_ context.Context, keys []string, inputCh <-chan reducestreamer.Datum, outputCh chan<- reducestreamer.Message, md reducestreamer.Metadata) { 13 | // count the incoming events 14 | var resultKeys = keys 15 | var resultVal []byte 16 | var counter = 0 17 | for range inputCh { 18 | counter++ 19 | if counter >= 10 { 20 | resultVal = []byte(strconv.Itoa(counter)) 21 | outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) 22 | counter = 0 23 | } 24 | } 25 | resultVal = []byte(strconv.Itoa(counter)) 26 | outputCh <- reducestreamer.NewMessage(resultVal).WithKeys(resultKeys) 27 | } 28 | 29 | func main() { 30 | reducestreamer.NewServer(reducestreamer.SimpleCreatorWithReduceStreamFn(reduceCounter)).Start(context.Background()) 31 | } 32 | -------------------------------------------------------------------------------- /examples/sourcetransformer/assign_event_time/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/assign-event-time-example-${TARGETARCH} /bin/assign-event-time-example 11 | RUN chmod +x /bin/assign-event-time-example 12 | 13 | #################################################################################################### 14 | # assign-event-time 15 | #################################################################################################### 16 | FROM scratch AS assign-event-time 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/assign-event-time-example /bin/assign-event-time-example 20 | ENTRYPOINT [ "/bin/assign-event-time-example" ] -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_filter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/event-time-filter-example-${TARGETARCH} /bin/event-time-filter-example 11 | RUN chmod +x /bin/event-time-filter-example 12 | 13 | #################################################################################################### 14 | # event-time-filter 15 | #################################################################################################### 16 | FROM scratch AS event-time-filter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/event-time-filter-example /bin/event-time-filter-example 20 | ENTRYPOINT [ "/bin/event-time-filter-example" ] 21 | -------------------------------------------------------------------------------- /examples/mapper/even_odd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "strconv" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/mapper" 9 | ) 10 | 11 | type EvenOdd struct { 12 | } 13 | 14 | func (e *EvenOdd) Map(ctx context.Context, keys []string, d mapper.Datum) mapper.Messages { 15 | msg := d.Value() 16 | _ = d.EventTime() // Event time is available 17 | _ = d.Watermark() // Watermark is available 18 | // If msg is not an integer, drop it, otherwise return it with "even" or "odd" key. 19 | if num, err := strconv.Atoi(string(msg)); err != nil { 20 | return mapper.MessagesBuilder().Append(mapper.MessageToDrop()) 21 | } else if num%2 == 0 { 22 | return mapper.MessagesBuilder().Append(mapper.NewMessage(msg).WithKeys([]string{"even"}).WithTags([]string{"even-tag"})) 23 | } else { 24 | return mapper.MessagesBuilder().Append(mapper.NewMessage(msg).WithKeys([]string{"odd"}).WithTags([]string{"odd-tag"})) 25 | } 26 | } 27 | 28 | func main() { 29 | err := mapper.NewServer(&EvenOdd{}).Start(context.Background()) 30 | if err != nil { 31 | log.Panic("Failed to start map function server: ", err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_extractor/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/event-time-extractor-example-${TARGETARCH} /bin/event-time-extractor-example 11 | RUN chmod +x /bin/event-time-extractor-example 12 | 13 | #################################################################################################### 14 | # event-time-extractor 15 | #################################################################################################### 16 | FROM scratch AS event-time-extractor 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/event-time-extractor-example /bin/event-time-extractor-example 20 | ENTRYPOINT [ "/bin/event-time-extractor-example" ] -------------------------------------------------------------------------------- /examples/sourcetransformer/metadata_event_time/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/metadata-event-time-example-${TARGETARCH} /bin/metadata-event-time-example 11 | RUN chmod +x /bin/metadata-event-time-example 12 | 13 | #################################################################################################### 14 | # metadata-event-time 15 | #################################################################################################### 16 | FROM scratch AS metadata-event-time 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/metadata-event-time-example /bin/metadata-event-time-example 20 | ENTRYPOINT [ "/bin/metadata-event-time-example" ] 21 | -------------------------------------------------------------------------------- /examples/sourcetransformer/time_extraction_filter/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/time-extraction-filter-example-${TARGETARCH} /bin/time-extraction-filter-example 11 | RUN chmod +x /bin/time-extraction-filter-example 12 | 13 | #################################################################################################### 14 | # time-extraction-filter 15 | #################################################################################################### 16 | FROM scratch AS time-extraction-filter 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/time-extraction-filter-example /bin/time-extraction-filter-example 20 | ENTRYPOINT [ "/bin/time-extraction-filter-example" ] -------------------------------------------------------------------------------- /examples/sideinput/simple_sideinput/pipeline.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: numaflow.numaproj.io/v1alpha1 2 | kind: Pipeline 3 | metadata: 4 | name: my-pipeline 5 | spec: 6 | sideInputs: 7 | - name: myticker 8 | container: 9 | image: "quay.io/numaio/numaflow-go/sideinput-example:v0.5.0" 10 | imagePullPolicy: Always 11 | trigger: 12 | schedule: "*/2 * * * *" 13 | # timezone: America/Los_Angeles 14 | vertices: 15 | - name: in 16 | source: 17 | # A self data generating source 18 | generator: 19 | rpu: 1 20 | duration: 1s 21 | - name: si-log 22 | udf: 23 | container: 24 | image: "quay.io/numaio/numaflow-go/udf-sideinput-example:v0.5.0" 25 | imagePullPolicy: Always 26 | containerTemplate: 27 | env: 28 | - name: NUMAFLOW_DEBUG 29 | value: "true" # DO NOT forget the double quotes!!! 30 | sideInputs: 31 | - myticker 32 | - name: out 33 | sink: 34 | # A simple log printing sink 35 | log: {} 36 | edges: 37 | - from: in 38 | to: si-log 39 | - from: si-log 40 | to: out -------------------------------------------------------------------------------- /pkg/mapper/options.go: -------------------------------------------------------------------------------- 1 | package mapper 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/reducer/options.go: -------------------------------------------------------------------------------- 1 | package reducer 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/sourcer/options.go: -------------------------------------------------------------------------------- 1 | package sourcer 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /examples/mapper/filter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "filter/util" 6 | "log" 7 | 8 | "github.com/numaproj/numaflow-go/pkg/mapper" 9 | ) 10 | 11 | type Filter struct { 12 | expression string 13 | } 14 | 15 | func (f Filter) Map(ctx context.Context, keys []string, d mapper.Datum) mapper.Messages { 16 | resultMsg, err := f.apply(d.Value()) 17 | if err != nil { 18 | log.Fatalf("Filter map function apply got an error: %v", err) 19 | } 20 | return mapper.MessagesBuilder().Append(resultMsg) 21 | } 22 | 23 | func (f Filter) apply(msg []byte) (mapper.Message, error) { 24 | result, err := util.EvalBool(f.expression, msg) 25 | if err != nil { 26 | return mapper.MessageToDrop(), err 27 | } 28 | if result { 29 | return mapper.NewMessage(msg), nil 30 | } 31 | return mapper.MessageToDrop(), nil 32 | } 33 | 34 | func main() { 35 | 36 | f := Filter{ 37 | expression: `int(json(payload).id) < 100 && json(payload).msg == 'hello' && sprig.contains('good', string(json(payload).desc))`, 38 | } 39 | err := mapper.NewServer(f).Start(context.Background()) 40 | if err != nil { 41 | log.Panic("Failed to start filter server: ", err) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pkg/accumulator/options.go: -------------------------------------------------------------------------------- 1 | package accumulator 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func DefaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/batchmapper/options.go: -------------------------------------------------------------------------------- 1 | package batchmapper 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/mapstreamer/options.go: -------------------------------------------------------------------------------- 1 | package mapstreamer 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/reducestreamer/options.go: -------------------------------------------------------------------------------- 1 | package reducestreamer 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/sessionreducer/options.go: -------------------------------------------------------------------------------- 1 | package sessionreducer 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /examples/sourcetransformer/filter/go.mod: -------------------------------------------------------------------------------- 1 | module filter 2 | 3 | go 1.24.1 4 | 5 | replace github.com/numaproj/numaflow-go => ../../.. 6 | 7 | require ( 8 | github.com/Masterminds/sprig/v3 v3.3.0 9 | github.com/expr-lang/expr v1.17.4 10 | github.com/numaproj/numaflow-go v0.10.1 11 | ) 12 | 13 | require ( 14 | dario.cat/mergo v1.0.1 // indirect 15 | github.com/Masterminds/goutils v1.1.1 // indirect 16 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 17 | github.com/google/uuid v1.6.0 // indirect 18 | github.com/huandu/xstrings v1.5.0 // indirect 19 | github.com/mitchellh/copystructure v1.2.0 // indirect 20 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 21 | github.com/shopspring/decimal v1.4.0 // indirect 22 | github.com/spf13/cast v1.7.0 // indirect 23 | golang.org/x/crypto v0.39.0 // indirect 24 | golang.org/x/net v0.41.0 // indirect 25 | golang.org/x/sync v0.15.0 // indirect 26 | golang.org/x/sys v0.33.0 // indirect 27 | golang.org/x/text v0.26.0 // indirect 28 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 29 | google.golang.org/grpc v1.67.1 // indirect 30 | google.golang.org/protobuf v1.34.2 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /pkg/batchmapper/types.go: -------------------------------------------------------------------------------- 1 | package batchmapper 2 | 3 | import "time" 4 | 5 | // handlerDatum implements the Datum interface and is used in the batch map function. 6 | type handlerDatum struct { 7 | value []byte 8 | eventTime time.Time 9 | watermark time.Time 10 | headers map[string]string 11 | id string 12 | keys []string 13 | } 14 | 15 | func NewHandlerDatum(value []byte, eventTime time.Time, watermark time.Time, headers map[string]string, id string, keys []string) Datum { 16 | return &handlerDatum{ 17 | value: value, 18 | eventTime: eventTime, 19 | watermark: watermark, 20 | headers: headers, 21 | id: id, 22 | keys: keys, 23 | } 24 | } 25 | 26 | func (h *handlerDatum) Value() []byte { 27 | return h.value 28 | } 29 | 30 | func (h *handlerDatum) EventTime() time.Time { 31 | return h.eventTime 32 | } 33 | 34 | func (h *handlerDatum) Watermark() time.Time { 35 | return h.watermark 36 | } 37 | 38 | func (h *handlerDatum) Headers() map[string]string { 39 | return h.headers 40 | } 41 | 42 | func (h *handlerDatum) Id() string { 43 | return h.id 44 | } 45 | 46 | func (h *handlerDatum) Keys() []string { 47 | return h.keys 48 | } 49 | -------------------------------------------------------------------------------- /pkg/sourcetransformer/options.go: -------------------------------------------------------------------------------- 1 | package sourcetransformer 2 | 3 | type options struct { 4 | sockAddr string 5 | maxMessageSize int 6 | serverInfoFilePath string 7 | } 8 | 9 | // Option is the interface to apply options. 10 | type Option func(*options) 11 | 12 | func defaultOptions() *options { 13 | return &options{ 14 | sockAddr: address, 15 | maxMessageSize: defaultMaxMessageSize, 16 | serverInfoFilePath: serverInfoFilePath, 17 | } 18 | } 19 | 20 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 21 | func WithMaxMessageSize(size int) Option { 22 | return func(opts *options) { 23 | opts.maxMessageSize = size 24 | } 25 | } 26 | 27 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purposes. 28 | func WithSockAddr(addr string) Option { 29 | return func(opts *options) { 30 | opts.sockAddr = addr 31 | } 32 | } 33 | 34 | // WithServerInfoFilePath sets the server info file path to the given path. 35 | func WithServerInfoFilePath(f string) Option { 36 | return func(opts *options) { 37 | opts.serverInfoFilePath = f 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /examples/mapper/filter/go.mod: -------------------------------------------------------------------------------- 1 | module filter 2 | 3 | go 1.24.1 4 | 5 | replace github.com/numaproj/numaflow-go => ../../.. 6 | 7 | require github.com/numaproj/numaflow-go v0.10.1 8 | 9 | require ( 10 | dario.cat/mergo v1.0.1 // indirect 11 | github.com/Masterminds/goutils v1.1.1 // indirect 12 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 13 | github.com/google/uuid v1.6.0 // indirect 14 | github.com/huandu/xstrings v1.5.0 // indirect 15 | github.com/mitchellh/copystructure v1.2.0 // indirect 16 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 17 | github.com/shopspring/decimal v1.4.0 // indirect 18 | github.com/spf13/cast v1.7.0 // indirect 19 | golang.org/x/crypto v0.39.0 // indirect 20 | ) 21 | 22 | require ( 23 | github.com/Masterminds/sprig/v3 v3.3.0 24 | github.com/expr-lang/expr v1.17.4 25 | golang.org/x/net v0.41.0 // indirect 26 | golang.org/x/sync v0.15.0 // indirect 27 | golang.org/x/sys v0.33.0 // indirect 28 | golang.org/x/text v0.26.0 // indirect 29 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 30 | google.golang.org/grpc v1.67.1 // indirect 31 | google.golang.org/protobuf v1.34.2 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /pkg/reducestreamer/server_test.go: -------------------------------------------------------------------------------- 1 | package reducestreamer 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strconv" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestReduceServer_Start(t *testing.T) { 14 | socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") 15 | defer func() { 16 | _ = os.RemoveAll(socketFile.Name()) 17 | }() 18 | 19 | serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") 20 | defer func() { 21 | _ = os.RemoveAll(serverInfoFile.Name()) 22 | }() 23 | 24 | var reduceStreamHandle = func(ctx context.Context, keys []string, rch <-chan Datum, och chan<- Message, md Metadata) { 25 | sum := 0 26 | for val := range rch { 27 | msgVal, _ := strconv.Atoi(string(val.Value())) 28 | sum += msgVal 29 | } 30 | och <- NewMessage([]byte(strconv.Itoa(sum))) 31 | } 32 | // note: using actual uds connection 33 | ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) 34 | defer cancel() 35 | err := NewServer(SimpleCreatorWithReduceStreamFn(reduceStreamHandle), WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) 36 | assert.NoError(t, err) 37 | } 38 | -------------------------------------------------------------------------------- /examples/sourcer/simple_source_with_metadata/Dockerfile: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # base 3 | #################################################################################################### 4 | FROM alpine:3.20 AS base 5 | ARG TARGETARCH 6 | RUN apk update && apk upgrade && \ 7 | apk add ca-certificates && \ 8 | apk --no-cache add tzdata 9 | 10 | COPY dist/simple-source-with-metadata-example-${TARGETARCH} /bin/simple-source-with-metadata-example 11 | RUN chmod +x /bin/simple-source-with-metadata-example 12 | 13 | #################################################################################################### 14 | # simple-source-with-metadata 15 | #################################################################################################### 16 | FROM scratch AS simple-source-with-metadata 17 | COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo 18 | COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=base /bin/simple-source-with-metadata-example /bin/simple-source-with-metadata-example 20 | ENTRYPOINT [ "/bin/simple-source-with-metadata-example" ] 21 | -------------------------------------------------------------------------------- /pkg/reducer/server_test.go: -------------------------------------------------------------------------------- 1 | package reducer 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strconv" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestReduceServer_Start(t *testing.T) { 14 | socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") 15 | defer func() { 16 | _ = os.RemoveAll(socketFile.Name()) 17 | }() 18 | 19 | serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") 20 | defer func() { 21 | _ = os.RemoveAll(serverInfoFile.Name()) 22 | }() 23 | 24 | var rfn = func(ctx context.Context, keys []string, rch <-chan Datum, md Metadata) Messages { 25 | sum := 0 26 | for val := range rch { 27 | msgVal, _ := strconv.Atoi(string(val.Value())) 28 | sum += msgVal 29 | } 30 | return MessagesBuilder().Append(NewMessage([]byte(strconv.Itoa(sum))).WithKeys([]string{keys[0] + "_test"})) 31 | } 32 | 33 | // note: using actual uds connection 34 | ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) 35 | defer cancel() 36 | err := NewServer(SimpleCreatorWithReduceFn(rfn), WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) 37 | assert.NoError(t, err) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/batchmapper/server_test.go: -------------------------------------------------------------------------------- 1 | package batchmapper 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestBatchMapServer_Start(t *testing.T) { 13 | socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") 14 | defer func() { 15 | _ = os.RemoveAll(socketFile.Name()) 16 | }() 17 | 18 | serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") 19 | defer func() { 20 | _ = os.RemoveAll(serverInfoFile.Name()) 21 | }() 22 | 23 | var batchMapHandler = BatchMapperFunc(func(ctx context.Context, datums <-chan Datum) BatchResponses { 24 | batchResponses := BatchResponsesBuilder() 25 | for d := range datums { 26 | results := NewBatchResponse(d.Id()) 27 | results.Append(NewMessage(d.Value()).WithKeys([]string{d.Keys()[0] + "_test"})) 28 | batchResponses.Append(results) 29 | } 30 | return batchResponses 31 | }) 32 | // note: using actual uds connection 33 | ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) 34 | defer cancel() 35 | err := NewServer(batchMapHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) 36 | assert.NoError(t, err) 37 | } 38 | -------------------------------------------------------------------------------- /examples/mapper/filter/util/expand.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func Expand(value map[string]any) map[string]any { 8 | return ExpandPrefixed(value, "") 9 | } 10 | 11 | func ExpandPrefixed(value map[string]any, prefix string) map[string]any { 12 | m := make(map[string]any) 13 | ExpandPrefixedToResult(value, prefix, m) 14 | return m 15 | } 16 | 17 | func ExpandPrefixedToResult(value map[string]any, prefix string, result map[string]any) { 18 | if prefix != "" { 19 | prefix += "." 20 | } 21 | for k, val := range value { 22 | if !strings.HasPrefix(k, prefix) { 23 | continue 24 | } 25 | 26 | key := k[len(prefix):] 27 | idx := strings.Index(key, ".") 28 | if idx != -1 { 29 | key = key[:idx] 30 | } 31 | // It is possible for the map to contain conflicts: 32 | // {"a.b": 1, "a": 2} 33 | // What should the result be? We overwrite the less-specific key. 34 | // {"a.b": 1, "a": 2} -> {"a.b": 1, "a": 2} 35 | if _, ok := result[key]; ok && idx == -1 { 36 | continue 37 | } 38 | if idx == -1 { 39 | result[key] = val 40 | continue 41 | } 42 | 43 | // It contains a period, so it is a more complex structure 44 | result[key] = ExpandPrefixed(value, k[:len(prefix)+len(key)]) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /examples/sourcetransformer/filter/util/expand.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func Expand(value map[string]any) map[string]any { 8 | return ExpandPrefixed(value, "") 9 | } 10 | 11 | func ExpandPrefixed(value map[string]any, prefix string) map[string]any { 12 | m := make(map[string]any) 13 | ExpandPrefixedToResult(value, prefix, m) 14 | return m 15 | } 16 | 17 | func ExpandPrefixedToResult(value map[string]any, prefix string, result map[string]any) { 18 | if prefix != "" { 19 | prefix += "." 20 | } 21 | for k, val := range value { 22 | if !strings.HasPrefix(k, prefix) { 23 | continue 24 | } 25 | 26 | key := k[len(prefix):] 27 | idx := strings.Index(key, ".") 28 | if idx != -1 { 29 | key = key[:idx] 30 | } 31 | // It is possible for the map to contain conflicts: 32 | // {"a.b": 1, "a": 2} 33 | // What should the result be? We overwrite the less-specific key. 34 | // {"a.b": 1, "a": 2} -> {"a.b": 1, "a": 2} 35 | if _, ok := result[key]; ok && idx == -1 { 36 | continue 37 | } 38 | if idx == -1 { 39 | result[key] = val 40 | continue 41 | } 42 | 43 | // It contains a period, so it is a more complex structure 44 | result[key] = ExpandPrefixed(value, k[:len(prefix)+len(key)]) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pkg/mapstreamer/interface.go: -------------------------------------------------------------------------------- 1 | package mapstreamer 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // Datum contains methods to get the payload information. 9 | type Datum interface { 10 | // Value returns the payload of the message. 11 | Value() []byte 12 | // EventTime returns the event time of the message. 13 | EventTime() time.Time 14 | // Watermark returns the watermark of the message. 15 | Watermark() time.Time 16 | // Headers returns the headers of the message. 17 | Headers() map[string]string 18 | } 19 | 20 | // MapStreamer is the interface of map stream function implementation. 21 | type MapStreamer interface { 22 | // MapStream is the function to process each coming message and streams 23 | // the result back using a channel. 24 | MapStream(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) 25 | } 26 | 27 | // MapStreamerFunc is a utility type used to convert a function to a MapStreamer. 28 | type MapStreamerFunc func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) 29 | 30 | // MapStream implements the function of map stream function. 31 | func (msf MapStreamerFunc) MapStream(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { 32 | msf(ctx, keys, datum, messageCh) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/sideinput/options.go: -------------------------------------------------------------------------------- 1 | package sideinput 2 | 3 | // options is the struct to hold the server options. 4 | type options struct { 5 | sockAddr string 6 | maxMessageSize int 7 | serverInfoFilePath string 8 | } 9 | 10 | // Option is the interface to apply options. 11 | type Option func(*options) 12 | 13 | // defaultOptions returns the default options. 14 | func defaultOptions() *options { 15 | return &options{ 16 | sockAddr: address, 17 | maxMessageSize: defaultMaxMessageSize, 18 | serverInfoFilePath: serverInfoFilePath, 19 | } 20 | } 21 | 22 | // WithMaxMessageSize sets the server max receive message size and the server max send message size to the given size. 23 | func WithMaxMessageSize(size int) Option { 24 | return func(opts *options) { 25 | opts.maxMessageSize = size 26 | } 27 | } 28 | 29 | // WithSockAddr start the server with the given sock addr. This is mainly used for testing purpose. 30 | func WithSockAddr(addr string) Option { 31 | return func(opts *options) { 32 | opts.sockAddr = addr 33 | } 34 | } 35 | 36 | // WithServerInfoFilePath sets the server info file path to the given path. 37 | func WithServerInfoFilePath(f string) Option { 38 | return func(opts *options) { 39 | opts.serverInfoFilePath = f 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_extractor/go.mod: -------------------------------------------------------------------------------- 1 | module event_time_extractor 2 | 3 | go 1.24.1 4 | 5 | replace github.com/numaproj/numaflow-go => ../../.. 6 | 7 | require ( 8 | github.com/Masterminds/sprig/v3 v3.3.0 9 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de 10 | github.com/expr-lang/expr v1.17.5 11 | github.com/numaproj/numaflow-go v0.10.1 12 | ) 13 | 14 | require ( 15 | dario.cat/mergo v1.0.1 // indirect 16 | github.com/Masterminds/goutils v1.1.1 // indirect 17 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 18 | github.com/google/uuid v1.6.0 // indirect 19 | github.com/huandu/xstrings v1.5.0 // indirect 20 | github.com/mitchellh/copystructure v1.2.0 // indirect 21 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 22 | github.com/shopspring/decimal v1.4.0 // indirect 23 | github.com/spf13/cast v1.7.0 // indirect 24 | golang.org/x/crypto v0.39.0 // indirect 25 | golang.org/x/net v0.41.0 // indirect 26 | golang.org/x/sync v0.15.0 // indirect 27 | golang.org/x/sys v0.33.0 // indirect 28 | golang.org/x/text v0.26.0 // indirect 29 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 30 | google.golang.org/grpc v1.67.1 // indirect 31 | google.golang.org/protobuf v1.34.2 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /examples/sourcetransformer/event_time_extractor/util/expand.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func Expand(value map[string]any) map[string]any { 8 | return ExpandPrefixed(value, "") 9 | } 10 | 11 | func ExpandPrefixed(value map[string]any, prefix string) map[string]any { 12 | m := make(map[string]any) 13 | ExpandPrefixedToResult(value, prefix, m) 14 | return m 15 | } 16 | 17 | func ExpandPrefixedToResult(value map[string]any, prefix string, result map[string]any) { 18 | if prefix != "" { 19 | prefix += "." 20 | } 21 | for k, val := range value { 22 | if !strings.HasPrefix(k, prefix) { 23 | continue 24 | } 25 | 26 | key := k[len(prefix):] 27 | idx := strings.Index(key, ".") 28 | if idx != -1 { 29 | key = key[:idx] 30 | } 31 | // It is possible for the map to contain conflicts: 32 | // {"a.b": 1, "a": 2} 33 | // What should the result be? We overwrite the less-specific key. 34 | // {"a.b": 1, "a": 2} -> {"a.b": 1, "a": 2} 35 | if _, ok := result[key]; ok && idx == -1 { 36 | continue 37 | } 38 | if idx == -1 { 39 | result[key] = val 40 | continue 41 | } 42 | 43 | // It contains a period, so it is a more complex structure 44 | result[key] = ExpandPrefixed(value, k[:len(prefix)+len(key)]) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /examples/sourcetransformer/time_extraction_filter/go.mod: -------------------------------------------------------------------------------- 1 | module timeextractionfilter 2 | 3 | go 1.24.1 4 | 5 | replace github.com/numaproj/numaflow-go => ../../.. 6 | 7 | require ( 8 | github.com/Masterminds/sprig/v3 v3.3.0 9 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de 10 | github.com/expr-lang/expr v1.17.5 11 | github.com/numaproj/numaflow-go v0.10.1 12 | ) 13 | 14 | require ( 15 | dario.cat/mergo v1.0.1 // indirect 16 | github.com/Masterminds/goutils v1.1.1 // indirect 17 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 18 | github.com/google/uuid v1.6.0 // indirect 19 | github.com/huandu/xstrings v1.5.0 // indirect 20 | github.com/mitchellh/copystructure v1.2.0 // indirect 21 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 22 | github.com/shopspring/decimal v1.4.0 // indirect 23 | github.com/spf13/cast v1.7.0 // indirect 24 | golang.org/x/crypto v0.39.0 // indirect 25 | golang.org/x/net v0.41.0 // indirect 26 | golang.org/x/sync v0.15.0 // indirect 27 | golang.org/x/sys v0.33.0 // indirect 28 | golang.org/x/text v0.26.0 // indirect 29 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 30 | google.golang.org/grpc v1.67.1 // indirect 31 | google.golang.org/protobuf v1.34.2 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /examples/sourcetransformer/time_extraction_filter/util/expand.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func Expand(value map[string]any) map[string]any { 8 | return ExpandPrefixed(value, "") 9 | } 10 | 11 | func ExpandPrefixed(value map[string]any, prefix string) map[string]any { 12 | m := make(map[string]any) 13 | ExpandPrefixedToResult(value, prefix, m) 14 | return m 15 | } 16 | 17 | func ExpandPrefixedToResult(value map[string]any, prefix string, result map[string]any) { 18 | if prefix != "" { 19 | prefix += "." 20 | } 21 | for k, val := range value { 22 | if !strings.HasPrefix(k, prefix) { 23 | continue 24 | } 25 | 26 | key := k[len(prefix):] 27 | idx := strings.Index(key, ".") 28 | if idx != -1 { 29 | key = key[:idx] 30 | } 31 | // It is possible for the map to contain conflicts: 32 | // {"a.b": 1, "a": 2} 33 | // What should the result be? We overwrite the less-specific key. 34 | // {"a.b": 1, "a": 2} -> {"a.b": 1, "a": 2} 35 | if _, ok := result[key]; ok && idx == -1 { 36 | continue 37 | } 38 | if idx == -1 { 39 | result[key] = val 40 | continue 41 | } 42 | 43 | // It contains a period, so it is a more complex structure 44 | result[key] = ExpandPrefixed(value, k[:len(prefix)+len(key)]) 45 | } 46 | } 47 | --------------------------------------------------------------------------------