├── config ├── .gitignore ├── rag │ ├── .gitignore │ ├── rpk.profile.yaml │ ├── env.sample │ ├── templates │ │ ├── ollama_embeddings.yaml │ │ ├── openai_embeddings.yaml │ │ ├── pgvector_query.yaml │ │ └── cohere_embeddings.yaml │ ├── ingestion │ │ └── redpanda-docs.yaml │ └── retrieval │ │ ├── ollama_pgvector.yaml │ │ ├── openai_pgvector.yaml │ │ └── cohere_pgvector.yaml ├── examples │ ├── resources │ │ ├── resources.yaml │ │ └── set_grab_cache.yaml │ └── site_analytics.yaml ├── test │ ├── bloblang │ │ ├── cities.blobl │ │ ├── csv_formatter.blobl │ │ ├── github_releases.blobl │ │ ├── cities_test.yaml │ │ ├── also_tests_boolean_operands.yaml │ │ ├── fans.yaml │ │ ├── boolean_operands.yaml │ │ ├── message_expansion.yaml │ │ ├── env.yaml │ │ └── windowed.yaml │ ├── resources │ │ ├── other_mappings.yaml │ │ ├── other_mappings_benthos_test.yaml │ │ └── some_mappings.yaml │ ├── files │ │ ├── input.txt │ │ └── output.txt │ ├── protobuf │ │ └── schema │ │ │ ├── envelope.proto │ │ │ ├── house.proto │ │ │ └── person.proto │ ├── cookbooks │ │ ├── filtering.yaml │ │ └── filtering_benthos_test.yaml │ ├── files_for_content.yaml │ ├── unit_test_example.yaml │ ├── awk.yaml │ ├── awk_benthos_test.yaml │ ├── infile_resource_mock.yaml │ ├── structured_metadata.yaml │ ├── json_contains_predicate.yaml │ ├── unit_test_example_benthos_test.yaml │ ├── filters.yaml │ ├── deduplicate.yaml │ ├── deduplicate_lru.yaml │ ├── deduplicate_ttlru.yaml │ ├── env_var_stuff.yaml │ ├── mock_http_proc_path.yaml │ ├── mock_http_proc.yaml │ └── deduplicate_by_batch.yaml ├── docker.yaml ├── template_examples │ ├── processor_log_message.yaml │ ├── processor_plugin_alias.yaml │ ├── input_sqs_example.yaml │ ├── processor_log_and_drop.yaml │ └── input_stdin_uppercase.yaml └── README.md ├── internal ├── telemetry │ ├── key.pem │ └── logger.go ├── impl │ ├── wasm │ │ ├── .gitignore │ │ └── build.sh │ ├── ollama │ │ ├── .gitignore │ │ └── subprocess.go │ ├── redpanda │ │ ├── .gitignore │ │ └── testdata │ │ │ └── uppercase │ │ │ ├── .gitignore │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ ├── README.md │ │ │ └── transform.go │ ├── snowflake │ │ ├── streaming │ │ │ ├── .gitignore │ │ │ ├── resources │ │ │ │ └── .gitignore │ │ │ ├── README.md │ │ │ └── streaming_test.go │ │ └── resources │ │ │ └── ssh_keys │ │ │ └── README.md │ ├── avro │ │ └── resources │ │ │ └── ocf.avro │ ├── sql │ │ └── resources │ │ │ ├── clickhouse_init.sql │ │ │ ├── docker-compose.yaml │ │ │ └── clickhouse │ │ │ └── clickhouse.xml │ ├── maxmind │ │ └── testdata │ │ │ ├── GeoIP2-City-Test.mmdb │ │ │ ├── GeoIP2-ISP-Test.mmdb │ │ │ ├── GeoIP2-Country-Test.mmdb │ │ │ ├── GeoIP2-Domain-Test.mmdb │ │ │ ├── GeoLite2-ASN-Test.mmdb │ │ │ ├── GeoIP2-Anonymous-IP-Test.mmdb │ │ │ ├── GeoIP2-Enterprise-Test.mmdb │ │ │ └── GeoIP2-Connection-Type-Test.mmdb │ ├── aws │ │ └── resources │ │ │ ├── aws_mk_test_queue │ │ │ ├── aws_mk_test_stream │ │ │ └── docker-compose.yaml │ ├── timeplus │ │ ├── interface.go │ │ └── http │ │ │ └── header.go │ ├── couchbase │ │ └── testdata │ │ │ └── configure-server.sh │ ├── postgresql │ │ └── pglogicalstream │ │ │ └── types.go │ ├── azure │ │ └── package.go │ ├── mqtt │ │ └── package.go │ ├── sftp │ │ └── package.go │ ├── amqp1 │ │ └── input_description.adoc │ ├── nsq │ │ └── docker-compose.yaml │ ├── mongodb │ │ └── cdc │ │ │ └── bson_util.go │ ├── sentry │ │ └── client.go │ ├── openai │ │ └── client.go │ └── twitter │ │ └── init.go ├── agent │ ├── template │ │ ├── .python-version │ │ ├── pyproject.toml │ │ ├── redpanda_agents.yaml │ │ ├── mcp │ │ │ └── resources │ │ │ │ └── processors │ │ │ │ └── check_weather_tool.yaml │ │ └── README.md │ └── template.go ├── rpcplugin │ ├── golangtemplate │ │ ├── input │ │ │ ├── go.mod.tmpl │ │ │ └── plugin.yaml │ │ ├── output │ │ │ ├── go.mod.tmpl │ │ │ ├── plugin.yaml │ │ │ └── main.go │ │ └── processor │ │ │ ├── go.mod.tmpl │ │ │ ├── plugin.yaml │ │ │ └── main.go │ ├── pythontemplate │ │ ├── input │ │ │ ├── pyproject.toml │ │ │ ├── main.py │ │ │ └── plugin.yaml │ │ ├── output │ │ │ ├── pyproject.toml │ │ │ ├── main.py │ │ │ └── plugin.yaml │ │ └── processor │ │ │ ├── pyproject.toml │ │ │ ├── main.py │ │ │ └── plugin.yaml │ ├── subprocess │ │ ├── signal.go │ │ └── signal_unix.go │ ├── protogen.go │ └── util.go ├── license │ └── public_key.pem ├── cli │ └── chroot_others.go ├── protoconnect │ └── package.go ├── asyncroutine │ └── doc.go └── plugins │ └── info_test.go ├── .versions ├── public ├── bundle │ ├── .gitignore │ ├── enterprise │ │ └── package.go │ └── free │ │ └── package.go ├── plugin │ └── python │ │ ├── .python-version │ │ ├── src │ │ └── redpanda_connect │ │ │ ├── py.typed │ │ │ └── _proto │ │ │ └── redpanda │ │ │ └── runtime │ │ │ └── v1alpha1 │ │ │ ├── message_pb2_grpc.py │ │ │ └── message_pb2_grpc.pyi │ │ ├── connect.yaml │ │ ├── examples │ │ ├── fizzbuzz_processor.yaml │ │ ├── logging_output.yaml │ │ └── json_input.yaml │ │ └── README.md └── components │ ├── timeplus │ └── package.go │ ├── ollama │ └── package.go │ ├── splunk │ └── package.go │ ├── cohere │ └── package.go │ ├── openai │ └── package.go │ ├── snowflake │ └── package.go │ ├── google │ └── package.go │ ├── mysql │ └── package.go │ ├── slack │ └── package.go │ ├── aws │ ├── enterprise │ │ └── package.go │ └── package.go │ ├── gcp │ ├── enterprise │ │ └── package.go │ └── package.go │ ├── kafka │ ├── enterprise │ │ └── package.go │ └── package.go │ ├── mongodb │ ├── enterprise │ │ └── package.go │ └── package.go │ ├── postgresql │ └── package.go │ ├── zeromq │ ├── package.go │ └── x_benthos_extra.go │ ├── pulsar │ ├── arm_32.go │ └── package.go │ ├── ockam │ ├── windows.go │ └── package.go │ ├── couchbase │ ├── package_32bit.go │ └── package.go │ ├── git │ └── package.go │ ├── nsq │ └── package.go │ ├── sql │ ├── snowflake.go │ ├── base │ │ └── package.go │ └── sqlite.go │ ├── avro │ └── package.go │ ├── hdfs │ └── package.go │ ├── mqtt │ └── package.go │ ├── nats │ └── package.go │ ├── otlp │ └── package.go │ ├── sftp │ └── package.go │ ├── text │ └── package.go │ ├── wasm │ └── package.go │ ├── amqp09 │ └── package.go │ ├── amqp1 │ └── package.go │ ├── azure │ └── package.go │ ├── crypto │ └── package.go │ ├── cypher │ └── package.go │ ├── dgraph │ └── package.go │ ├── jaeger │ └── package.go │ ├── pusher │ └── package.go │ ├── qdrant │ └── package.go │ ├── redis │ └── package.go │ ├── sentry │ └── package.go │ ├── statsd │ └── package.go │ ├── discord │ └── package.go │ ├── gateway │ └── package.go │ ├── influxdb │ └── package.go │ ├── maxmind │ └── package.go │ ├── msgpack │ └── package.go │ ├── nanomsg │ └── package.go │ ├── pinecone │ └── package.go │ ├── questdb │ └── package.go │ ├── redpanda │ └── package.go │ ├── spicedb │ └── package.go │ ├── twitter │ └── package.go │ ├── cassandra │ └── package.go │ ├── changelog │ └── package.go │ ├── confluent │ └── package.go │ ├── javascript │ └── package.go │ ├── memcached │ └── package.go │ ├── beanstalkd │ └── package.go │ ├── cockroachdb │ └── package.go │ ├── opensearch │ └── package.go │ ├── prometheus │ └── package.go │ └── elasticsearch │ └── v8 │ └── package.go ├── tools └── spanner │ ├── benchmark │ ├── .gitignore │ ├── benchmark.yml │ └── config.tmpl.yml │ ├── README.md │ ├── terraform │ ├── terraform.yml │ ├── outputs.tf │ ├── .gitignore │ └── main.tf │ └── Taskfile.yml ├── resources ├── plugin_uploader │ ├── test_data │ │ └── dist │ │ │ ├── cow_darwin_arm64 │ │ │ └── redpanda-cow │ │ │ ├── cow_linux_amd64_v1 │ │ │ └── redpanda-cow │ │ │ ├── metadata_v4_34_0.json │ │ │ ├── metadata_v4_35_0.json │ │ │ ├── metadata_v4_36_0_rc1.json │ │ │ └── artifacts.json │ ├── requirements.txt │ └── requirements_test.txt ├── docker │ ├── profiling │ │ ├── grafana │ │ │ ├── config.monitoring │ │ │ └── provisioning │ │ │ │ ├── dashboards │ │ │ │ └── dashboard.yml │ │ │ │ └── datasources │ │ │ │ └── datasource.yml │ │ ├── prometheus │ │ │ └── prometheus.yml │ │ ├── config.yaml │ │ ├── README.md │ │ └── docker-compose.yaml │ ├── redpanda_benchmarking │ │ ├── grafana │ │ │ ├── config.monitoring │ │ │ └── provisioning │ │ │ │ ├── dashboards │ │ │ │ └── dashboard.yml │ │ │ │ └── datasources │ │ │ │ └── datasource.yml │ │ ├── out_stdout.yaml │ │ ├── README.md │ │ ├── generate.yaml │ │ ├── out_bridge.yaml │ │ ├── prometheus │ │ │ └── prometheus.yml │ │ └── out_order_verify.yaml │ ├── schema_registry │ │ ├── insert_schema.sh │ │ ├── blob_schema.json │ │ ├── out.yaml │ │ ├── in.yaml │ │ ├── docker-compose.yaml │ │ └── README.md │ └── README.md └── scripts │ ├── third_party.md.tpl │ ├── fips_patchelf.sh │ ├── update_bundles.sh │ ├── sign_for_darwin.sh │ ├── third_party_licenses.sh │ ├── fips_wrapper.sh │ ├── release_notes.sh │ ├── tag_bundles.sh │ └── add_license_headers.sh ├── docs ├── antora.yml └── modules │ └── components │ └── pages │ ├── tracers │ └── none.adoc │ ├── metrics │ ├── none.adoc │ └── json_api.adoc │ ├── outputs │ └── drop.adoc │ ├── scanners │ ├── json_documents.adoc │ ├── tar.adoc │ ├── chunker.adoc │ ├── skip_bom.adoc │ ├── to_the_end.adoc │ └── decompress.adoc │ ├── processors │ ├── noop.adoc │ ├── crash.adoc │ ├── rate_limit.adoc │ └── decompress.adoc │ ├── caches │ ├── noop.adoc │ └── file.adoc │ └── inputs │ └── beanstalkd.adoc ├── .dockerignore ├── .gitignore ├── README-FIPS.md ├── licenses ├── rcl_header.go.txt ├── README.md └── Apache-2.0_header.go.txt ├── .github ├── actions │ └── setup-task │ │ └── action.yml ├── dependabot.yaml └── workflows │ ├── govulncheck.yml │ └── release_python_sdk.yaml ├── taskfiles └── tools.yml └── cmd ├── serverless └── connect-lambda │ └── main.go ├── redpanda-connect-ai └── sqlite.go └── redpanda-connect-cloud └── sqlite.go /config/.gitignore: -------------------------------------------------------------------------------- 1 | dev.yaml -------------------------------------------------------------------------------- /internal/telemetry/key.pem: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.versions: -------------------------------------------------------------------------------- 1 | GOLANGCI_LINT_VERSION=2.1.2 -------------------------------------------------------------------------------- /internal/impl/wasm/.gitignore: -------------------------------------------------------------------------------- 1 | *.wasm -------------------------------------------------------------------------------- /public/bundle/.gitignore: -------------------------------------------------------------------------------- 1 | go.sum 2 | -------------------------------------------------------------------------------- /config/rag/.gitignore: -------------------------------------------------------------------------------- 1 | env 2 | results 3 | -------------------------------------------------------------------------------- /internal/impl/ollama/.gitignore: -------------------------------------------------------------------------------- 1 | ollama 2 | -------------------------------------------------------------------------------- /internal/impl/redpanda/.gitignore: -------------------------------------------------------------------------------- 1 | *.wasm -------------------------------------------------------------------------------- /internal/agent/template/.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /public/plugin/python/.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /tools/spanner/benchmark/.gitignore: -------------------------------------------------------------------------------- 1 | config.yml -------------------------------------------------------------------------------- /public/plugin/python/src/redpanda_connect/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /internal/impl/snowflake/streaming/.gitignore: -------------------------------------------------------------------------------- 1 | *.parquet 2 | -------------------------------------------------------------------------------- /internal/impl/redpanda/testdata/uppercase/.gitignore: -------------------------------------------------------------------------------- 1 | *.wasm 2 | -------------------------------------------------------------------------------- /internal/impl/snowflake/streaming/resources/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /resources/plugin_uploader/test_data/dist/cow_darwin_arm64/redpanda-cow: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/plugin_uploader/test_data/dist/cow_linux_amd64_v1/redpanda-cow: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/antora.yml: -------------------------------------------------------------------------------- 1 | name: redpanda-connect 2 | title: Redpanda Connect 3 | version: ~ -------------------------------------------------------------------------------- /resources/plugin_uploader/requirements.txt: -------------------------------------------------------------------------------- 1 | pydantic>=2.8 2 | boto3>=1.26 3 | click==8.1.7 -------------------------------------------------------------------------------- /config/examples/resources/resources.yaml: -------------------------------------------------------------------------------- 1 | cache_resources: 2 | - label: foocache 3 | memory: {} -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/input/go.mod.tmpl: -------------------------------------------------------------------------------- 1 | module PROJECT_NAME_HERE 2 | 3 | go GO_VERSION 4 | -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/output/go.mod.tmpl: -------------------------------------------------------------------------------- 1 | module PROJECT_NAME_HERE 2 | 3 | go GO_VERSION 4 | -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/processor/go.mod.tmpl: -------------------------------------------------------------------------------- 1 | module PROJECT_NAME_HERE 2 | 3 | go GO_VERSION 4 | -------------------------------------------------------------------------------- /resources/docker/profiling/grafana/config.monitoring: -------------------------------------------------------------------------------- 1 | GF_SECURITY_ADMIN_PASSWORD=admin 2 | GF_USERS_ALLOW_SIGN_UP=false 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | resources 2 | icon.png 3 | LICENSE 4 | README.md 5 | target/bin 6 | target/dist 7 | public/plugin/python/.venv 8 | -------------------------------------------------------------------------------- /internal/impl/avro/resources/ocf.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/avro/resources/ocf.avro -------------------------------------------------------------------------------- /resources/plugin_uploader/requirements_test.txt: -------------------------------------------------------------------------------- 1 | pydantic>=2.8 2 | boto3>=1.26 3 | click==8.1.7 4 | moto[s3]==5.0.13 5 | pytest==8.3.2 -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/grafana/config.monitoring: -------------------------------------------------------------------------------- 1 | GF_SECURITY_ADMIN_PASSWORD=admin 2 | GF_USERS_ALLOW_SIGN_UP=false 3 | -------------------------------------------------------------------------------- /internal/impl/sql/resources/clickhouse_init.sql: -------------------------------------------------------------------------------- 1 | create table test ( 2 | foo String, 3 | bar Int64, 4 | baz String 5 | ) engine=Memory 6 | -------------------------------------------------------------------------------- /internal/impl/wasm/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | tinygo build -scheduler=none -target=wasi -o uppercase.wasm ../../../public/wasm/examples/tinygo 3 | -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoIP2-City-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoIP2-City-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoIP2-ISP-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoIP2-ISP-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/aws/resources/aws_mk_test_queue: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | aws sqs create-queue --endpoint http://localhost:4566 --region eu-west-1 --queue-name benthostestqueue -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoIP2-Country-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoIP2-Country-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoIP2-Domain-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoIP2-Domain-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoLite2-ASN-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoLite2-ASN-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoIP2-Anonymous-IP-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoIP2-Anonymous-IP-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoIP2-Enterprise-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoIP2-Enterprise-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/redpanda/testdata/uppercase/go.mod: -------------------------------------------------------------------------------- 1 | module uppercase 2 | 3 | go 1.22 4 | 5 | require github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.0.2 6 | -------------------------------------------------------------------------------- /resources/scripts/third_party.md.tpl: -------------------------------------------------------------------------------- 1 | # Licenses 2 | 3 | | Software | License | 4 | | :------- | :------ | 5 | {{ range . }}| {{ .Name }} | {{ .LicenseName }} | 6 | {{ end }} 7 | 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | target 3 | vendor 4 | tmp 5 | site 6 | .tags 7 | .DS_Store 8 | TODO.md 9 | release_notes.md 10 | .idea 11 | .task 12 | .vscode 13 | .op 14 | __pycache__ 15 | -------------------------------------------------------------------------------- /internal/impl/maxmind/testdata/GeoIP2-Connection-Type-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephwoodward/connect/main/internal/impl/maxmind/testdata/GeoIP2-Connection-Type-Test.mmdb -------------------------------------------------------------------------------- /internal/impl/aws/resources/aws_mk_test_stream: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | aws kinesis create-stream --endpoint http://localhost:4566 --region eu-west-1 --stream-name BenthosTestStream --shard-count 4 -------------------------------------------------------------------------------- /public/plugin/python/connect.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | json_gen: 3 | count: 3 4 | 5 | # pipeline: 6 | # processors: 7 | # - fizzbuzz: {} 8 | 9 | # output: 10 | # py_log: {} 11 | -------------------------------------------------------------------------------- /resources/scripts/fips_patchelf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | test -z "$1" && echo "usage: $0 " && exit 4 | 5 | patchelf --set-interpreter "${PREFIX:=/opt/redpanda/rpk-fips}/lib/ld.so" $1 -------------------------------------------------------------------------------- /config/test/bloblang/cities.blobl: -------------------------------------------------------------------------------- 1 | root.Cities = this.locations. 2 | filter(loc -> loc.state == "WA"). 3 | map_each(loc -> loc.name). 4 | sort().join(", ") -------------------------------------------------------------------------------- /public/components/timeplus/package.go: -------------------------------------------------------------------------------- 1 | package timeplus 2 | 3 | import ( 4 | // Bring in the internal plugin definitions. 5 | _ "github.com/redpanda-data/connect/v4/internal/impl/timeplus" 6 | ) 7 | -------------------------------------------------------------------------------- /config/test/resources/other_mappings.yaml: -------------------------------------------------------------------------------- 1 | processor_resources: 2 | - label: prefix 3 | bloblang: 'root = "bar " + content()' 4 | 5 | - label: upper 6 | bloblang: 'root = content().uppercase()' 7 | -------------------------------------------------------------------------------- /config/test/files/input.txt: -------------------------------------------------------------------------------- 1 | hello world 2 | 3 | this file 4 | 5 | is a test input 6 | 7 | and it lives in a file because 8 | 9 | it's very large and would 10 | 11 | look ugly if it were inline in the test 12 | -------------------------------------------------------------------------------- /config/test/files/output.txt: -------------------------------------------------------------------------------- 1 | HELLO WORLD 2 | 3 | THIS FILE 4 | 5 | IS A TEST INPUT 6 | 7 | AND IT LIVES IN A FILE BECAUSE 8 | 9 | IT'S VERY LARGE AND WOULD 10 | 11 | LOOK UGLY IF IT WERE INLINE IN THE TEST 12 | -------------------------------------------------------------------------------- /public/plugin/python/examples/fizzbuzz_processor.yaml: -------------------------------------------------------------------------------- 1 | name: fizzbuzz 2 | summary: Your favorite interview question - as a plugin! 3 | command: ["uv", "run", "examples/fizzbuzz_processor.py"] 4 | type: processor 5 | fields: [] 6 | -------------------------------------------------------------------------------- /config/test/protobuf/schema/envelope.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package testing; 3 | 4 | import "google/protobuf/any.proto"; 5 | import "google/protobuf/timestamp.proto"; 6 | 7 | message Envelope { 8 | int32 id = 1; 9 | google.protobuf.Any content = 2; 10 | } -------------------------------------------------------------------------------- /internal/impl/sql/resources/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | clickhouse: 5 | image: clickhouse/clickhouse-server 6 | volumes: 7 | - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql 8 | ports: 9 | - 9000:9000 10 | -------------------------------------------------------------------------------- /config/test/bloblang/csv_formatter.blobl: -------------------------------------------------------------------------------- 1 | let header_row = this.0.keys().sort().join(",") 2 | 3 | root = $header_row + "\n" + this.map_each(element -> element.key_values(). 4 | sort_by(item -> item.key). 5 | map_each(item -> item.value.string()). 6 | join(",") 7 | ).join("\n") 8 | -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/input/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "PROJECT_NAME_HERE" 3 | version = "0.1.0" 4 | description = "Add your description here" 5 | readme = "README.md" 6 | requires-python = ">=3.12" 7 | dependencies = [ 8 | "redpanda-connect", 9 | ] 10 | -------------------------------------------------------------------------------- /config/test/cookbooks/filtering.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | root = match { 5 | meta("topic").or("") == "foo" || 6 | doc.type.or("") == "bar" || 7 | doc.urls.contains("https://www.benthos.dev/").catch(false) => deleted() 8 | } -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/output/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "PROJECT_NAME_HERE" 3 | version = "0.1.0" 4 | description = "Add your description here" 5 | readme = "README.md" 6 | requires-python = ">=3.12" 7 | dependencies = [ 8 | "redpanda-connect", 9 | ] 10 | -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/processor/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "PROJECT_NAME_HERE" 3 | version = "0.1.0" 4 | description = "Add your description here" 5 | readme = "README.md" 6 | requires-python = ">=3.12" 7 | dependencies = [ 8 | "redpanda-connect", 9 | ] 10 | -------------------------------------------------------------------------------- /resources/docker/schema_registry/insert_schema.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | curl -s \ 3 | -X POST "http://localhost:8081/subjects/benthos_example/versions" \ 4 | -H "Content-Type: application/vnd.schemaregistry.v1+json" \ 5 | -d "$(cat blob_schema.json | jq '{schema: . | tostring}')" \ 6 | | jq 7 | -------------------------------------------------------------------------------- /internal/impl/redpanda/testdata/uppercase/go.sum: -------------------------------------------------------------------------------- 1 | github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.0.2 h1:34F42buBTGuK1uaXKky1PdxAZzqMh6kQE1ojCLf/hWw= 2 | github.com/redpanda-data/redpanda/src/transform-sdk/go/transform v1.0.2/go.mod h1:QGgiwwf/BIsD1b7EiyQ/Apzw+RLSpasRDdpOCiefQFQ= 3 | -------------------------------------------------------------------------------- /public/plugin/python/examples/logging_output.yaml: -------------------------------------------------------------------------------- 1 | name: py_log 2 | summary: Just log it 3 | command: ["uv", "run", "examples/logging_output.py"] 4 | type: output 5 | fields: 6 | - name: repeat 7 | type: int 8 | description: the number of times to repeat the log 9 | default: 1 10 | -------------------------------------------------------------------------------- /resources/docker/profiling/grafana/provisioning/dashboards/dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'Prometheus' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | options: 11 | path: /etc/grafana/provisioning/dashboards 12 | -------------------------------------------------------------------------------- /config/test/files_for_content.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = content().uppercase()' 4 | 5 | tests: 6 | - name: should be uppercased 7 | input_batch: 8 | - file_content: ./files/input.txt 9 | output_batches: 10 | - - file_equals: ./files/output.txt 11 | -------------------------------------------------------------------------------- /README-FIPS.md: -------------------------------------------------------------------------------- 1 | # README (FIPS tar.gz archive) 2 | 3 | This tar contains a redpanda-connect-fips binary intended for 4 | automated installation by `rpk`. You probably want to install 5 | the `redpanda-connect-fips` RPM or debian package instead, if 6 | you want to actually use this software on a FIPS-enabled system. 7 | 8 | -------------------------------------------------------------------------------- /config/rag/rpk.profile.yaml: -------------------------------------------------------------------------------- 1 | name: docker-compose 2 | description: "" 3 | prompt: "" 4 | from_cloud: false 5 | kafka_api: 6 | brokers: 7 | - localhost:19092 8 | admin_api: 9 | addresses: 10 | - localhost:19644 11 | schema_registry: 12 | addresses: 13 | - localhost:18081 14 | 15 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/grafana/provisioning/dashboards/dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'Prometheus' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | options: 11 | path: /etc/grafana/provisioning/dashboards 12 | -------------------------------------------------------------------------------- /tools/spanner/README.md: -------------------------------------------------------------------------------- 1 | # GCP Spanner 2 | 3 | Manage a Spanner instance for integration tests. 4 | 5 | ## Running tests 6 | 7 | Procedure: 8 | 9 | * Run `task terraform:create` to create the resources. 10 | * Run `task test` to run the integration tests. 11 | * Run `task terraform:destroy` to destroy the resources. 12 | -------------------------------------------------------------------------------- /config/test/protobuf/schema/house.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package testing; 3 | 4 | import "person.proto"; 5 | 6 | message House { 7 | message Mailbox { 8 | string color = 1; 9 | string identifier = 2; 10 | } 11 | repeated testing.Person people = 1; 12 | string address = 2; 13 | Mailbox mailbox = 3; 14 | } 15 | -------------------------------------------------------------------------------- /internal/impl/aws/resources/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | localstack: 5 | image: localstack/localstack 6 | environment: 7 | DEBUG: 1 8 | LOCALSTACK_HOST: localhost:4566 9 | ports: 10 | - "4566:4566" 11 | # volumes: 12 | # - "/var/run/docker.sock:/var/run/docker.sock" 13 | -------------------------------------------------------------------------------- /internal/impl/sql/resources/clickhouse/clickhouse.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 1 5 | 2 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /tools/spanner/terraform/terraform.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | tasks: 4 | create: 5 | desc: Initialize and apply Terraform configuration 6 | cmds: 7 | - terraform init 8 | - terraform apply -auto-approve 9 | 10 | destroy: 11 | desc: Destroy Terraform infrastructure 12 | cmds: 13 | - terraform destroy -auto-approve 14 | -------------------------------------------------------------------------------- /config/docker.yaml: -------------------------------------------------------------------------------- 1 | # This is the default configuration file shipped with docker builds. It's 2 | # extremely unlikely that a user would want to run Benthos without a custom 3 | # configuration, so the purpose of this file is mostly to be a placeholder. 4 | http: 5 | enabled: true 6 | 7 | metrics: 8 | prometheus: {} 9 | 10 | logger: 11 | format: json 12 | -------------------------------------------------------------------------------- /public/plugin/python/examples/json_input.yaml: -------------------------------------------------------------------------------- 1 | name: json_gen 2 | summary: Just generate some JSON 3 | # switch to the `examples/json_input.py` to see the simple example run instead 4 | command: ["uv", "run", "examples/batch_json_input.py"] 5 | type: input 6 | fields: 7 | - name: count 8 | type: int 9 | description: number of messages to generate 10 | -------------------------------------------------------------------------------- /resources/docker/schema_registry/blob_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "BenthosExample", 4 | "fields": [ 5 | { "name": "ID", "type": "string" }, 6 | { "name": "Name", "type": "string" }, 7 | { "name": "Gooeyness", "type": "double", "default": 0 }, 8 | { "name": "Bouncing", "type": "boolean", "default": true } 9 | ] 10 | } -------------------------------------------------------------------------------- /licenses/rcl_header.go.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | -------------------------------------------------------------------------------- /resources/plugin_uploader/test_data/dist/metadata_v4_34_0.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "cow", 3 | "tag": "v4.34.0", 4 | "previous_tag": "v4.33.0-rc2", 5 | "version": "4.34.0", 6 | "commit": "7eb28f2a994e277f17bf0530097d99208e65cddb", 7 | "date": "2024-08-29T23:53:58.388135715Z", 8 | "runtime": { 9 | "goos": "linux", 10 | "goarch": "arm64" 11 | } 12 | } -------------------------------------------------------------------------------- /resources/plugin_uploader/test_data/dist/metadata_v4_35_0.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "cow", 3 | "tag": "v4.35.0", 4 | "previous_tag": "v4.34.0-rc2", 5 | "version": "4.35.0", 6 | "commit": "7eb28f2a994e277f17bf0530097d99208e65cddb", 7 | "date": "2024-08-29T23:53:58.388135715Z", 8 | "runtime": { 9 | "goos": "linux", 10 | "goarch": "arm64" 11 | } 12 | } -------------------------------------------------------------------------------- /config/test/unit_test_example.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | kafka: 3 | addresses: [ TODO ] 4 | topics: [ foo, bar ] 5 | consumer_group: foogroup 6 | 7 | pipeline: 8 | processors: 9 | - mapping: 'root = "%vend".format(content().uppercase().string())' 10 | 11 | output: 12 | aws_s3: 13 | bucket: TODO 14 | path: '${! meta("kafka_topic") }/${! json("message.id") }.json' -------------------------------------------------------------------------------- /internal/agent/template/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "REDPANDA_PROJECT_NAME" 3 | version = "0.1.0" 4 | description = "Add your description here" 5 | readme = "README.md" 6 | requires-python = ">=3.13" 7 | dependencies = [ 8 | "redpanda-agents", 9 | ] 10 | 11 | [tool.uv.sources] 12 | redpanda-agents = { git = "http://github.com/redpanda-data/agent.git", branch = "main" } 13 | -------------------------------------------------------------------------------- /resources/plugin_uploader/test_data/dist/metadata_v4_36_0_rc1.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "cow", 3 | "tag": "v4.36.0-rc1", 4 | "previous_tag": "v4.34.0-rc2", 5 | "version": "4.36.0-rc1", 6 | "commit": "7eb28f2a994e277f17bf0530097d99208e65cddb", 7 | "date": "2024-08-29T23:53:58.388135715Z", 8 | "runtime": { 9 | "goos": "linux", 10 | "goarch": "arm64" 11 | } 12 | } -------------------------------------------------------------------------------- /config/test/bloblang/github_releases.blobl: -------------------------------------------------------------------------------- 1 | root = this.map_each(release -> release.assets.map_each(asset -> { 2 | "source": "github", 3 | "dist": asset.name.re_replace_all("^benthos-?((lambda_)|_)[0-9\\.]+(-rc[0-9]+)?_([^\\.]+).*", "$2$4"), 4 | "download_count": asset.download_count, 5 | "version": release.tag_name.trim("v"), 6 | }).filter(asset -> asset.dist != "checksums")).flatten() -------------------------------------------------------------------------------- /resources/docker/profiling/grafana/provisioning/datasources/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | # list of datasources that should be deleted from the database 4 | deleteDatasources: 5 | - name: Prometheus 6 | orgId: 1 7 | 8 | datasources: 9 | - name: Prometheus 10 | type: prometheus 11 | access: proxy 12 | orgId: 1 13 | url: http://prometheus:9090 14 | version: 1 15 | editable: true 16 | -------------------------------------------------------------------------------- /resources/scripts/update_bundles.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script should be run from the root of the repository. 4 | # 5 | # Iterates each bundle we provide for Redpanda Connect plugins (enterprise, 6 | # community, etc) and upgrades all dependencies (go get -u). 7 | 8 | for dir in $(ls ./public/bundle); do 9 | ( cd "./public/bundle/$dir" && go get -u . && go mod tidy ) 10 | done 11 | 12 | -------------------------------------------------------------------------------- /internal/agent/template/redpanda_agents.yaml: -------------------------------------------------------------------------------- 1 | agents: 2 | # The key here determines where the agent entrypoint is found: "agents/weather.py" 3 | weather: 4 | # Define how your agent recieves input 5 | input: 6 | stdin: {} 7 | # Define the tools your agent has access too 8 | tools: 9 | - check_weather 10 | # Define where the agent's output goes 11 | output: 12 | stdout: {} 13 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/grafana/provisioning/datasources/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | # list of datasources that should be deleted from the database 4 | deleteDatasources: 5 | - name: Prometheus 6 | orgId: 1 7 | 8 | datasources: 9 | - name: Prometheus 10 | type: prometheus 11 | access: proxy 12 | orgId: 1 13 | url: http://prometheus:9090 14 | version: 1 15 | editable: true 16 | -------------------------------------------------------------------------------- /config/test/awk.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - awk: 4 | codec: text 5 | program: | 6 | { 7 | json_set_int("result", json_get("result") + metadata_get("foo") + metadata_get("bar")); 8 | } 9 | 10 | # This will be ignored during test execution 11 | output_resources: 12 | - label: foo 13 | kafka: 14 | addresses: [ example.com:1234 ] 15 | topic: foo 16 | -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/processor/main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from redpanda_connect import processor, processor_main, Message 4 | 5 | @processor 6 | def my_processor(msg: Message) -> Message: 7 | logging.info(f"Processing message: {msg}") 8 | return msg 9 | 10 | if __name__ == "__main__": 11 | logging.basicConfig(level=logging.INFO) 12 | asyncio.run(processor_main(my_processor)) 13 | -------------------------------------------------------------------------------- /.github/actions/setup-task/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Setup Task' 2 | description: 'Install Task' 3 | 4 | runs: 5 | using: "composite" 6 | steps: 7 | - name: Install Task 8 | shell: bash 9 | run: | 10 | sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin 11 | echo "$HOME/.local/bin" >> $GITHUB_PATH 12 | echo "Installed Task version: $(~/.local/bin/task --version)" 13 | -------------------------------------------------------------------------------- /config/test/awk_benthos_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: Example test case 1 3 | environment: {} 4 | target_processors: /pipeline/processors 5 | input_batch: 6 | - content: '{"result":10}' 7 | metadata: 8 | foo: "5" 9 | bar: "7" 10 | output_batches: 11 | - - content_equals: '{"result":22}' 12 | metadata_equals: 13 | foo: "5" 14 | bar: "7" -------------------------------------------------------------------------------- /config/rag/env.sample: -------------------------------------------------------------------------------- 1 | # These environment variables are assuming the local docker compose setup is running 2 | REDPANDA_BROKERS="localhost:19092" 3 | REDPANDA_USER="" 4 | REDPANDA_PASS="" 5 | SASL_MECHANISM="" 6 | INDEXING_CONSUMER="rp.ai.rag.indexing.v1" 7 | POSTGRES_DSN="postgresql://myuser:mypassword@localhost:5432/mydatabase?sslmode=disable" 8 | POSTGRES_TABLE="rp_ai_rag" 9 | OPENAI_API_KEY='' 10 | COHERE_API_KEY='' 11 | GCP_PROJECT='' 12 | -------------------------------------------------------------------------------- /licenses/README.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | There are 2 licenses for Redpanda Connect. Apache-2.0 covers the majority of connectors and functionality, and RCL (Redpanda Community License) 4 | which covers enterprise features. 5 | 6 | 1. [Apache-2.0](Apache-2.0.txt): Covers the majority of connectors and functionality. 7 | 8 | 2. [RCL](rcl.md): Redpanda Community License - is intended to allow you to use enterprise features 9 | that you pay for. 10 | -------------------------------------------------------------------------------- /tools/spanner/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "database_connection_string" { 2 | description = "Connection string for the Spanner database" 3 | value = "projects/${var.project_id}/instances/${google_spanner_instance.main.name}/databases/${google_spanner_database.database.name}" 4 | } 5 | 6 | output "instance_state" { 7 | description = "The current state of the Spanner instance" 8 | value = google_spanner_instance.main.state 9 | } 10 | -------------------------------------------------------------------------------- /internal/agent/template/mcp/resources/processors/check_weather_tool.yaml: -------------------------------------------------------------------------------- 1 | label: 'check_weather' 2 | processors: 3 | - http: 4 | verb: GET 5 | url: 'https://wttr.in/${!content().string()}?T' 6 | headers: 7 | User-Agent: curl/8.11.1 # Returns a text string from the weather website 8 | 9 | meta: 10 | mcp: 11 | enabled: true 12 | description: 'A tool that can tell you what the weather is in a city passed as the value' 13 | -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/input/main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from redpanda_connect import input, input_main, Value, Message 4 | 5 | @input 6 | async def my_input(config: Value): 7 | _ = config 8 | yield Message(payload="Hello") 9 | yield Message(payload="World") 10 | yield Message(payload="!") 11 | 12 | if __name__ == "__main__": 13 | logging.basicConfig(level=logging.INFO) 14 | asyncio.run(input_main(my_input)) 15 | -------------------------------------------------------------------------------- /resources/docker/profiling/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | external_labels: 5 | monitor: 'rpcn-benchmark' 6 | 7 | scrape_configs: 8 | - job_name: 'prometheus' 9 | scrape_interval: 5s 10 | static_configs: 11 | - targets: ['localhost:9090'] 12 | 13 | - job_name: 'rpcn' 14 | scrape_interval: 5s 15 | static_configs: 16 | - targets: ['host.docker.internal:4195'] 17 | 18 | -------------------------------------------------------------------------------- /config/template_examples/processor_log_message.yaml: -------------------------------------------------------------------------------- 1 | name: log_message 2 | type: processor 3 | summary: Print a log line that shows the contents of a message. 4 | 5 | fields: 6 | - name: level 7 | description: The level to log at. 8 | type: string 9 | default: INFO 10 | 11 | mapping: | 12 | root.log.level = this.level 13 | root.log.message = "${! content() }" 14 | root.log.fields.metadata = "${! meta() }" 15 | root.log.fields.error = "${! error() }" 16 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | groups: 8 | production-dependencies: 9 | dependency-type: "production" 10 | development-dependencies: 11 | dependency-type: "development" 12 | open-pull-requests-limit: 10 13 | - package-ecosystem: "github-actions" 14 | directory: "/" 15 | schedule: 16 | interval: "weekly" 17 | -------------------------------------------------------------------------------- /internal/license/public_key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt0Y2jGOLI70xkF4rmpNM 3 | hBqU3cUrwYCREgjT9TT77KusvhPVc16cdK83bpaGQy+Or1WyZpN+TCxT2vlaZet6 4 | RDo+55jRk7epazAHx9s+DLd6IzhSXakf6Sxh5JRK7Zn/75C1hYJMspcJ75EhLv4H 5 | qXj12dkyivcLAecGhWdIGK95J0P7f4EQQGwGL3rilCSlfkVVmE4qaPUaLqULKelq 6 | 7T2d+AklR+KwgtHINyKDPJ9+cCAMoEOrRBDPjcQ79k0yvP3BdHV394F+2Vt/AYOL 7 | dcVQBm3tqIySLGFtiJp+RIa+nJhMrd+G4sqwm4FhsmG35Fbr0XQJY0sM6MaFJcDH 8 | swIDAQAB 9 | -----END PUBLIC KEY----- -------------------------------------------------------------------------------- /tools/spanner/benchmark/benchmark.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | vars: 4 | BENCHMARK_CONFIG_FILE: config.yml 5 | 6 | tasks: 7 | gen-config: 8 | desc: Generate config file for benchmark 9 | cmds: 10 | - go test -v -run TestBenchmarkInsert . {{.SPANNER_ARGS}} -output-config-file {{.BENCHMARK_CONFIG_FILE}} 11 | status: 12 | - test -f {{.BENCHMARK_CONFIG_FILE}} 13 | 14 | clean: 15 | desc: Remove config file 16 | cmds: 17 | - rm -f {{.BENCHMARK_CONFIG_FILE}} 18 | -------------------------------------------------------------------------------- /internal/cli/chroot_others.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | //go:build !linux 10 | 11 | package cli 12 | 13 | func chroot(_ string, _ []string) error { 14 | return nil 15 | } 16 | -------------------------------------------------------------------------------- /resources/scripts/sign_for_darwin.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | _OS=$1 6 | _PATH_TO_SIGN=$2 7 | _IS_SNAPSHOT=$3 8 | 9 | check_cmd() { 10 | command -v "$1" > /dev/null 2>&1 11 | } 12 | 13 | if [ "$_OS" = "darwin" ]; then 14 | if check_cmd "quill"; then 15 | quill sign-and-notarize "$_PATH_TO_SIGN" --dry-run="$_IS_SNAPSHOT" --ad-hoc="$_IS_SNAPSHOT" -vv 16 | else 17 | echo "Aborted, missing quill" 18 | fi 19 | else 20 | echo "No need to sign binaries for ${_OS}" 21 | fi 22 | -------------------------------------------------------------------------------- /resources/docker/schema_registry/out.yaml: -------------------------------------------------------------------------------- 1 | http: 2 | enabled: false 3 | 4 | input: 5 | kafka: 6 | addresses: [ redpanda:9092 ] 7 | consumer_group: benthos_consumer_group 8 | topics: [ benthos_redpanda ] 9 | 10 | pipeline: 11 | processors: 12 | - schema_registry_decode: 13 | url: http://redpanda:8081 14 | 15 | - catch: 16 | - log: 17 | level: ERROR 18 | message: ${! error() } 19 | - bloblang: root = deleted() 20 | 21 | output: 22 | stdout: {} 23 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/out_stdout.yaml: -------------------------------------------------------------------------------- 1 | http: 2 | address: 0.0.0.0:4195 3 | enabled: true 4 | 5 | input: 6 | redpanda_common: 7 | consumer_group: cg_b 8 | topics: [ '.*' ] 9 | regexp_topics: true 10 | auto_replay_nacks: false 11 | partition_buffer_bytes: 1KiB 12 | 13 | processors: 14 | - mutation: 'root.source_topic = @kafka_topic' 15 | 16 | output: 17 | stdout: {} 18 | 19 | redpanda: 20 | seed_brokers: [ localhost:9092 ] 21 | 22 | metrics: 23 | prometheus: {} 24 | 25 | -------------------------------------------------------------------------------- /resources/scripts/third_party_licenses.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script should be run from the root of the repository. 4 | # 5 | # Creates a summary of all third party dependencies and their licenses. 6 | # 7 | # This script requires `go-licenses` to be installed: 8 | # 9 | # go install github.com/google/go-licenses@latest 10 | 11 | go-licenses report github.com/redpanda-data/connect/v4/cmd/redpanda-connect \ 12 | --template ./resources/scripts/third_party.md.tpl \ 13 | > licenses/third_party.md 14 | 15 | -------------------------------------------------------------------------------- /resources/scripts/fips_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this wrapper gets installed as /usr/bin/redpanda-connect-fips 4 | # and overrides several environment variables to work with rpk-fips 5 | 6 | export PATH="/opt/redpanda/bin:${PATH}" 7 | export GOFIPS="1" 8 | export LD_LIBRARY_PATH="/opt/redpanda/rpk-fips/lib" 9 | export OPENSSL_CONF="/opt/redpanda/rpk-fips/openssl/openssl-rpk.cnf" 10 | export OPENSSL_MODULES="/opt/redpanda/rpk-fips/lib/ossl-modules/" 11 | 12 | exec -a "$0" "/opt/redpanda/libexec/redpanda-connect-fips" "$@" 13 | -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/output/main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from collections.abc import AsyncIterator 3 | import logging 4 | from redpanda_connect import output, output_main, Value, Message 5 | 6 | @output(max_in_flight=1) 7 | async def my_output(config: Value, messages: AsyncIterator[Message]): 8 | _ = config 9 | async for message in messages: 10 | print(f"Outputting message: {message}") 11 | 12 | if __name__ == "__main__": 13 | logging.basicConfig(level=logging.INFO) 14 | asyncio.run(output_main(my_output)) 15 | -------------------------------------------------------------------------------- /public/components/ollama/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | package ollama 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/ollama" 14 | ) 15 | -------------------------------------------------------------------------------- /public/components/splunk/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | package splunk 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/splunk" 14 | ) 15 | -------------------------------------------------------------------------------- /.github/workflows/govulncheck.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Vulnerability Check 3 | 4 | on: 5 | # Disabled due to ollama issues that are not fixed yet. 6 | # schedule: 7 | # - cron: '0 1 * * *' # run at 1 AM UTC 8 | workflow_dispatch: 9 | 10 | jobs: 11 | vulnerability-check: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v4 16 | 17 | - name: Run Go Vulnerability Check 18 | uses: golang/govulncheck-action@v1 19 | with: 20 | go-version-input: stable 21 | -------------------------------------------------------------------------------- /config/examples/resources/set_grab_cache.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - cache: 4 | resource: foocache 5 | operator: set 6 | key: foo 7 | value: "static value" 8 | - cache: 9 | resource: foocache 10 | operator: get 11 | key: foo 12 | 13 | tests: 14 | - name: Example test case 1 15 | environment: {} 16 | target_processors: /pipeline/processors 17 | input_batch: 18 | - content: 'ignored value' 19 | output_batches: 20 | - - content_equals: 'static value' -------------------------------------------------------------------------------- /public/components/cohere/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 8 | 9 | package cohere 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/cohere" 14 | ) 15 | -------------------------------------------------------------------------------- /public/components/openai/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 8 | 9 | package openai 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/openai" 14 | ) 15 | -------------------------------------------------------------------------------- /public/components/snowflake/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | package snowflake 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/snowflake" 14 | ) 15 | -------------------------------------------------------------------------------- /internal/impl/snowflake/resources/ssh_keys/README.md: -------------------------------------------------------------------------------- 1 | # Commands used to generate private SSH keys for Snowpipe tests 2 | 3 | ```shell 4 | > openssl genrsa 2048 | openssl pkcs8 -topk8 -v2 des3 -inform PEM -passout pass:test123 -out internal/impl/snowflake/resources/ssh_keys/snowflake_rsa_key.p8 5 | > openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -nocrypt -out internal/impl/snowflake/resources/ssh_keys/snowflake_rsa_key.pem 6 | ``` 7 | 8 | Note: For the encrypted key we're using `-v2 des3` because we only support PKCS#5 v2.0: https://linux.die.net/man/1/pkcs8 9 | -------------------------------------------------------------------------------- /config/test/protobuf/schema/person.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package testing; 3 | 4 | import "google/protobuf/timestamp.proto"; 5 | 6 | message Person { 7 | enum Device { 8 | DEVICE_UNSPECIFIED = 0; 9 | DEVICE_IOS = 1; 10 | DEVICE_ANDROID = 2; 11 | } 12 | 13 | string first_name = 1; 14 | string last_name = 2; 15 | string full_name = 3; 16 | int32 age = 4; 17 | int32 id = 5; // Unique ID number for this person. 18 | string email = 6; 19 | 20 | google.protobuf.Timestamp last_updated = 7; 21 | 22 | Device device = 8; 23 | } 24 | -------------------------------------------------------------------------------- /public/components/google/package.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Redpanda Data, Inc. 3 | * 4 | * Licensed as a Redpanda Enterprise file under the Redpanda Community 5 | * License (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * 8 | * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 9 | */ 10 | 11 | package google 12 | 13 | import ( 14 | // Bring in the internal plugin definitions. 15 | _ "github.com/redpanda-data/connect/v4/internal/impl/google" 16 | ) 17 | -------------------------------------------------------------------------------- /public/components/mysql/package.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Redpanda Data, Inc. 3 | * 4 | * Licensed as a Redpanda Enterprise file under the Redpanda Community 5 | * License (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * 8 | * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 9 | */ 10 | 11 | package mysql 12 | 13 | import ( 14 | // Bring in the internal plugin definitions. 15 | _ "github.com/redpanda-data/connect/v4/internal/impl/mysql" 16 | ) 17 | -------------------------------------------------------------------------------- /public/components/slack/package.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Redpanda Data, Inc. 3 | * 4 | * Licensed as a Redpanda Enterprise file under the Redpanda Community 5 | * License (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * 8 | * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 9 | */ 10 | 11 | package slack 12 | 13 | import ( 14 | // Bring in the internal plugin definitions. 15 | _ "github.com/redpanda-data/connect/v4/internal/impl/slack" 16 | ) 17 | -------------------------------------------------------------------------------- /public/components/aws/enterprise/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 8 | 9 | package enterprise 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/aws/enterprise" 14 | ) 15 | -------------------------------------------------------------------------------- /public/components/gcp/enterprise/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 8 | 9 | package enterprise 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/gcp/enterprise" 14 | ) 15 | -------------------------------------------------------------------------------- /public/components/kafka/enterprise/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | package enterprise 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/kafka/enterprise" 14 | ) 15 | -------------------------------------------------------------------------------- /public/components/mongodb/enterprise/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | package enterprise 10 | 11 | import ( 12 | // Bring in the internal plugin definitions. 13 | _ "github.com/redpanda-data/connect/v4/internal/impl/mongodb/cdc" 14 | ) 15 | -------------------------------------------------------------------------------- /config/rag/templates/ollama_embeddings.yaml: -------------------------------------------------------------------------------- 1 | name: ollama_embed 2 | type: processor 3 | 4 | fields: 5 | - name: input_type 6 | type: string 7 | 8 | mapping: | 9 | root.ollama_embeddings = { 10 | "model": "nomic-embed-text", 11 | "text": "%s: ${!content().string()}".format(this.input_type), 12 | } 13 | 14 | tests: 15 | - name: ollama_embeddings test 16 | config: 17 | input_type: "search_document" 18 | expected: 19 | ollama_embeddings: 20 | model: nomic-embed-text 21 | text: "search_document: ${!content().string()}" 22 | -------------------------------------------------------------------------------- /public/components/postgresql/package.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Redpanda Data, Inc. 3 | * 4 | * Licensed as a Redpanda Enterprise file under the Redpanda Community 5 | * License (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * 8 | * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 9 | */ 10 | 11 | package postgresql 12 | 13 | import ( 14 | // Bring in the internal plugin definitions. 15 | _ "github.com/redpanda-data/connect/v4/internal/impl/postgresql" 16 | ) 17 | -------------------------------------------------------------------------------- /config/test/bloblang/cities_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: test cities mapping 3 | target_mapping: './cities.blobl' 4 | environment: {} 5 | input_batch: 6 | - content: | 7 | { 8 | "locations": [ 9 | {"name": "Seattle", "state": "WA"}, 10 | {"name": "New York", "state": "NY"}, 11 | {"name": "Bellevue", "state": "WA"}, 12 | {"name": "Olympia", "state": "WA"} 13 | ] 14 | } 15 | output_batches: 16 | - 17 | - json_equals: {"Cities": "Bellevue, Olympia, Seattle"} -------------------------------------------------------------------------------- /config/test/infile_resource_mock.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - resource: http_submit 4 | 5 | processor_resources: 6 | - label: http_submit 7 | http: 8 | url: http://nonexistant.foo/ 9 | verb: POST 10 | 11 | tests: 12 | - name: test_case 13 | target_processors: /pipeline/processors 14 | mocks: 15 | http_submit: 16 | mapping: 'root = {"abc": 123}' 17 | input_batch: 18 | - json_content: 19 | foo: bar 20 | output_batches: 21 | - - json_equals: 22 | abc: 123 23 | bloblang: '!errored()' 24 | -------------------------------------------------------------------------------- /internal/impl/timeplus/interface.go: -------------------------------------------------------------------------------- 1 | package timeplus 2 | 3 | import "context" 4 | 5 | // Writer is the interface. Currently only http writer is implemented. Caller needs to make sure all writes contain the same `cols` 6 | type Writer interface { 7 | Write(ctx context.Context, cols []string, rows [][]any) error 8 | } 9 | 10 | // Reader is the interface. Called MUST guarantee that the `Run` method is called before `Read` or `Close` 11 | type Reader interface { 12 | Run(sql string) error 13 | Read(ctx context.Context) (map[string]any, error) 14 | Close(ctx context.Context) error 15 | } 16 | -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/input/plugin.yaml: -------------------------------------------------------------------------------- 1 | name: PROJECT_NAME_HERE 2 | summary: Add your summary here 3 | command: ["./main"] 4 | type: input 5 | fields: [] 6 | # Example of how to add configuration fields: 7 | # fields: 8 | # - name: foo 9 | # description: "The foo field" 10 | # type: string # options: string, int, float, bool, unknown 11 | # kind: scalar # or list or map 12 | # default: "fizzbuzz" 13 | # - name: bar 14 | # description: "The bar field" 15 | # type: int 16 | # kind: list 17 | # # omitting default means that it's a required field 18 | -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/output/plugin.yaml: -------------------------------------------------------------------------------- 1 | name: PROJECT_NAME_HERE 2 | summary: Add your summary here 3 | command: ["./main"] 4 | type: output 5 | fields: [] 6 | # Example of how to add configuration fields: 7 | # fields: 8 | # - name: foo 9 | # description: "The foo field" 10 | # type: string # options: string, int, float, bool, unknown 11 | # kind: scalar # or list or map 12 | # default: "fizzbuzz" 13 | # - name: bar 14 | # description: "The bar field" 15 | # type: int 16 | # kind: list 17 | # # omitting default means that it's a required field 18 | -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/processor/plugin.yaml: -------------------------------------------------------------------------------- 1 | name: PROJECT_NAME_HERE 2 | summary: Add your summary here 3 | command: ["./main"] 4 | type: processor 5 | fields: [] 6 | # Example of how to add configuration fields: 7 | # fields: 8 | # - name: foo 9 | # description: "The foo field" 10 | # type: string # options: string, int, float, bool, unknown 11 | # kind: scalar # or list or map 12 | # default: "fizzbuzz" 13 | # - name: bar 14 | # description: "The bar field" 15 | # type: int 16 | # kind: list 17 | # # omitting default means that it's a required field 18 | -------------------------------------------------------------------------------- /internal/impl/ollama/subprocess.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 8 | 9 | //go:build !unix 10 | 11 | package ollama 12 | 13 | import ( 14 | "os/exec" 15 | 16 | "github.com/redpanda-data/connect/v4/internal/singleton" 17 | ) 18 | 19 | var ollamaProcess *singleton.Singleton[*exec.Cmd] 20 | -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/input/plugin.yaml: -------------------------------------------------------------------------------- 1 | name: PROJECT_NAME_HERE 2 | summary: Add your summary here 3 | command: ["uv", "run", "main.py"] 4 | type: input 5 | fields: [] 6 | # Example of how to add configuration fields: 7 | # fields: 8 | # - name: foo 9 | # description: "The foo field" 10 | # type: string # options: string, int, float, bool, unknown 11 | # kind: scalar # or list or map 12 | # default: "fizzbuzz" 13 | # - name: bar 14 | # description: "The bar field" 15 | # type: int 16 | # kind: list 17 | # # omitting default means that it's a required field 18 | -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/output/plugin.yaml: -------------------------------------------------------------------------------- 1 | name: PROJECT_NAME_HERE 2 | summary: Add your summary here 3 | command: ["uv", "run", "main.py"] 4 | type: output 5 | fields: [] 6 | # Example of how to add configuration fields: 7 | # fields: 8 | # - name: foo 9 | # description: "The foo field" 10 | # type: string # options: string, int, float, bool, unknown 11 | # kind: scalar # or list or map 12 | # default: "fizzbuzz" 13 | # - name: bar 14 | # description: "The bar field" 15 | # type: int 16 | # kind: list 17 | # # omitting default means that it's a required field 18 | -------------------------------------------------------------------------------- /resources/docker/profiling/config.yaml: -------------------------------------------------------------------------------- 1 | http: 2 | address: 0.0.0.0:4195 3 | debug_endpoints: true 4 | 5 | input: 6 | generate: 7 | interval: "1s" 8 | mapping: | 9 | root.id = uuid_v4() 10 | root.bar = [] # [ "foo", "bar" ] 11 | 12 | output: 13 | sql_insert: 14 | driver: clickhouse 15 | dsn: clickhouse://localhost:9000/ 16 | table: foo 17 | columns: [ id, bar ] 18 | args_mapping: '[ 19 | this.id, 20 | this.bar, 21 | ]' 22 | 23 | metrics: 24 | prometheus: {} 25 | 26 | # tracer: 27 | # jaeger: 28 | # agent_address: 'localhost:6831' 29 | -------------------------------------------------------------------------------- /config/test/cookbooks/filtering_benthos_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: Basic filter 3 | environment: {} 4 | target_processors: /pipeline/processors/0 5 | input_batch: 6 | - content: '{"doc":{"should":"remain"},"id":"1"}' 7 | - content: '{"doc":{"should":"not remain"},"id":"2"}' 8 | metadata: 9 | topic: foo 10 | - content: '{"doc":{"should":"not remain","type":"bar"},"id":"3"}' 11 | - content: '{"doc":{"should":"not remain","urls":["https://www.benthos.dev/"]},"id":"4"}' 12 | output_batches: 13 | - - content_equals: '{"doc":{"should":"remain"},"id":"1"}' -------------------------------------------------------------------------------- /internal/rpcplugin/pythontemplate/processor/plugin.yaml: -------------------------------------------------------------------------------- 1 | name: PROJECT_NAME_HERE 2 | summary: Add your summary here 3 | command: ["uv", "run", "main.py"] 4 | type: processor 5 | fields: [] 6 | # Example of how to add configuration fields: 7 | # fields: 8 | # - name: foo 9 | # description: "The foo field" 10 | # type: string # options: string, int, float, bool, unknown 11 | # kind: scalar # or list or map 12 | # default: "fizzbuzz" 13 | # - name: bar 14 | # description: "The bar field" 15 | # type: int 16 | # kind: list 17 | # # omitting default means that it's a required field 18 | -------------------------------------------------------------------------------- /internal/impl/timeplus/http/header.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/base64" 5 | "net/http" 6 | ) 7 | 8 | // NewHeader creates a standard Timeplus HTTP header. 9 | func NewHeader(apikey, username, password string) http.Header { 10 | header := http.Header{} 11 | 12 | header.Add("Content-Type", "application/json") 13 | 14 | if len(username)+len(password) > 0 { 15 | auth := username + ":" + password 16 | header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth))) 17 | } else if len(apikey) > 0 { 18 | header.Add("X-Api-Key", apikey) 19 | } 20 | 21 | return header 22 | } 23 | -------------------------------------------------------------------------------- /resources/scripts/release_notes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo "For installation instructions check out the [getting started guide](https://docs.redpanda.com/redpanda-connect/guides/getting_started)." 3 | cat CHANGELOG.md | awk ' 4 | /^## [0-9]/ { 5 | release++; 6 | } 7 | /TBD$/ { 8 | print ""; 9 | print "NOTE: This is a release candidate, you can download a binary from this page."; 10 | } 11 | !/^## [0-9]/ { 12 | if ( release == 1 ) print; 13 | if ( release > 1 ) exit; 14 | }' 15 | echo "The full change log can be [found here](https://github.com/redpanda-data/connect/blob/main/CHANGELOG.md)." 16 | -------------------------------------------------------------------------------- /config/test/bloblang/also_tests_boolean_operands.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: neither exists 3 | target_processors: ./boolean_operands.yaml#/pipeline/processors 4 | input_batch: 5 | - content: '{"none":"of the target values"}' 6 | - content: '{"first":true}' 7 | - content: '{"first":false}' 8 | - content: '{"first":true,"second":true}' 9 | output_batches: 10 | - - content_equals: '{"ands":"failed","ors":"failed"}' 11 | - content_equals: '{"ands":"failed","ors":true}' 12 | - content_equals: '{"ands":false,"ors":"failed"}' 13 | - content_equals: '{"ands":true,"ors":true}' 14 | -------------------------------------------------------------------------------- /config/rag/templates/openai_embeddings.yaml: -------------------------------------------------------------------------------- 1 | name: oai_embed 2 | type: processor 3 | 4 | fields: 5 | - name: api_key 6 | type: string 7 | - name: dimensions 8 | type: int 9 | 10 | mapping: | 11 | root.openai_embeddings = { 12 | "api_key": this.api_key, 13 | "model": "text-embedding-3-small", 14 | "dimensions": this.dimensions, 15 | } 16 | 17 | tests: 18 | - name: openai_embeddings test 19 | config: 20 | api_key: "sk-foo" 21 | dimensions: 768 22 | expected: 23 | openai_embeddings: 24 | api_key: sk-foo 25 | model: text-embedding-3-small 26 | dimensions: 768 27 | -------------------------------------------------------------------------------- /internal/impl/snowflake/streaming/README.md: -------------------------------------------------------------------------------- 1 | # Snowflake Integration SDK for Redpanda Connect 2 | 3 | 4 | ### Testing 5 | 6 | To enable integration tests, you need to follow the instructions here to generate a public/private key for snowflake: https://docs.snowflake.com/en/user-guide/key-pair-auth 7 | 8 | Run the `openssl` commands from that guide in the `resources` directory to generate the correct keys for the integration test (the test requires the private key is unencrypted), then run the following: 9 | 10 | ``` 11 | SNOWFLAKE_USER=XXX \ 12 | SNOWFLAKE_ACCOUNT=alskjd-asdaks \ 13 | SNOWFLAKE_DB=xxx \ 14 | go test -v . 15 | ``` 16 | -------------------------------------------------------------------------------- /config/test/structured_metadata.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | stdin: 3 | codec: lines 4 | pipeline: 5 | processors: 6 | - mapping: | 7 | meta foo = { "a": "hello" } 8 | meta bar = { "b": { "c": "hello" } } 9 | meta baz = [ { "a": "hello" }, { "b": { "c": "hello" } } ] 10 | output: 11 | stdout: 12 | codec: lines 13 | 14 | tests: 15 | - name: Should not fail 16 | input_batch: 17 | - content: hello 18 | output_batches: 19 | - - metadata_equals: 20 | foo: { "a": "hello" } 21 | bar: { "b": { "c": "hello" } } 22 | baz: [ { "a": "hello" }, { "b": { "c": "hello" } } ] 23 | -------------------------------------------------------------------------------- /tools/spanner/Taskfile.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | vars: 4 | GIT_ROOT: 5 | sh: git rev-parse --show-toplevel 6 | SPANNER_ARGS: -spanner.project_id=sandbox-rpcn-457914 -spanner.instance_id=rpcn-tests-spanner -spanner.database_id=rpcn-tests 7 | 8 | includes: 9 | benchmark: 10 | taskfile: ./benchmark/benchmark.yml 11 | dir: benchmark 12 | terraform: 13 | taskfile: ./terraform/terraform.yml 14 | dir: terraform 15 | 16 | tasks: 17 | test: 18 | desc: Run Spanner integration tests 19 | dir: '{{.GIT_ROOT}}' 20 | cmds: 21 | - go test -v -run TestIntegrationReal ./internal/impl/gcp/enterprise/... {{.SPANNER_ARGS}} 22 | -------------------------------------------------------------------------------- /tools/spanner/benchmark/config.tmpl.yml: -------------------------------------------------------------------------------- 1 | http: 2 | enabled: true 3 | address: 0.0.0.0:4195 4 | debug_endpoints: true 5 | 6 | input: 7 | gcp_spanner_cdc: 8 | project_id: {{.ProjectID}} 9 | instance_id: {{.InstanceID}} 10 | database_id: {{.DatabaseID}} 11 | stream_id: {{.StreamID}} 12 | start_timestamp: {{.StartTimestamp}} 13 | end_timestamp: {{.EndTimestamp}} 14 | heartbeat_interval: "5s" 15 | batching: 16 | count: 1000 17 | 18 | pipeline: 19 | processors: 20 | - benchmark: 21 | interval: 5s 22 | count_bytes: true 23 | 24 | output: 25 | drop: {} 26 | 27 | metrics: 28 | prometheus: {} 29 | -------------------------------------------------------------------------------- /internal/impl/snowflake/streaming/streaming_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Redpanda Data, Inc. 3 | * 4 | * Licensed as a Redpanda Enterprise file under the Redpanda Community 5 | * License (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * 8 | * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 9 | */ 10 | 11 | package streaming 12 | 13 | import ( 14 | "testing" 15 | 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestDebugModeDisabled(t *testing.T) { 20 | // So I can't forget to disable this! 21 | require.False(t, debug) 22 | } 23 | -------------------------------------------------------------------------------- /licenses/Apache-2.0_header.go.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /internal/impl/redpanda/testdata/uppercase/README.md: -------------------------------------------------------------------------------- 1 | # Redpanda Golang WASM Transform 2 | 3 | To get started you first need to have at least go 1.20 installed. 4 | 5 | You can get started by modifying the transform.go file 6 | with your logic. 7 | 8 | Once you're ready to test out your transform live you need to: 9 | 10 | 1. Make sure you have a container running via rpk container start 11 | 1. Run rpk transform build 12 | 1. Create your topics via rpk topic create 13 | 1. Run rpk transform deploy 14 | 1. Then use rpk topic produce and rpk topic consume 15 | to see your transformation live! 16 | -------------------------------------------------------------------------------- /resources/scripts/tag_bundles.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script should be run from the root of the repository. 4 | # 5 | # Creates a new tag for each bundle we provide for Redpanda Connect plugins, 6 | # where the tag matches the pattern public/bundle//, where 7 | # is the bundle name and matches the version of RPCN that the 8 | # bundle references. 9 | 10 | for dir in $(ls ./public/bundle); do 11 | bundle_path="public/bundle/$dir" 12 | modline=$( cd $bundle_path && cat go.mod | grep "redpanda-data/connect/v" ) 13 | modline_split=( $modline ) 14 | version=${modline_split[2]} 15 | git tag "$bundle_path/$version" 16 | done 17 | 18 | -------------------------------------------------------------------------------- /internal/impl/couchbase/testdata/configure-server.sh: -------------------------------------------------------------------------------- 1 | #!bin/bash 2 | 3 | set -m 4 | 5 | /entrypoint.sh couchbase-server & 6 | 7 | sleep 8 8 | 9 | # Setup initial cluster/ Initialize Node 10 | couchbase-cli cluster-init -c 127.0.0.1 --cluster-name $CLUSTER_NAME --cluster-username $COUCHBASE_ADMINISTRATOR_USERNAME \ 11 | --cluster-password $COUCHBASE_ADMINISTRATOR_PASSWORD --services data --cluster-ramsize 1024 12 | 13 | sleep 2 14 | 15 | # Setup Administrator username and password 16 | curl -s http://127.0.0.1:8091/settings/web -d port=8091 -d username=$COUCHBASE_ADMINISTRATOR_USERNAME -d password=$COUCHBASE_ADMINISTRATOR_PASSWORD 17 | 18 | sleep 2 19 | 20 | touch /is-ready 21 | 22 | fg 1 23 | -------------------------------------------------------------------------------- /public/components/zeromq/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package zeromq 16 | -------------------------------------------------------------------------------- /config/README.md: -------------------------------------------------------------------------------- 1 | Config 2 | ====== 3 | 4 | This directory shows some config examples. Some are real world applications, some are examples of [config unit tests][unit-tests]. 5 | 6 | If you're looking for specific config examples for a use case you have then try generating one with the `redpanda-connect create` subcommand. For example, to create a config that reads Kafka messages, decodes them with a schema registry service, and writes them to NATS JetStream you could use the following command: 7 | 8 | ```sh 9 | rpk connect create kafka/schema_registry_decode/nats_jetstream > example.yaml 10 | ``` 11 | 12 | [unit-tests]: https://www.docs.redpanda.com/redpanda-connect/docs/configuration/unit_testing 13 | -------------------------------------------------------------------------------- /config/test/bloblang/fans.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - mutation: | 4 | root.fans = this.fans.filter(fan -> fan.obsession > 0.5) 5 | 6 | tests: 7 | - name: Bloblang fans test 8 | input_batch: 9 | - json_content: 10 | id: foo 11 | fans: 12 | - {"name":"bev","obsession":0.57} 13 | - {"name":"grace","obsession":0.21} 14 | - {"name":"ali","obsession":0.89} 15 | - {"name":"vic","obsession":0.43} 16 | output_batches: 17 | - - json_equals: 18 | id: foo 19 | fans: 20 | - {"name":"bev","obsession":0.57} 21 | - {"name":"ali","obsession":0.89} 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /docs/modules/components/pages/tracers/none.adoc: -------------------------------------------------------------------------------- 1 | = none 2 | :type: tracer 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Do not send tracing events anywhere. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | tracer: 30 | none: {} 31 | ``` 32 | 33 | 34 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/README.md: -------------------------------------------------------------------------------- 1 | Redpanda Benchmarking 2 | ===================== 3 | 4 | I've created this directory as a convenient way to create Redpanda topics and benchmark Redpanda Connect instances against them with various configs. 5 | 6 | ## Getting Started 7 | 8 | ```sh 9 | # Start redpanda, grafana, etc 10 | docker-compose up -d 11 | 12 | # Create some test topics 13 | rpk topic create testing_a -p 10 14 | rpk topic create testing_b -p 10 15 | rpk topic create testing_c -p 10 16 | rpk topic create testing_d -p 10 17 | ``` 18 | 19 | ## Generate Data 20 | 21 | ```sh 22 | # Inserts 100,000,000 records into topic testing_a 23 | redpanda-connect run ./generate.yaml 24 | ``` 25 | 26 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/generate.yaml: -------------------------------------------------------------------------------- 1 | http: 2 | address: 0.0.0.0:4197 3 | enabled: true 4 | 5 | input: 6 | generate: 7 | interval: 1s 8 | count: 100_000_000 9 | batch_size: 1 10 | mapping: | 11 | root.ID = counter() 12 | root.Name = [ "frosty", "spot", "oodles" ].index(random_int() % 3) 13 | root.Gooeyness = (random_int() % 100) / 100 14 | 15 | output: 16 | redpanda_common: 17 | topic: testing_a 18 | # max_in_flight: 1 # Ensures ordering from the generate input 19 | 20 | redpanda: 21 | seed_brokers: [ localhost:9092 ] 22 | logs_topic: generate.logs 23 | status_topic: generate.status 24 | 25 | metrics: 26 | prometheus: {} 27 | 28 | -------------------------------------------------------------------------------- /taskfiles/tools.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | tasks: 4 | install-all: 5 | deps: 6 | - install-golangci-lint 7 | - install-govulncheck 8 | 9 | install-golangci-lint: 10 | desc: Install golangci-lint 11 | required: 12 | vars: 13 | - GOLANGCI_LINT_VERSION 14 | run: once 15 | silent: true 16 | sources: 17 | - .versions 18 | cmds: 19 | - GOBIN={{.GOBIN}} go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v{{.GOLANGCI_LINT_VERSION}} 20 | 21 | install-govulncheck: 22 | desc: Install govulncheck 23 | run: always 24 | silent: true 25 | cmds: 26 | - GOBIN={{.GOBIN}} go install golang.org/x/vuln/cmd/govulncheck@latest -------------------------------------------------------------------------------- /docs/modules/components/pages/metrics/none.adoc: -------------------------------------------------------------------------------- 1 | = none 2 | :type: metrics 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Disable metrics entirely. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | metrics: 30 | none: {} 31 | mapping: "" 32 | ``` 33 | 34 | 35 | -------------------------------------------------------------------------------- /public/components/pulsar/arm_32.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build arm 16 | 17 | package pulsar 18 | -------------------------------------------------------------------------------- /public/components/ockam/windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build windows || arm 16 | 17 | package ockam 18 | -------------------------------------------------------------------------------- /config/examples/site_analytics.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | http_server: 3 | address: 0.0.0.0:4196 4 | path: /poke 5 | allowed_verbs: [ POST, HEAD ] 6 | cors: 7 | enabled: true 8 | allowed_origins: 9 | - '*' 10 | processors: 11 | - metric: 12 | type: counter 13 | name: site_visit 14 | labels: 15 | host: ${! meta("h") } 16 | path: ${! meta("p") } 17 | referrer: ${! meta("r") } 18 | - bloblang: 'root = deleted()' 19 | 20 | metrics: 21 | mapping: | 22 | # Only emit our custom metric, and no internal Redpanda Connect metrics. 23 | root = if ![ 24 | "site_visit", 25 | ].contains($path) { deleted() } else { $path } 26 | prometheus: {} 27 | -------------------------------------------------------------------------------- /docs/modules/components/pages/outputs/drop.adoc: -------------------------------------------------------------------------------- 1 | = drop 2 | :type: output 3 | :status: stable 4 | :categories: ["Utility"] 5 | 6 | 7 | 8 | //// 9 | THIS FILE IS AUTOGENERATED! 10 | 11 | To make changes, edit the corresponding source file under: 12 | 13 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 14 | 15 | And: 16 | 17 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 18 | //// 19 | 20 | // © 2024 Redpanda Data Inc. 21 | 22 | 23 | component_type_dropdown::[] 24 | 25 | 26 | Drops all messages. 27 | 28 | ```yml 29 | # Config fields, showing default values 30 | output: 31 | label: "" 32 | drop: {} 33 | ``` 34 | 35 | 36 | -------------------------------------------------------------------------------- /public/components/couchbase/package_32bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build arm 16 | 17 | package couchbase 18 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/out_bridge.yaml: -------------------------------------------------------------------------------- 1 | http: 2 | address: 0.0.0.0:4196 3 | enabled: true 4 | 5 | input: 6 | redpanda_common: 7 | consumer_group: cg_d 8 | topics: [ testing_a ] 9 | auto_replay_nacks: false 10 | partition_buffer_bytes: 1KiB 11 | 12 | pipeline: 13 | processors: 14 | - sleep: 15 | duration: 1ns 16 | 17 | output: 18 | fallback: 19 | - redpanda_common: 20 | topic: testing_b 21 | - stdout: {} 22 | processors: 23 | - mapping: | 24 | root = "Uh oh: %v failed to deliver due to: %v".format(content().string(), @fallback_error) 25 | 26 | redpanda: 27 | seed_brokers: [ localhost:9092 ] 28 | 29 | metrics: 30 | prometheus: {} 31 | 32 | -------------------------------------------------------------------------------- /config/test/bloblang/boolean_operands.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | ands = (first && second).catch("failed") 5 | ors = (first || second).catch("failed") 6 | 7 | tests: 8 | - name: neither exists 9 | target_processors: /pipeline/processors 10 | input_batch: 11 | - content: '{"none":"of the target values"}' 12 | - content: '{"first":true}' 13 | - content: '{"first":false}' 14 | - content: '{"first":true,"second":true}' 15 | output_batches: 16 | - - content_equals: '{"ands":"failed","ors":"failed"}' 17 | - content_equals: '{"ands":"failed","ors":true}' 18 | - content_equals: '{"ands":false,"ors":"failed"}' 19 | - content_equals: '{"ands":true,"ors":true}' 20 | -------------------------------------------------------------------------------- /config/test/resources/other_mappings_benthos_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: run all resources 3 | target_processors: '/processor_resources' 4 | input_batch: 5 | - content: 'example content' 6 | output_batches: 7 | - 8 | - content_equals: BAR EXAMPLE CONTENT 9 | 10 | - name: run just prefix 11 | target_processors: '/processor_resources/0' 12 | input_batch: 13 | - content: 'example content' 14 | output_batches: 15 | - 16 | - content_equals: bar example content 17 | 18 | - name: run just upper 19 | target_processors: '/processor_resources/1' 20 | input_batch: 21 | - content: 'example content' 22 | output_batches: 23 | - 24 | - content_equals: EXAMPLE CONTENT 25 | -------------------------------------------------------------------------------- /config/template_examples/processor_plugin_alias.yaml: -------------------------------------------------------------------------------- 1 | name: plugin_alias 2 | type: processor 3 | status: experimental 4 | summary: This is a test template to check that plugin aliases work. 5 | 6 | fields: 7 | - name: url 8 | description: the url of the thing. 9 | type: string 10 | default: http://defaultschemas.example.com 11 | 12 | mapping: 'root.schema_registry_decode.url = this.url' 13 | 14 | tests: 15 | - name: Basic fields 16 | config: 17 | url: 'http://schemas.example.com' 18 | expected: 19 | schema_registry_decode: 20 | url: 'http://schemas.example.com' 21 | 22 | - name: Use Default 23 | config: {} 24 | expected: 25 | schema_registry_decode: 26 | url: 'http://defaultschemas.example.com' 27 | -------------------------------------------------------------------------------- /tools/spanner/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # Crash log files 9 | crash.log 10 | crash.*.log 11 | 12 | # Exclude all .tfvars files, which are likely to contain sensitive data 13 | *.tfvars 14 | *.tfvars.json 15 | 16 | # Ignore override files as they're usually used for local dev 17 | override.tf 18 | override.tf.json 19 | *_override.tf 20 | *_override.tf.json 21 | 22 | # Ignore CLI configuration files 23 | .terraformrc 24 | terraform.rc 25 | 26 | # Ignore lock files 27 | .terraform.lock.hcl 28 | 29 | # Ignore any credentials 30 | *-key.json 31 | *.json.key 32 | credentials.json 33 | 34 | # Logs 35 | *.log 36 | 37 | # Local development 38 | .env 39 | .envrc 40 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | external_labels: 5 | monitor: 'rpcn-benchmark' 6 | 7 | scrape_configs: 8 | - job_name: 'prometheus' 9 | scrape_interval: 5s 10 | static_configs: 11 | - targets: ['localhost:9090'] 12 | 13 | - job_name: 'rpcn-stdout' 14 | scrape_interval: 5s 15 | static_configs: 16 | - targets: ['host.docker.internal:4195'] 17 | 18 | - job_name: 'rpcn-bridge' 19 | scrape_interval: 5s 20 | static_configs: 21 | - targets: ['host.docker.internal:4196'] 22 | 23 | - job_name: 'rpcn-generate' 24 | scrape_interval: 5s 25 | static_configs: 26 | - targets: ['host.docker.internal:4197'] 27 | 28 | -------------------------------------------------------------------------------- /config/rag/templates/pgvector_query.yaml: -------------------------------------------------------------------------------- 1 | name: pgvector 2 | type: processor 3 | 4 | fields: 5 | - name: table 6 | type: string 7 | - name: dsn 8 | type: string 9 | - name: limit 10 | type: int 11 | default: 3 12 | 13 | mapping: | 14 | root.sql_raw = { 15 | "driver": "postgres", 16 | "dsn": this.dsn, 17 | "query": """ 18 | SELECT ( 19 | SELECT STRING_AGG(t2.document, '' ORDER BY chunk_id ASC) 20 | FROM %s t2 21 | WHERE t1.key = t2.key AND t1.topic = t2.topic 22 | GROUP BY key 23 | ) AS document, key, topic 24 | FROM %s t1 25 | ORDER BY embeddings <-> $1 26 | LIMIT %d 27 | """.format(this.table, this.table, this.limit), 28 | "args_mapping": "[ this.vector() ]", 29 | } 30 | -------------------------------------------------------------------------------- /docs/modules/components/pages/scanners/json_documents.adoc: -------------------------------------------------------------------------------- 1 | = json_documents 2 | :type: scanner 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Consumes a stream of one or more JSON documents. 26 | 27 | Introduced in version 4.27.0. 28 | 29 | ```yml 30 | # Config fields, showing default values 31 | json_documents: {} 32 | ``` 33 | 34 | 35 | -------------------------------------------------------------------------------- /resources/docker/schema_registry/in.yaml: -------------------------------------------------------------------------------- 1 | http: 2 | enabled: false 3 | 4 | input: 5 | generate: 6 | interval: 1s 7 | mapping: | 8 | root.ID = uuid_v4() 9 | root.Name = [ "frosty", "spot", "oodles" ].index(random_int() % 3) 10 | root.Gooeyness = (random_int() % 100) / 100 11 | root.Bouncing = random_int() % 2 == 0 12 | 13 | pipeline: 14 | processors: 15 | - schema_registry_encode: 16 | url: http://redpanda:8081 17 | subject: benthos_example 18 | refresh_period: 15s 19 | 20 | - catch: 21 | - log: 22 | level: ERROR 23 | message: ${! error() } 24 | - bloblang: root = deleted() 25 | 26 | output: 27 | kafka: 28 | addresses: [ redpanda:9092 ] 29 | topic: benthos_redpanda 30 | -------------------------------------------------------------------------------- /config/rag/ingestion/redpanda-docs.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | git: 3 | repository_url: https://github.com/redpanda-data/docs.git 4 | branch: main 5 | poll_interval: "10s" 6 | include_patterns: 7 | - 'modules/**/*.adoc' 8 | exclude_patterns: 9 | - 'modules/ROOT/**' 10 | max_file_size: 1048576 11 | 12 | pipeline: 13 | processors: 14 | - mapping: | 15 | meta = @.map_each_key(key -> key.trim_prefix("git_")) 16 | root = if @is_binary { 17 | deleted() 18 | } 19 | output: 20 | kafka_franz: 21 | seed_brokers: ["${REDPANDA_BROKERS}"] 22 | sasl: [] 23 | tls: 24 | enabled: false 25 | topic: "rp.ai.rag.rpdocs" 26 | key: ${!meta("git_file_path")} 27 | metadata: 28 | include_patterns: [".*"] 29 | -------------------------------------------------------------------------------- /config/rag/templates/cohere_embeddings.yaml: -------------------------------------------------------------------------------- 1 | name: cohere_embed 2 | type: processor 3 | 4 | fields: 5 | - name: api_key 6 | type: string 7 | - name: input_type 8 | type: string 9 | - name: dimensions 10 | type: int 11 | 12 | mapping: | 13 | root.cohere_embeddings = { 14 | "api_key": this.api_key, 15 | "model": "embed-v4.0", 16 | "input_type": this.input_type, 17 | "dimensions": this.dimensions 18 | } 19 | 20 | tests: 21 | - name: cohere_embeddings test 22 | config: 23 | api_key: "sk-foo" 24 | input_type: "search_document" 25 | dimensions: 1536 26 | expected: 27 | cohere_embeddings: 28 | api_key: sk-foo 29 | model: embed-v4.0 30 | input_type: search_document 31 | dimensions: 1536 32 | -------------------------------------------------------------------------------- /internal/rpcplugin/subprocess/signal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !unix 16 | 17 | package subprocess 18 | 19 | import "os" 20 | 21 | var stopSignal = os.Interrupt 22 | -------------------------------------------------------------------------------- /docs/modules/components/pages/processors/noop.adoc: -------------------------------------------------------------------------------- 1 | = noop 2 | :type: processor 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Noop is a processor that does nothing, the message passes through unchanged. Why? Sometimes doing nothing is the braver option. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | label: "" 30 | noop: {} 31 | ``` 32 | 33 | 34 | -------------------------------------------------------------------------------- /internal/protoconnect/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:generate protoc -I=../../proto/redpanda/api/connect/v1alpha1 --go_out=../.. status.proto 16 | 17 | package protoconnect 18 | -------------------------------------------------------------------------------- /internal/impl/postgresql/pglogicalstream/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md 8 | 9 | package pglogicalstream 10 | 11 | import "fmt" 12 | 13 | // TableFQN is both a table name AND a schema name 14 | // 15 | // TableFQN should always be SAFE and validated before creating 16 | type TableFQN struct { 17 | Schema string 18 | Table string 19 | } 20 | 21 | // String satifies the Stringer interface 22 | func (t TableFQN) String() string { 23 | return fmt.Sprintf("%s.%s", t.Schema, t.Table) 24 | } 25 | -------------------------------------------------------------------------------- /internal/rpcplugin/subprocess/signal_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build unix 16 | 17 | package subprocess 18 | 19 | import "syscall" 20 | 21 | var stopSignal = syscall.SIGTERM 22 | -------------------------------------------------------------------------------- /config/test/json_contains_predicate.yaml: -------------------------------------------------------------------------------- 1 | processor_resources: 2 | - label: woof_drop 3 | mapping: | 4 | root = if this.resource."service.name" == "woof" { deleted() } 5 | 6 | tests: 7 | - name: woof drop test 8 | target_processors: 'woof_drop' 9 | input_batch: 10 | - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"aaa","service.name":"meow"}}' 11 | - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"bbb","service.name":"woof"}}' 12 | - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"ccc","service.name":"quack"}}' 13 | output_batches: 14 | - 15 | - json_contains: { "resource": { "cloud.platform": "aws_eks", "host.id": "aaa" } } 16 | - json_contains: { "resource": { "cloud.platform": "aws_eks", "host.id": "ccc" } } 17 | -------------------------------------------------------------------------------- /docs/modules/components/pages/scanners/tar.adoc: -------------------------------------------------------------------------------- 1 | = tar 2 | :type: scanner 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Consume a tar archive file by file. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | tar: {} 30 | ``` 31 | 32 | == Metadata 33 | 34 | This scanner adds the following metadata to each message: 35 | 36 | - `tar_name` 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /docs/modules/components/pages/caches/noop.adoc: -------------------------------------------------------------------------------- 1 | = noop 2 | :type: cache 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Noop is a cache that stores nothing, all gets returns not found. Why? Sometimes doing nothing is the braver option. 26 | 27 | Introduced in version 4.27.0. 28 | 29 | ```yml 30 | # Config fields, showing default values 31 | label: "" 32 | noop: {} 33 | ``` 34 | 35 | 36 | -------------------------------------------------------------------------------- /config/test/bloblang/message_expansion.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | let doc_root = this.without("items") 5 | root = items.map_each($doc_root.merge(this)) 6 | - unarchive: 7 | format: json_array 8 | 9 | tests: 10 | - name: Sample object 11 | target_processors: /pipeline/processors 12 | input_batch: 13 | - content: | 14 | { 15 | "id": "foobar", 16 | "items": [ 17 | {"content":"foo"}, 18 | {"content":"bar"}, 19 | {"content":"baz"} 20 | ] 21 | } 22 | output_batches: 23 | - - content_equals: '{"content":"foo","id":"foobar"}' 24 | - content_equals: '{"content":"bar","id":"foobar"}' 25 | - content_equals: '{"content":"baz","id":"foobar"}' 26 | -------------------------------------------------------------------------------- /internal/impl/azure/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package azure will eventually contain all implementations of Azure 16 | // components (that are currently within ./internal/old) 17 | package azure 18 | -------------------------------------------------------------------------------- /internal/impl/mqtt/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package mqtt will eventually contain all implementations of MQTT components 16 | // (that are currently within ./internal/old) 17 | package mqtt 18 | -------------------------------------------------------------------------------- /internal/impl/sftp/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package sftp will eventually contain all implementations of SFTP components 16 | // (that are currently within ./internal/old) 17 | package sftp 18 | -------------------------------------------------------------------------------- /public/components/gcp/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package gcp 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/gcp" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/git/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package git 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/git" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/nsq/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nsq 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/nsq" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/sql/snowflake.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !arm 16 | 17 | package sql 18 | 19 | import ( 20 | // Import snowflake specifically. 21 | _ "github.com/snowflakedb/gosnowflake" 22 | ) 23 | -------------------------------------------------------------------------------- /resources/docker/schema_registry/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | services: 3 | redpanda: 4 | image: docker.redpanda.com/redpandadata/redpanda 5 | ports: 6 | - 8081:8081 7 | command: 8 | - 'redpanda start' 9 | - '--smp 1' 10 | - '--overprovisioned' 11 | - '--kafka-addr 0.0.0.0:9092' 12 | - '--advertise-kafka-addr redpanda:9092' 13 | - '--pandaproxy-addr 0.0.0.0:8082' 14 | - '--advertise-pandaproxy-addr redpanda:8082' 15 | 16 | connect-in: 17 | image: ghcr.io/redpanda-data/connect 18 | command: [ '-w', '-c', '/connect.yaml' ] 19 | volumes: 20 | - ./in.yaml:/connect.yaml 21 | 22 | connect-out: 23 | image: ghcr.io/redpanda-data/connect 24 | command: [ '-w', '-c', '/connect.yaml' ] 25 | volumes: 26 | - ./out.yaml:/connect.yaml 27 | -------------------------------------------------------------------------------- /public/components/avro/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package avro 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/avro" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/hdfs/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package hdfs 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/hdfs" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/mqtt/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package mqtt 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/mqtt" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/nats/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nats 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/nats" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/otlp/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package otlp 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/otlp" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/sftp/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sftp 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/sftp" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/text/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package text 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/text" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/wasm/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package wasm 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/wasm" 20 | ) 21 | -------------------------------------------------------------------------------- /config/test/unit_test_example_benthos_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: example test 3 | target_processors: '/pipeline/processors' 4 | environment: {} 5 | input_batch: 6 | - content: 'example content' 7 | metadata: 8 | example_key: example metadata value 9 | output_batches: 10 | - 11 | - content_equals: EXAMPLE CONTENTend 12 | metadata_equals: 13 | example_key: example metadata value 14 | 15 | - name: empty message test 16 | target_processors: '/pipeline/processors' 17 | environment: {} 18 | input_batch: 19 | - content: '' 20 | metadata: 21 | example_key: example metadata value 22 | output_batches: 23 | - 24 | - content_equals: end 25 | metadata_equals: 26 | example_key: example metadata value 27 | -------------------------------------------------------------------------------- /public/bundle/enterprise/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | // Package enterprise imports all enterprise licensed plugin implementations 10 | // that ship with Redpanda Connect, along with all free plugin implementations. 11 | // This is a convenient way of importing every single connector at the cost of a 12 | // larger dependency tree for your application. 13 | package enterprise 14 | 15 | import ( 16 | // Import all public sub-categories. 17 | _ "github.com/redpanda-data/connect/v4/public/components/all" 18 | ) 19 | -------------------------------------------------------------------------------- /public/components/amqp09/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package amqp09 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/amqp09" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/amqp1/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package amqp1 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/amqp1" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/azure/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package azure 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/azure" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/crypto/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crypto 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/crypto" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/cypher/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cypher 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/cypher" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/dgraph/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package dgraph 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/dgraph" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/jaeger/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package jaeger 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/jaeger" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/kafka/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kafka 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/kafka" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/pusher/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package pusher 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/pusher" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/qdrant/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package qdrant 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/qdrant" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/redis/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package redis 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/redis" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/sentry/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sentry 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/sentry" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/statsd/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package statsd 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/statsd" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/discord/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package discord 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/discord" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/gateway/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rpingress 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/gateway" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/influxdb/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package influxdb 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/influxdb" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/maxmind/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package maxmind 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/maxmind" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/mongodb/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package mongodb 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/mongodb" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/msgpack/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package msgpack 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/msgpack" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/nanomsg/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nanomsg 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/nanomsg" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/pinecone/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package pinecone 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/pinecone" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/questdb/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package questdb 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/questdb" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/redpanda/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package wasm 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/redpanda" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/spicedb/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package spicedb 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/spicedb" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/twitter/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package twitter 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/twitter" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/cassandra/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cassandra 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/cassandra" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/changelog/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package changelog 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/changelog" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/confluent/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package confluent 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/confluent" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/javascript/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package couchbase 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/javascript" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/memcached/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package memcached 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/memcached" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/beanstalkd/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package beanstalkd 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/beanstalkd" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/cockroachdb/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cockroachdb 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/cockroachdb" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/opensearch/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package opensearch 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/opensearch" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/prometheus/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package prometheus 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/prometheus" 20 | ) 21 | -------------------------------------------------------------------------------- /internal/agent/template.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Redpanda Data, Inc. 3 | * 4 | * Licensed as a Redpanda Enterprise file under the Redpanda Community 5 | * License (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * 8 | * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md 9 | */ 10 | 11 | package agent 12 | 13 | import ( 14 | "embed" 15 | 16 | "github.com/redpanda-data/connect/v4/internal/template" 17 | ) 18 | 19 | //go:embed template/* 20 | var embeddedTemplate embed.FS 21 | 22 | // CreateTemplate generates the agent SDK template for RPCN. 23 | func CreateTemplate(dir string, vars map[string]string) error { 24 | return template.CreateTemplate(embeddedTemplate, dir, template.WithStrippedPrefix("template"), template.WithVariables(vars)) 25 | } 26 | -------------------------------------------------------------------------------- /internal/impl/amqp1/input_description.adoc: -------------------------------------------------------------------------------- 1 | == Metadata 2 | 3 | This input adds the following metadata fields to each message: 4 | 5 | ```text 6 | - amqp_content_type 7 | - amqp_content_encoding 8 | - amqp_creation_time 9 | - All string typed message annotations 10 | ``` 11 | 12 | You can access these metadata fields using xref:configuration:interpolation.adoc#bloblang-queries[function interpolation]. 13 | 14 | By setting `read_header` to `true`, additional message header properties will be added to each message: 15 | 16 | ```text 17 | - amqp_durable 18 | - amqp_priority 19 | - amqp_ttl 20 | - amqp_first_acquirer 21 | - amqp_delivery_count 22 | ``` 23 | 24 | == Performance 25 | 26 | This input benefits from receiving multiple messages in flight in parallel for improved performance. 27 | You can tune the max number of in flight messages with the field `credit`. 28 | -------------------------------------------------------------------------------- /public/components/elasticsearch/v8/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package elasticsearch 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/elasticsearch/v8" 20 | ) 21 | -------------------------------------------------------------------------------- /public/components/pulsar/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !arm 16 | 17 | package pulsar 18 | 19 | import ( 20 | // Bring in the internal plugin definitions. 21 | _ "github.com/redpanda-data/connect/v4/internal/impl/pulsar" 22 | ) 23 | -------------------------------------------------------------------------------- /public/components/couchbase/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !arm 16 | 17 | package couchbase 18 | 19 | import ( 20 | // Bring in the internal plugin definitions. 21 | _ "github.com/redpanda-data/connect/v4/internal/impl/couchbase" 22 | ) 23 | -------------------------------------------------------------------------------- /public/components/ockam/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !windows && !arm 16 | 17 | package ockam 18 | 19 | import ( 20 | // Bring in the internal plugin definitions. 21 | _ "github.com/redpanda-data/connect/v4/internal/impl/ockam" 22 | ) 23 | -------------------------------------------------------------------------------- /resources/plugin_uploader/test_data/dist/artifacts.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "metadata.json", 4 | "path": "dist/metadata.json", 5 | "internal_type": 30, 6 | "type": "Metadata" 7 | }, 8 | { 9 | "name": "redpanda-cow", 10 | "path": "dist/cow_linux_amd64_v1/redpanda-cow", 11 | "goos": "linux", 12 | "goarch": "amd64", 13 | "goamd64": "v1", 14 | "internal_type": 4, 15 | "type": "Binary", 16 | "extra": { 17 | "Binary": "redpanda-cow", 18 | "Ext": "", 19 | "ID": "cow" 20 | } 21 | }, 22 | { 23 | "name": "redpanda-cow", 24 | "path": "dist/cow_darwin_arm64/redpanda-cow", 25 | "goos": "darwin", 26 | "goarch": "arm64", 27 | "internal_type": 4, 28 | "type": "Binary", 29 | "extra": { 30 | "Binary": "redpanda-cow", 31 | "Ext": "", 32 | "ID": "cow" 33 | } 34 | } 35 | ] -------------------------------------------------------------------------------- /.github/workflows/release_python_sdk.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Publish Python Plugin Package 2 | 3 | on: 4 | workflow_dispatch: # Manual trigger 5 | 6 | jobs: 7 | build-and-publish: 8 | runs-on: ubuntu-latest 9 | 10 | # See: https://docs.pypi.org/trusted-publishers/using-a-publisher/ 11 | environment: pypi 12 | permissions: 13 | id-token: write 14 | 15 | defaults: 16 | run: 17 | working-directory: public/plugin/python 18 | 19 | steps: 20 | - name: Checkout code 21 | uses: actions/checkout@v4 22 | 23 | - name: Set up uv 24 | uses: astral-sh/setup-uv@v6 25 | 26 | - name: Build the package with uv 27 | run: uv build 28 | 29 | - name: Publish to PyPI 30 | uses: pypa/gh-action-pypi-publish@release/v1 31 | with: 32 | packages-dir: public/plugin/python/dist 33 | 34 | -------------------------------------------------------------------------------- /docs/modules/components/pages/scanners/chunker.adoc: -------------------------------------------------------------------------------- 1 | = chunker 2 | :type: scanner 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Split an input stream into chunks of a given number of bytes. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | chunker: 30 | size: 0 # No default (required) 31 | ``` 32 | 33 | == Fields 34 | 35 | === `size` 36 | 37 | The size of each chunk in bytes. 38 | 39 | 40 | *Type*: `int` 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /public/components/zeromq/x_benthos_extra.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build x_benthos_extra 16 | 17 | package zeromq 18 | 19 | import ( 20 | // Bring in the internal plugin definitions. 21 | _ "github.com/redpanda-data/connect/v4/internal/impl/zeromq" 22 | ) 23 | -------------------------------------------------------------------------------- /resources/scripts/add_license_headers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script should be run from the root of the repository. 4 | # 5 | # Scans all files with a .go suffix and filters for files that are missing a 6 | # Copyright notice at the top. Each detected file is then modified to have the 7 | # Apache 2.0 license header at the top, as this is the default license for the 8 | # repository. 9 | # 10 | # Therefore, it is important before running this script that any enterprise 11 | # licensed files are already annotated with the appropriate license header. 12 | 13 | tmpFile="./license_script.tmp" 14 | 15 | for file in $(find . -name \*.go); do 16 | topLine=$(head -n 1 $file) 17 | if [[ $topLine != *"Copyright"* ]]; then 18 | cat ./licenses/Apache-2.0_header.go.txt > $tmpFile 19 | cat $file >> $tmpFile 20 | cat $tmpFile > $file 21 | fi 22 | done 23 | 24 | rm -f $tmpFile 25 | -------------------------------------------------------------------------------- /config/test/filters.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = if content().contains("delete me") { deleted() }' 4 | 5 | tests: 6 | - name: delete one of one message 7 | input_batch: 8 | - content: "hello world delete me please" 9 | 10 | - name: delete all messages 11 | input_batch: 12 | - content: "hello world delete me please" 13 | - content: "hello world 2 delete me please" 14 | - content: "hello world 3 delete me please" 15 | - content: "hello world 4 delete me please" 16 | 17 | - name: delete some messages 18 | input_batch: 19 | - content: "hello world delete me please" 20 | - content: "hello world 2" 21 | - content: "hello world 3 delete me please" 22 | - content: "hello world 4" 23 | output_batches: 24 | - - content_equals: "hello world 2" 25 | - content_equals: "hello world 4" 26 | -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/processor/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/redpanda-data/benthos/v4/public/service" 7 | "github.com/redpanda-data/connect/v4/public/plugin/go/rpcn" 8 | ) 9 | 10 | type config struct{} 11 | 12 | func main() { 13 | rpcn.ProcessorMain(func(cfg config) (service.BatchProcessor, error) { 14 | return &myProcessor{cfg: cfg}, nil 15 | }) 16 | } 17 | 18 | type myProcessor struct { 19 | cfg config 20 | } 21 | 22 | var _ service.BatchProcessor = (*myProcessor)(nil) 23 | 24 | // ProcessBatch implements service.BatchProcessor. 25 | func (*myProcessor) ProcessBatch(_ context.Context, batch service.MessageBatch) ([]service.MessageBatch, error) { 26 | return []service.MessageBatch{batch}, nil 27 | } 28 | 29 | // Close implements service.BatchProcessor. 30 | func (*myProcessor) Close(context.Context) error { 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /config/test/deduplicate.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - dedupe: 4 | cache: local 5 | key: ${! content() } 6 | 7 | cache_resources: 8 | - label: local 9 | memory: 10 | default_ttl: 1m 11 | 12 | tests: 13 | - name: de-duplicate across batches 14 | input_batches: 15 | - 16 | - content: '1' 17 | - content: '2' 18 | - content: '3' 19 | - content: '4' 20 | - content: '3' 21 | - content: '3' 22 | - content: '3' 23 | - 24 | - content: '4' 25 | - content: '1' 26 | - content: '1' 27 | - content: '3' 28 | - content: '4' 29 | - content: '4' 30 | - content: '2' 31 | - content: '1' 32 | output_batches: 33 | - 34 | - content_equals: 1 35 | - content_equals: 2 36 | - content_equals: 3 37 | - content_equals: 4 38 | -------------------------------------------------------------------------------- /config/test/deduplicate_lru.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - dedupe: 4 | cache: local_lru 5 | key: ${! content() } 6 | 7 | cache_resources: 8 | - label: local_lru 9 | lru: {} 10 | 11 | tests: 12 | - name: de-duplicate across batches using lru cache 13 | input_batches: 14 | - 15 | - content: '1' 16 | - content: '2' 17 | - content: '3' 18 | - content: '4' 19 | - content: '3' 20 | - content: '3' 21 | - content: '3' 22 | - 23 | - content: '4' 24 | - content: '1' 25 | - content: '1' 26 | - content: '3' 27 | - content: '4' 28 | - content: '4' 29 | - content: '2' 30 | - content: '1' 31 | output_batches: 32 | - 33 | - content_equals: 1 34 | - content_equals: 2 35 | - content_equals: 3 36 | - content_equals: 4 37 | -------------------------------------------------------------------------------- /config/test/resources/some_mappings.yaml: -------------------------------------------------------------------------------- 1 | processor_resources: 2 | - label: prefix 3 | bloblang: 'root = "foo " + content()' 4 | 5 | - label: upper 6 | bloblang: 'root = content().uppercase()' 7 | 8 | tests: 9 | - name: run all resources 10 | target_processors: '/processor_resources' 11 | input_batch: 12 | - content: 'example content' 13 | output_batches: 14 | - 15 | - content_equals: FOO EXAMPLE CONTENT 16 | 17 | - name: run just prefix 18 | target_processors: '/processor_resources/0' 19 | input_batch: 20 | - content: 'example content' 21 | output_batches: 22 | - 23 | - content_equals: foo example content 24 | 25 | - name: run just upper 26 | target_processors: '/processor_resources/1' 27 | input_batch: 28 | - content: 'example content' 29 | output_batches: 30 | - 31 | - content_equals: EXAMPLE CONTENT 32 | -------------------------------------------------------------------------------- /public/plugin/python/src/redpanda_connect/_proto/redpanda/runtime/v1alpha1/message_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | """Client and server classes corresponding to protobuf-defined services.""" 2 | import grpc 3 | import warnings 4 | GRPC_GENERATED_VERSION = '1.71.0' 5 | GRPC_VERSION = grpc.__version__ 6 | _version_not_supported = False 7 | try: 8 | from grpc._utilities import first_version_is_lower 9 | _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) 10 | except ImportError: 11 | _version_not_supported = True 12 | if _version_not_supported: 13 | raise RuntimeError(f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in redpanda/runtime/v1alpha1/message_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.') -------------------------------------------------------------------------------- /docs/modules/components/pages/metrics/json_api.adoc: -------------------------------------------------------------------------------- 1 | = json_api 2 | :type: metrics 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Serves metrics as JSON object with the service wide HTTP service at the endpoints `/stats` and `/metrics`. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | metrics: 30 | json_api: {} 31 | mapping: "" 32 | ``` 33 | 34 | This metrics type is useful for debugging as it provides a human readable format that you can parse with tools such as `jq` 35 | 36 | 37 | -------------------------------------------------------------------------------- /internal/impl/nsq/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Surprisingly, there still seems to be absolutely no options available for 2 | # running a single node set up of NSQ for testing purposes, which means it's 3 | # extremely awkward to write real integration tests. Instead, we have this 4 | # docker-compose set up where if you run it and then execute unit tests for this 5 | # package it'll run them. 6 | version: '3' 7 | services: 8 | nsqlookupd: 9 | image: nsqio/nsq 10 | command: /nsqlookupd 11 | ports: 12 | - "4160:4160" 13 | - "4161" 14 | nsqd: 15 | image: nsqio/nsq 16 | command: /nsqd --lookupd-tcp-address=nsqlookupd:4160 17 | depends_on: 18 | - nsqlookupd 19 | ports: 20 | - "4150:4150" 21 | - "4151" 22 | nsqadmin: 23 | image: nsqio/nsq 24 | command: /nsqadmin --lookupd-http-address=nsqlookupd:4161 25 | depends_on: 26 | - nsqlookupd 27 | ports: 28 | - "4171" 29 | -------------------------------------------------------------------------------- /config/test/deduplicate_ttlru.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - dedupe: 4 | cache: local_ttlru 5 | key: ${! content() } 6 | 7 | cache_resources: 8 | - label: local_ttlru 9 | ttlru: 10 | default_ttl: 1m 11 | 12 | tests: 13 | - name: de-duplicate across batches using ttlru cache 14 | input_batches: 15 | - 16 | - content: '1' 17 | - content: '2' 18 | - content: '3' 19 | - content: '4' 20 | - content: '3' 21 | - content: '3' 22 | - content: '3' 23 | - 24 | - content: '4' 25 | - content: '1' 26 | - content: '1' 27 | - content: '3' 28 | - content: '4' 29 | - content: '4' 30 | - content: '2' 31 | - content: '1' 32 | output_batches: 33 | - 34 | - content_equals: 1 35 | - content_equals: 2 36 | - content_equals: 3 37 | - content_equals: 4 38 | -------------------------------------------------------------------------------- /docs/modules/components/pages/scanners/skip_bom.adoc: -------------------------------------------------------------------------------- 1 | = skip_bom 2 | :type: scanner 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Skip one or more byte order marks for each opened child scanner. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | skip_bom: 30 | into: 31 | to_the_end: {} 32 | ``` 33 | 34 | == Fields 35 | 36 | === `into` 37 | 38 | The child scanner to feed the resulting stream into. 39 | 40 | 41 | *Type*: `scanner` 42 | 43 | *Default*: `{"to_the_end":{}}` 44 | 45 | 46 | -------------------------------------------------------------------------------- /config/test/bloblang/env.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | foo_env = env("FOO") 5 | bar_env = env("BAR") 6 | 7 | tests: 8 | - name: both exist 9 | target_processors: /pipeline/processors 10 | environment: 11 | FOO: fooval 12 | BAR: barval 13 | input_batch: 14 | - content: '{}' 15 | output_batches: 16 | - - content_equals: '{"bar_env":"barval","foo_env":"fooval"}' 17 | 18 | - name: foo exists 19 | target_processors: /pipeline/processors 20 | environment: 21 | FOO: fooval 22 | input_batch: 23 | - content: '{}' 24 | output_batches: 25 | - - content_equals: '{"bar_env":null,"foo_env":"fooval"}' 26 | 27 | - name: neither exists 28 | target_processors: /pipeline/processors 29 | environment: {} 30 | input_batch: 31 | - content: '{}' 32 | output_batches: 33 | - - content_equals: '{"bar_env":null,"foo_env":null}' 34 | 35 | -------------------------------------------------------------------------------- /cmd/serverless/connect-lambda/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/redpanda-data/connect/v4/internal/impl/aws" 19 | 20 | // Import all plugins defined within the repo. 21 | _ "github.com/redpanda-data/connect/v4/public/components/all" 22 | ) 23 | 24 | func main() { 25 | aws.RunLambda() 26 | } 27 | -------------------------------------------------------------------------------- /config/template_examples/input_sqs_example.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sqs_list 2 | type: input 3 | 4 | fields: 5 | - name: urls 6 | type: string 7 | kind: list 8 | - name: region 9 | type: string 10 | default: us-east-1 11 | 12 | mapping: | 13 | root.broker.inputs = this.urls.map_each(url -> { 14 | "aws_sqs": { 15 | "url": url, 16 | "region": this.region, 17 | } 18 | }) 19 | 20 | tests: 21 | - name: urls array 22 | config: 23 | urls: 24 | - https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue1 25 | - https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue2 26 | expected: 27 | broker: 28 | inputs: 29 | - aws_sqs: 30 | url: https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue1 31 | region: us-east-1 32 | - aws_sqs: 33 | url: https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue2 34 | region: us-east-1 35 | -------------------------------------------------------------------------------- /internal/asyncroutine/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // package asyncroutine contains several small common patterns around async goroutines 16 | // that allows for clean shutdown and allows for writing plugins and ignoring some of 17 | // the boilerplate around launching goroutines and shutting them down cleanly. 18 | package asyncroutine 19 | -------------------------------------------------------------------------------- /resources/docker/profiling/README.md: -------------------------------------------------------------------------------- 1 | Profiling 2 | ========= 3 | 4 | This docker compose sets a Benthos instance up with a custom config, [Prometheus][prometheus], [Grafana][grafana] and [Jaeger][jaeger] in order to observe its performance under different configurations. 5 | 6 | # Set up 7 | 8 | - Run Grafana and Prometheus with `docker-compose up`. 9 | - Edit `config.yaml` and add whatever components you want to profile with. 10 | - Run Redpanda Connect with `redpanda-connect -c ./config.yaml`. 11 | - Open up Grafana at [http://localhost:3000/d/PHrVlmniz/benthos-dash](http://localhost:3000/d/PHrVlmniz/benthos-dash) 12 | - Go to [http://localhost:16686](http://localhost:16686) in order to observe opentracing events with Jaeger. 13 | - Use `go tool pprof http://localhost:4195/debug/pprof/profile` and similar endpoints to get profiling data. 14 | 15 | [prometheus]: https://prometheus.io/ 16 | [grafana]: https://grafana.com/ 17 | [jaeger]: https://www.jaegertracing.io/ 18 | -------------------------------------------------------------------------------- /public/components/aws/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package aws 16 | 17 | import ( 18 | // Bring in the internal plugin definitions. 19 | _ "github.com/redpanda-data/connect/v4/internal/impl/aws" 20 | _ "github.com/redpanda-data/connect/v4/internal/impl/kafka/aws" 21 | _ "github.com/redpanda-data/connect/v4/internal/impl/opensearch/aws" 22 | ) 23 | -------------------------------------------------------------------------------- /tools/spanner/terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | google = { 4 | source = "hashicorp/google" 5 | version = "~> 4.0" 6 | } 7 | } 8 | required_version = ">= 1.0" 9 | } 10 | 11 | provider "google" { 12 | project = var.project_id 13 | region = var.region 14 | } 15 | 16 | data "google_project" "current" {} 17 | 18 | resource "google_spanner_instance" "main" { 19 | name = var.instance_name 20 | config = var.instance_config 21 | display_name = var.instance_display_name 22 | num_nodes = var.instance_nodes 23 | 24 | labels = { 25 | environment = var.environment 26 | } 27 | } 28 | 29 | resource "google_spanner_database" "database" { 30 | instance = google_spanner_instance.main.name 31 | name = var.database_name 32 | 33 | deletion_protection = false 34 | version_retention_period = "1h" # Disable backups by setting retention to minimum 35 | enable_drop_protection = false 36 | } 37 | -------------------------------------------------------------------------------- /docs/modules/components/pages/processors/crash.adoc: -------------------------------------------------------------------------------- 1 | = crash 2 | :type: processor 3 | :status: beta 4 | :categories: ["Utility"] 5 | 6 | 7 | 8 | //// 9 | THIS FILE IS AUTOGENERATED! 10 | 11 | To make changes, edit the corresponding source file under: 12 | 13 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 14 | 15 | And: 16 | 17 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 18 | //// 19 | 20 | // © 2024 Redpanda Data Inc. 21 | 22 | 23 | component_type_dropdown::[] 24 | 25 | 26 | Crashes the process using a fatal log message. The log message can be set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries] which allows you to log the contents and metadata of messages. 27 | 28 | ```yml 29 | # Config fields, showing default values 30 | label: "" 31 | crash: "" # No default (required) 32 | ``` 33 | 34 | 35 | -------------------------------------------------------------------------------- /resources/docker/schema_registry/README.md: -------------------------------------------------------------------------------- 1 | Schema Registry 2 | =============== 3 | 4 | This is a neat little example of using a schema registry service with Benthos. Both the Kafka implementation and the schema registry service are being handled with [Redpanda](https://redpanda.com/). 5 | 6 | Video run through of this demo: [https://youtu.be/HzuqbNw-vMo](https://youtu.be/HzuqbNw-vMo) 7 | More information about schema registry service: [https://docs.confluent.io/platform/current/schema-registry/index.html](https://docs.confluent.io/platform/current/schema-registry/index.html) 8 | How to set up a schema registry with Redpanda: [https://docs.redpanda.com/current/manage/schema-reg/](https://docs.redpanda.com/current/manage/schema-reg/) 9 | 10 | ## Run 11 | 12 | ```sh 13 | docker-compose up -d 14 | ``` 15 | 16 | ## Register initial schema 17 | 18 | ```sh 19 | ./insert_schema.sh 20 | ``` 21 | 22 | ## See generated messages 23 | 24 | ```sh 25 | docker-compose logs -f connect-out 26 | ``` 27 | -------------------------------------------------------------------------------- /config/test/env_var_stuff.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - mutation: | 4 | root.foo = "${BENTHOS_TEST_FOO:woof}" 5 | root.bar = env("BENTHOS_TEST_BAR").or("meow") 6 | 7 | tests: 8 | - name: only defaults 9 | environment: {} 10 | input_batch: 11 | - content: '{"id":"1"}' 12 | output_batches: 13 | - 14 | - json_equals: { "id": "1", "foo": "woof", "bar": "meow" } 15 | 16 | - name: both defined 17 | environment: 18 | BENTHOS_TEST_FOO: quack 19 | BENTHOS_TEST_BAR: moo 20 | input_batch: 21 | - content: '{"id":"1"}' 22 | output_batches: 23 | - 24 | - json_equals: { "id": "1", "foo": "quack", "bar": "moo" } 25 | 26 | - name: both defined again 27 | environment: 28 | BENTHOS_TEST_FOO: tweet 29 | BENTHOS_TEST_BAR: neigh 30 | input_batch: 31 | - content: '{"id":"1"}' 32 | output_batches: 33 | - 34 | - json_equals: { "id": "1", "foo": "tweet", "bar": "neigh" } -------------------------------------------------------------------------------- /docs/modules/components/pages/scanners/to_the_end.adoc: -------------------------------------------------------------------------------- 1 | = to_the_end 2 | :type: scanner 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Read the input stream all the way until the end and deliver it as a single message. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | to_the_end: {} 30 | ``` 31 | 32 | [CAUTION] 33 | ==== 34 | Some sources of data may not have a logical end, therefore caution should be made to exclusively use this scanner when the end of an input stream is clearly defined (and well within memory). 35 | ==== 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /public/components/sql/base/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package base brings in only the sql components, but none of the drivers for 16 | // them. It is up to you to import specifically the drivers you want to include. 17 | package base 18 | 19 | import ( 20 | // Bring in the internal plugin definitions. 21 | _ "github.com/redpanda-data/connect/v4/internal/impl/sql" 22 | ) 23 | -------------------------------------------------------------------------------- /internal/impl/mongodb/cdc/bson_util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/v4/blob/main/licenses/rcl.md 8 | 9 | package cdc 10 | 11 | import ( 12 | "math" 13 | 14 | "go.mongodb.org/mongo-driver/v2/bson" 15 | ) 16 | 17 | func bsonGetPath(doc bson.M, path ...string) any { 18 | var current any 19 | current = doc 20 | for _, segment := range path { 21 | d, ok := current.(bson.M) 22 | if !ok { 23 | return nil 24 | } 25 | current, ok = d[segment] 26 | if !ok { 27 | return nil 28 | } 29 | } 30 | return current 31 | } 32 | 33 | func nextTimestamp(ts bson.Timestamp) bson.Timestamp { 34 | if ts.I == math.MaxUint32 { 35 | return bson.Timestamp{T: ts.T + 1} 36 | } 37 | return bson.Timestamp{T: ts.T, I: ts.I + 1} 38 | } 39 | -------------------------------------------------------------------------------- /resources/docker/redpanda_benchmarking/out_order_verify.yaml: -------------------------------------------------------------------------------- 1 | http: 2 | enabled: false 3 | 4 | input: 5 | redpanda: 6 | seed_brokers: [ localhost:9092 ] 7 | consumer_group: cg_a 8 | topics: [ testing_b ] 9 | auto_replay_nacks: false 10 | 11 | output: 12 | drop: {} 13 | processors: 14 | - for_each: 15 | - mapping: | 16 | let count = counter(min: this.ID) 17 | root.mismatch = if $count != this.ID { 18 | "expected %v, got %v".format($count, this.ID) 19 | } 20 | - while: 21 | check: 'this.mismatch != null' 22 | processors: 23 | - log: 24 | level: WARN 25 | message: "Blocking pipeline after ordering mismatch detected: ${! this.mismatch }" 26 | - sleep: 27 | duration: 1m 28 | 29 | metrics: 30 | prometheus: 31 | push_interval: 1s 32 | push_job_name: benthos_push 33 | push_url: "http://localhost:9091" 34 | 35 | shutdown_timeout: 1s 36 | 37 | -------------------------------------------------------------------------------- /internal/rpcplugin/golangtemplate/output/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/redpanda-data/benthos/v4/public/service" 7 | "github.com/redpanda-data/connect/v4/public/plugin/go/rpcn" 8 | ) 9 | 10 | type config struct{} 11 | 12 | func main() { 13 | rpcn.OutputMain(func(cfg config) (output service.BatchOutput, maxInFlight int, batchPolicy service.BatchPolicy, err error) { 14 | output = &myOutput{cfg: cfg} 15 | maxInFlight = 1 16 | return 17 | }) 18 | } 19 | 20 | type myOutput struct { 21 | cfg config 22 | } 23 | 24 | var _ service.BatchOutput = (*myOutput)(nil) 25 | 26 | // Connect implements service.BatchOutput. 27 | func (*myOutput) Connect(context.Context) error { 28 | return nil 29 | } 30 | 31 | // WriteBatch implements service.BatchOutput. 32 | func (*myOutput) WriteBatch(context.Context, service.MessageBatch) error { 33 | return nil 34 | } 35 | 36 | // Close implements service.BatchOutput. 37 | func (*myOutput) Close(context.Context) error { 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /internal/impl/sentry/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sentry 16 | 17 | import "github.com/getsentry/sentry-go" 18 | 19 | type clientOptionsFunc func(opts *sentry.ClientOptions) *sentry.ClientOptions 20 | 21 | func withTransport(t sentry.Transport) clientOptionsFunc { 22 | return func(opts *sentry.ClientOptions) *sentry.ClientOptions { 23 | opts.Transport = t 24 | 25 | return opts 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /config/test/bloblang/windowed.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | root = this 5 | doc.count = json("doc.count").from_all().sum() 6 | doc.max = json("doc.count").from_all().fold(0, match { 7 | tally < value => value 8 | _ => tally 9 | }) 10 | 11 | # Drop all documents except the first. 12 | root = match { 13 | batch_index() > 0 => deleted() 14 | } 15 | 16 | tests: 17 | - name: Bloblang windowed functions test 18 | environment: {} 19 | target_processors: /pipeline/processors 20 | input_batch: 21 | - content: '{"doc":{"count":243,"contents":"foobar 1"}}' 22 | - content: '{"doc":{"count":71,"contents":"foobar 2"}}' 23 | - content: '{"doc":{"count":10,"contents":"foobar 3"}}' 24 | - content: '{"doc":{"count":333,"contents":"foobar 4"}}' 25 | - content: '{"doc":{"count":164,"contents":"foobar 5"}}' 26 | output_batches: 27 | - - content_equals: | 28 | {"doc":{"contents":"foobar 1","count":821,"max":333}} -------------------------------------------------------------------------------- /internal/rpcplugin/protogen.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rpcplugin 16 | 17 | //go:generate protoc -I=../../proto --go_opt=module=github.com/redpanda-data/connect/v4 --go-grpc_opt=module=github.com/redpanda-data/connect/v4 --go_out=../.. --go-grpc_out=../.. redpanda/runtime/v1alpha1/message.proto redpanda/runtime/v1alpha1/input.proto redpanda/runtime/v1alpha1/output.proto redpanda/runtime/v1alpha1/processor.proto 18 | -------------------------------------------------------------------------------- /internal/agent/template/README.md: -------------------------------------------------------------------------------- 1 | # Redpanda Agents 2 | 3 | This is a project generated from Redpanda Connect's agentic developer framework. 4 | 5 | You can define new agents in the `agents` folder as python, and hook them up to 6 | [`inputs`][inputs] and [`outputs`][outputs] using Redpanda Connect. 7 | 8 | Each agent can also be given a set of tools (exposed over MCP) as [resources][resources]. 9 | 10 | To showcase each of these, there is an example weather agent, that processes messages 11 | from `stdin` and writes it's output to `stdout` using an example `http` processor tool 12 | to lookup the weather in a given location. 13 | 14 | Running this example requires [`uv`](https://docs.astral.sh/uv/) to be installed on the 15 | host. Then you can run the agent using `rpk connect agent run`. 16 | 17 | [inputs]: https://docs.redpanda.com/redpanda-connect/components/inputs/about/ 18 | [outputs]: https://docs.redpanda.com/redpanda-connect/components/outputs/about/ 19 | [resources]: https://docs.redpanda.com/redpanda-connect/configuration/resources/ 20 | 21 | -------------------------------------------------------------------------------- /docs/modules/components/pages/inputs/beanstalkd.adoc: -------------------------------------------------------------------------------- 1 | = beanstalkd 2 | :type: input 3 | :status: experimental 4 | :categories: ["Services"] 5 | 6 | 7 | 8 | //// 9 | THIS FILE IS AUTOGENERATED! 10 | 11 | To make changes, edit the corresponding source file under: 12 | 13 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 14 | 15 | And: 16 | 17 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 18 | //// 19 | 20 | // © 2024 Redpanda Data Inc. 21 | 22 | 23 | component_type_dropdown::[] 24 | 25 | 26 | Reads messages from a Beanstalkd queue. 27 | 28 | Introduced in version 4.7.0. 29 | 30 | ```yml 31 | # Config fields, showing default values 32 | input: 33 | label: "" 34 | beanstalkd: 35 | address: 127.0.0.1:11300 # No default (required) 36 | ``` 37 | 38 | == Fields 39 | 40 | === `address` 41 | 42 | An address to connect to. 43 | 44 | 45 | *Type*: `string` 46 | 47 | 48 | ```yml 49 | # Examples 50 | 51 | address: 127.0.0.1:11300 52 | ``` 53 | 54 | 55 | -------------------------------------------------------------------------------- /internal/plugins/info_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package plugins 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | ) 22 | 23 | func TestInfoCSV(t *testing.T) { 24 | // This test parses the base csv and checks for any malformed fields. 25 | for k, v := range BaseInfo { 26 | assert.NotEmpty(t, v.Type, "plugin %v type field", k) 27 | assert.NotEmpty(t, v.Support, "plugin %v support field", k) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /public/plugin/python/README.md: -------------------------------------------------------------------------------- 1 | # Redpanda Connect Python Plugins 2 | 3 | This library allows you to create python plugins for [Redpanda Connect](https://www.redpanda.com/connect). 4 | 5 | In order to use create a processor plugin you can follow these steps: 6 | 7 | ```shell 8 | uv init project 9 | 10 | cd project 11 | 12 | uv add redpanda_connect 13 | 14 | cat < main.py 15 | import asyncio 16 | import logging 17 | import redpanda_connect 18 | 19 | 20 | @redpanda_connect.processor 21 | def yell(msg: redpanda_connect.Message) -> redpanda_connect.Message: 22 | msg.payload = msg.payload.upper() 23 | return msg 24 | 25 | if __name__ == "__main__": 26 | asyncio.run(redpanda_connect.processor_main(yell)) 27 | EOF 28 | 29 | cat < plugin.yaml 30 | name: foo 31 | summary: Just the simplest example 32 | command: ["uv", "run", "main.py"] 33 | type: processor 34 | fields: [] 35 | EOF 36 | 37 | cat < connect.yaml 38 | pipeline: 39 | processors: 40 | - foo: {} 41 | EOF 42 | 43 | rpk connect run --rpc-plugins=plugin.yaml connect.yaml 44 | ``` 45 | -------------------------------------------------------------------------------- /resources/docker/README.md: -------------------------------------------------------------------------------- 1 | Benthos Docker 2 | ============== 3 | 4 | This directory contains two Dockerfile definitions, one is a pure Go image based on [`busybox`][docker.busybox] (`Dockerfile`), the other (`Dockerfile.cgo`) is a CGO enabled build based on [`debian`][docker.debian]. 5 | 6 | The image has a [default config][default.config] but it's not particularly useful, so you'll either want to use the `-s` cli flag to define config values or copy a config into the path `/connect.yaml` as a volume. 7 | 8 | ```shell 9 | # Using a config file 10 | docker run --rm -v /path/to/your/config.yaml:/connect.yaml ghcr.io/redpanda-data/connect 11 | 12 | # Using a series of -s flags 13 | docker run --rm -p 4195:4195 ghcr.io/redpanda-data/connect \ 14 | -s "input.type=http_server" \ 15 | -s "output.type=kafka" \ 16 | -s "output.kafka.addresses=kafka-server:9092" \ 17 | -s "output.kafka.topic=benthos_topic" 18 | ``` 19 | 20 | [docker.busybox]: https://hub.docker.com/_/busybox/ 21 | [docker.debian]: https://hub.docker.com/_/debian 22 | [default.config]: ../config/docker.yaml 23 | -------------------------------------------------------------------------------- /public/bundle/free/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package free imports all free, open source plugin implementations that ship 16 | // with Redpanda Connect. This is a convenient way of importing every single 17 | // free connector at the cost of a larger dependency tree for your application. 18 | package free 19 | 20 | import ( 21 | // Import all public sub-categories. 22 | _ "github.com/redpanda-data/connect/v4/public/components/community" 23 | ) 24 | -------------------------------------------------------------------------------- /config/rag/retrieval/ollama_pgvector.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | http_server: 3 | path: /query 4 | allowed_verbs: 5 | - POST 6 | sync_response: 7 | status: "${!this.status.or(200)}" 8 | timeout: 60s 9 | pipeline: 10 | processors: 11 | - try: 12 | - json_schema: 13 | schema: | 14 | { 15 | "$schema": "http://json-schema.org/draft-07/schema#", 16 | "title": "HTTP Request schema", 17 | "type": "object", 18 | "properties": { 19 | "query": { 20 | "type": "string" 21 | } 22 | }, 23 | "required": [ 24 | "query" 25 | ] 26 | } 27 | - ollama_embed: 28 | input_type: search_query 29 | - pgvector: 30 | dsn: ${POSTGRES_DSN} 31 | table: ${POSTGRES_TABLE}_ollama 32 | output: 33 | processors: 34 | - mutation: | 35 | if errored() { 36 | root = {"status": 500, "error": error()} 37 | } 38 | sync_response: {} 39 | -------------------------------------------------------------------------------- /resources/docker/profiling/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | volumes: 2 | prometheus_data: {} 3 | grafana_data: {} 4 | 5 | services: 6 | jaeger: 7 | image: jaegertracing/all-in-one 8 | ports: 9 | - 6831:6831/udp 10 | - 16686:16686 11 | 12 | prometheus: 13 | image: prom/prometheus 14 | volumes: 15 | - ./prometheus/:/etc/prometheus/ 16 | - prometheus_data:/prometheus 17 | extra_hosts: 18 | - host.docker.internal:host-gateway 19 | command: 20 | - '--config.file=/etc/prometheus/prometheus.yml' 21 | - '--storage.tsdb.path=/prometheus' 22 | - '--web.console.libraries=/usr/share/prometheus/console_libraries' 23 | - '--web.console.templates=/usr/share/prometheus/consoles' 24 | ports: 25 | - 9090:9090 26 | 27 | grafana: 28 | image: grafana/grafana 29 | depends_on: 30 | - prometheus 31 | ports: 32 | - 3000:3000 33 | volumes: 34 | - grafana_data:/var/lib/grafana 35 | - ./grafana/provisioning/:/etc/grafana/provisioning/ 36 | env_file: 37 | - ./grafana/config.monitoring 38 | -------------------------------------------------------------------------------- /docs/modules/components/pages/caches/file.adoc: -------------------------------------------------------------------------------- 1 | = file 2 | :type: cache 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Stores each item in a directory as a file, where an item ID is the path relative to the configured directory. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | label: "" 30 | file: 31 | directory: "" # No default (required) 32 | ``` 33 | 34 | This type currently offers no form of item expiry or garbage collection, and is intended to be used for development and debugging purposes only. 35 | 36 | == Fields 37 | 38 | === `directory` 39 | 40 | The directory within which to store items. 41 | 42 | 43 | *Type*: `string` 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /internal/telemetry/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package telemetry 16 | 17 | import "github.com/redpanda-data/benthos/v4/public/service" 18 | 19 | type logWrapper struct { 20 | l *service.Logger 21 | } 22 | 23 | func (l *logWrapper) Errorf(format string, v ...any) { 24 | l.l.With("component", "resty").Debugf(format, v...) 25 | } 26 | 27 | func (*logWrapper) Warnf(string, ...any) { 28 | // Ignore 29 | } 30 | 31 | func (*logWrapper) Debugf(string, ...any) { 32 | // Ignore 33 | } 34 | -------------------------------------------------------------------------------- /config/rag/retrieval/openai_pgvector.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | http_server: 3 | path: /query 4 | allowed_verbs: 5 | - POST 6 | sync_response: 7 | status: "${!this.status.or(200)}" 8 | timeout: 60s 9 | pipeline: 10 | processors: 11 | - try: 12 | - json_schema: 13 | schema: | 14 | { 15 | "$schema": "http://json-schema.org/draft-07/schema#", 16 | "title": "HTTP Request schema", 17 | "type": "object", 18 | "properties": { 19 | "query": { 20 | "type": "string" 21 | } 22 | }, 23 | "required": [ 24 | "query" 25 | ] 26 | } 27 | - oai_embed: 28 | api_key: ${OPENAI_API_KEY} 29 | dimensions: 768 30 | - pgvector: 31 | dsn: ${POSTGRES_DSN} 32 | table: ${POSTGRES_TABLE}_openai 33 | output: 34 | processors: 35 | - mutation: | 36 | if errored() { 37 | root = {"status": 500, "error": error()} 38 | } 39 | sync_response: {} 40 | -------------------------------------------------------------------------------- /internal/impl/redpanda/testdata/uppercase/transform.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bytes" 19 | 20 | "github.com/redpanda-data/redpanda/src/transform-sdk/go/transform" 21 | ) 22 | 23 | func main() { 24 | transform.OnRecordWritten(makeUppercase) 25 | } 26 | 27 | func makeUppercase(e transform.WriteEvent, w transform.RecordWriter) error { 28 | return w.Write(transform.Record{ 29 | Key: e.Record().Key, 30 | Value: bytes.ToUpper(e.Record().Value), 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /public/components/sql/sqlite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Platforms and architectures list from https://pkg.go.dev/modernc.org/sqlite?utm_source=godoc#hdr-Supported_platforms_and_architectures 16 | // Last updated from modernc.org/sqlite@v1.19.1 17 | //go:build (darwin && (amd64 || arm64)) || (freebsd && (amd64 || arm64)) || (linux && (386 || amd64 || arm || arm64 || riscv64)) || (windows && (amd64 || arm64)) 18 | 19 | package sql 20 | 21 | import ( 22 | // Import sqlite specifically. 23 | _ "modernc.org/sqlite" 24 | ) 25 | -------------------------------------------------------------------------------- /public/plugin/python/src/redpanda_connect/_proto/redpanda/runtime/v1alpha1/message_pb2_grpc.pyi: -------------------------------------------------------------------------------- 1 | """ 2 | @generated by mypy-protobuf. Do not edit manually! 3 | isort:skip_file 4 | Copyright 2025 Redpanda Data, Inc. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | """ 18 | import abc 19 | import collections.abc 20 | import grpc 21 | import grpc.aio 22 | import typing 23 | _T = typing.TypeVar('_T') 24 | 25 | class _MaybeAsyncIterator(collections.abc.AsyncIterator[_T], collections.abc.Iterator[_T], metaclass=abc.ABCMeta): 26 | ... 27 | 28 | class _ServicerContext(grpc.ServicerContext, grpc.aio.ServicerContext): 29 | ... -------------------------------------------------------------------------------- /cmd/redpanda-connect-ai/sqlite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Platforms and architectures list from https://pkg.go.dev/modernc.org/sqlite?utm_source=godoc#hdr-Supported_platforms_and_architectures 16 | // Last updated from modernc.org/sqlite@v1.19.1 17 | //go:build (darwin && (amd64 || arm64)) || (freebsd && (amd64 || arm64)) || (linux && (386 || amd64 || arm || arm64 || riscv64)) || (windows && (amd64 || arm64)) 18 | 19 | package main 20 | 21 | import ( 22 | // Import sqlite specifically. 23 | _ "modernc.org/sqlite" 24 | ) 25 | -------------------------------------------------------------------------------- /cmd/redpanda-connect-cloud/sqlite.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Platforms and architectures list from https://pkg.go.dev/modernc.org/sqlite?utm_source=godoc#hdr-Supported_platforms_and_architectures 16 | // Last updated from modernc.org/sqlite@v1.19.1 17 | //go:build (darwin && (amd64 || arm64)) || (freebsd && (amd64 || arm64)) || (linux && (386 || amd64 || arm || arm64 || riscv64)) || (windows && (amd64 || arm64)) 18 | 19 | package main 20 | 21 | import ( 22 | // Import sqlite specifically. 23 | _ "modernc.org/sqlite" 24 | ) 25 | -------------------------------------------------------------------------------- /docs/modules/components/pages/processors/rate_limit.adoc: -------------------------------------------------------------------------------- 1 | = rate_limit 2 | :type: processor 3 | :status: stable 4 | :categories: ["Utility"] 5 | 6 | 7 | 8 | //// 9 | THIS FILE IS AUTOGENERATED! 10 | 11 | To make changes, edit the corresponding source file under: 12 | 13 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 14 | 15 | And: 16 | 17 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 18 | //// 19 | 20 | // © 2024 Redpanda Data Inc. 21 | 22 | 23 | component_type_dropdown::[] 24 | 25 | 26 | Throttles the throughput of a pipeline according to a specified xref:components:rate_limits/about.adoc[`rate_limit`] resource. Rate limits are shared across components and therefore apply globally to all processing pipelines. 27 | 28 | ```yml 29 | # Config fields, showing default values 30 | label: "" 31 | rate_limit: 32 | resource: "" # No default (required) 33 | ``` 34 | 35 | == Fields 36 | 37 | === `resource` 38 | 39 | The target xref:components:rate_limits/about.adoc[`rate_limit` resource]. 40 | 41 | 42 | *Type*: `string` 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /config/rag/retrieval/cohere_pgvector.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | http_server: 3 | path: /query 4 | allowed_verbs: 5 | - POST 6 | sync_response: 7 | status: "${!this.status.or(200)}" 8 | timeout: 60s 9 | pipeline: 10 | processors: 11 | - try: 12 | - json_schema: 13 | schema: | 14 | { 15 | "$schema": "http://json-schema.org/draft-07/schema#", 16 | "title": "HTTP Request schema", 17 | "type": "object", 18 | "properties": { 19 | "query": { 20 | "type": "string" 21 | } 22 | }, 23 | "required": [ 24 | "query" 25 | ] 26 | } 27 | - cohere_embed: 28 | api_key: ${COHERE_API_KEY} 29 | input_type: search_query 30 | dimensions: 1536 31 | - pgvector: 32 | dsn: ${POSTGRES_DSN} 33 | table: ${POSTGRES_TABLE}_cohere 34 | output: 35 | processors: 36 | - mutation: | 37 | if errored() { 38 | root = {"status": 500, "error": error()} 39 | } 40 | sync_response: {} 41 | -------------------------------------------------------------------------------- /docs/modules/components/pages/processors/decompress.adoc: -------------------------------------------------------------------------------- 1 | = decompress 2 | :type: processor 3 | :status: stable 4 | :categories: ["Parsing"] 5 | 6 | 7 | 8 | //// 9 | THIS FILE IS AUTOGENERATED! 10 | 11 | To make changes, edit the corresponding source file under: 12 | 13 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 14 | 15 | And: 16 | 17 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 18 | //// 19 | 20 | // © 2024 Redpanda Data Inc. 21 | 22 | 23 | component_type_dropdown::[] 24 | 25 | 26 | Decompresses messages according to the selected algorithm. Supported decompression algorithms are: [bzip2 flate gzip lz4 pgzip snappy zlib] 27 | 28 | ```yml 29 | # Config fields, showing default values 30 | label: "" 31 | decompress: 32 | algorithm: "" # No default (required) 33 | ``` 34 | 35 | == Fields 36 | 37 | === `algorithm` 38 | 39 | The decompression algorithm to use. 40 | 41 | 42 | *Type*: `string` 43 | 44 | 45 | Options: 46 | `bzip2` 47 | , `flate` 48 | , `gzip` 49 | , `lz4` 50 | , `pgzip` 51 | , `snappy` 52 | , `zlib` 53 | . 54 | 55 | 56 | -------------------------------------------------------------------------------- /internal/impl/openai/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed as a Redpanda Enterprise file under the Redpanda Community 4 | // License (the "License"); you may not use this file except in compliance with 5 | // the License. You may obtain a copy of the License at 6 | // 7 | // https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md 8 | 9 | package openai 10 | 11 | import ( 12 | "context" 13 | 14 | oai "github.com/sashabaranov/go-openai" 15 | ) 16 | 17 | // A mockable client for unit testing 18 | type client interface { 19 | CreateChatCompletion(ctx context.Context, body oai.ChatCompletionRequest) (oai.ChatCompletionResponse, error) 20 | CreateEmbeddings(ctx context.Context, body oai.EmbeddingRequestConverter) (oai.EmbeddingResponse, error) 21 | CreateSpeech(ctx context.Context, body oai.CreateSpeechRequest) (oai.RawResponse, error) 22 | CreateTranscription(ctx context.Context, body oai.AudioRequest) (oai.AudioResponse, error) 23 | CreateTranslation(ctx context.Context, body oai.AudioRequest) (oai.AudioResponse, error) 24 | CreateImage(ctx context.Context, body oai.ImageRequest) (oai.ImageResponse, error) 25 | } 26 | -------------------------------------------------------------------------------- /config/test/mock_http_proc_path.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = "simon says: " + content()' 4 | - http: 5 | url: http://example.com/foobar 6 | verb: GET 7 | - bloblang: 'root = content().uppercase()' 8 | 9 | tests: 10 | - name: mocks the http proc 11 | mocks: 12 | /pipeline/processors/1: 13 | bloblang: 'root = content().string() + " this is some mock content"' 14 | input_batch: 15 | - content: "hello world" 16 | output_batches: 17 | - - content_equals: "SIMON SAYS: HELLO WORLD THIS IS SOME MOCK CONTENT" 18 | 19 | - name: mocks the http proc and also adds another processor to expose error 20 | mocks: 21 | /pipeline/processors/1: 22 | bloblang: 'root = throw("the processor failed")' 23 | /pipeline/processors/-: 24 | bloblang: | 25 | root.content = content().string() 26 | root.error = error() 27 | input_batch: 28 | - content: "hello world" 29 | output_batches: 30 | - - json_equals: 31 | content: 'SIMON SAYS: HELLO WORLD' 32 | error: 'failed assignment (line 1): the processor failed' 33 | -------------------------------------------------------------------------------- /internal/impl/twitter/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package twitter 16 | 17 | import ( 18 | _ "embed" 19 | 20 | "github.com/redpanda-data/benthos/v4/public/service" 21 | 22 | // bloblang functions are registered in init functions under this package 23 | // so ensure they are loaded first 24 | _ "github.com/redpanda-data/benthos/v4/public/components/pure" 25 | ) 26 | 27 | //go:embed template_search_input.yaml 28 | var searchInputTemplate []byte 29 | 30 | func init() { 31 | service.MustRegisterTemplateYAML(string(searchInputTemplate)) 32 | } 33 | -------------------------------------------------------------------------------- /internal/rpcplugin/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rpcplugin 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "path/filepath" 21 | "time" 22 | ) 23 | 24 | const ( 25 | retryCount = 3 26 | maxStartupTime = 30 * time.Second 27 | ) 28 | 29 | func newUnixSocketAddr() (string, error) { 30 | dir, err := os.MkdirTemp(os.TempDir(), "rpcn_plugin_*") 31 | if err != nil { 32 | return "", fmt.Errorf("unable to create temp dir: %w", err) 33 | } 34 | socketPath := filepath.Join(dir, "plugin.sock") 35 | return "unix:" + socketPath, nil 36 | } 37 | -------------------------------------------------------------------------------- /config/test/mock_http_proc.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = "simon says: " + content()' 4 | - label: get_foobar_api 5 | http: 6 | url: http://example.com/foobar 7 | verb: GET 8 | - bloblang: 'root = content().uppercase()' 9 | 10 | tests: 11 | - name: mocks the http proc 12 | mocks: 13 | get_foobar_api: 14 | bloblang: 'root = content().string() + " this is some mock content"' 15 | input_batch: 16 | - content: "hello world" 17 | output_batches: 18 | - - content_equals: "SIMON SAYS: HELLO WORLD THIS IS SOME MOCK CONTENT" 19 | 20 | - name: mocks the http proc and also adds another processor to expose error 21 | mocks: 22 | get_foobar_api: 23 | bloblang: 'root = throw("the processor failed")' 24 | /pipeline/processors/-: 25 | bloblang: | 26 | root.content = content().string() 27 | root.error = error() 28 | input_batch: 29 | - content: "hello world" 30 | output_batches: 31 | - - json_equals: 32 | content: 'SIMON SAYS: HELLO WORLD' 33 | error: 'failed assignment (line 1): the processor failed' 34 | -------------------------------------------------------------------------------- /config/template_examples/processor_log_and_drop.yaml: -------------------------------------------------------------------------------- 1 | name: log_and_drop 2 | type: processor 3 | categories: [ Utility ] 4 | summary: A common lossy error handling pattern. 5 | description: If a message has failed in a previous processor this one will log the error and the contents of the message and then drop it. This is a common pattern when working with data that isn't considered important. 6 | 7 | fields: [] 8 | 9 | mapping: | 10 | root.catch = [ 11 | { 12 | "log": { 13 | "level": "ERROR", 14 | "fields": { 15 | "content": "${! content() }" 16 | }, 17 | "message": "${! error() }" 18 | } 19 | }, 20 | { 21 | "bloblang": "root = deleted()" 22 | } 23 | ] 24 | 25 | metrics_mapping: | 26 | root = if this.has_suffix("1.dropped") { 27 | this.replace_all("1.dropped", "dropped") 28 | } else { deleted() } 29 | 30 | tests: 31 | - name: No fields 32 | config: {} 33 | expected: 34 | catch: 35 | - log: 36 | level: ERROR 37 | fields: 38 | content: "${! content() }" 39 | message: "${! error() }" 40 | - bloblang: root = deleted() 41 | -------------------------------------------------------------------------------- /docs/modules/components/pages/scanners/decompress.adoc: -------------------------------------------------------------------------------- 1 | = decompress 2 | :type: scanner 3 | :status: stable 4 | 5 | 6 | 7 | //// 8 | THIS FILE IS AUTOGENERATED! 9 | 10 | To make changes, edit the corresponding source file under: 11 | 12 | https://github.com/redpanda-data/connect/tree/main/internal/impl/. 13 | 14 | And: 15 | 16 | https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl 17 | //// 18 | 19 | // © 2024 Redpanda Data Inc. 20 | 21 | 22 | component_type_dropdown::[] 23 | 24 | 25 | Decompress the stream of bytes according to an algorithm, before feeding it into a child scanner. 26 | 27 | ```yml 28 | # Config fields, showing default values 29 | decompress: 30 | algorithm: "" # No default (required) 31 | into: 32 | to_the_end: {} 33 | ``` 34 | 35 | == Fields 36 | 37 | === `algorithm` 38 | 39 | One of `gzip`, `pgzip`, `zlib`, `bzip2`, `flate`, `snappy`, `lz4`, `zstd`. 40 | 41 | 42 | *Type*: `string` 43 | 44 | 45 | === `into` 46 | 47 | The child scanner to feed the decompressed stream into. 48 | 49 | 50 | *Type*: `scanner` 51 | 52 | *Default*: `{"to_the_end":{}}` 53 | 54 | 55 | -------------------------------------------------------------------------------- /config/template_examples/input_stdin_uppercase.yaml: -------------------------------------------------------------------------------- 1 | name: stdin_uppercase 2 | type: input 3 | status: experimental 4 | categories: [ Pointless ] 5 | summary: Reads messages from stdin but uppercases everything for some reason. 6 | 7 | mapping: | 8 | root.stdin = {} 9 | root.processors = [] 10 | root.processors."-".bloblang = """ 11 | root = content().uppercase().string() 12 | """.trim() 13 | 14 | metrics_mapping: | 15 | map decrement_processor { 16 | let start_index = this.index_of("processor") 17 | let prefix = this.slice(0, $start_index) 18 | let suffix = this.slice($start_index) 19 | 20 | let index = $suffix.split(".").1.number().floor() 21 | 22 | root = $prefix + if $index == 0 { 23 | $suffix.replace_all("processor.0.", "mapping.") 24 | } else { 25 | $suffix.re_replace_all("processor\\.[0-9]+\\.", "processor.%v.".format($index - 1)) 26 | } 27 | } 28 | 29 | root = if this.contains("processor") { 30 | this.apply("decrement_processor") 31 | } 32 | 33 | tests: 34 | - name: no fields 35 | config: {} 36 | expected: 37 | stdin: {} 38 | processors: 39 | - bloblang: "root = content().uppercase().string()" 40 | -------------------------------------------------------------------------------- /config/test/deduplicate_by_batch.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - mapping: | 4 | meta batch_tag = if batch_index() == 0 { 5 | nanoid(10) 6 | } 7 | - dedupe: 8 | cache: local 9 | key: ${! meta("batch_tag").from(0) + content() } 10 | 11 | cache_resources: 12 | - label: local 13 | memory: 14 | default_ttl: 1m 15 | 16 | tests: 17 | - name: de-duplicate by batches 18 | input_batches: 19 | - 20 | - content: '1' 21 | - content: '2' 22 | - content: '3' 23 | - content: '4' 24 | - content: '3' 25 | - content: '3' 26 | - content: '3' 27 | - 28 | - content: '4' 29 | - content: '1' 30 | - content: '1' 31 | - content: '3' 32 | - content: '4' 33 | - content: '4' 34 | - content: '2' 35 | - content: '1' 36 | output_batches: 37 | - 38 | - content_equals: 1 39 | - content_equals: 2 40 | - content_equals: 3 41 | - content_equals: 4 42 | - 43 | - content_equals: 4 44 | - content_equals: 1 45 | - content_equals: 3 46 | - content_equals: 2 47 | --------------------------------------------------------------------------------