├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── release_checklist.md ├── PULL_REQUEST_TEMPLATE │ ├── pull_request_template.md │ └── release_template.md ├── actions │ ├── dl-smdk │ │ └── action.yml │ └── setup-fluvio │ │ └── action.yml ├── assets │ ├── fluvio-event-streaming.gif │ └── fluvio-overview.svg ├── dependabot.yml └── workflows │ ├── benchmarks.yml │ ├── cd_dev.yaml │ ├── cd_dev_mac.yaml │ ├── cd_release.yml │ ├── ci.yml │ ├── ci_mac.yaml │ ├── ci_unused_deps.yml │ ├── connector-publish.yml │ ├── hourly.yml │ ├── lint_pr.yml │ ├── merge_queue.yaml │ ├── publish.yml │ ├── publish_crates.yml │ ├── release.yml │ ├── smartmodule-publish.yml │ ├── stale.yml │ └── weekly.yml ├── .gitignore ├── CHANGELOG.md ├── CODE-OF-CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── DEVELOPER.md ├── LICENSE ├── Makefile ├── README.md ├── RELEASE.md ├── VERSION ├── action.yaml ├── actions ├── action-install-fluvio-cluster.sh ├── action-install-minikube.sh ├── ci-replace-helm.sh ├── post-release-changelog.sh ├── publish-pkgset.sh ├── sccache-terminate.sh ├── upload-bpkg.sh ├── view-changelog.sh ├── zig-install.sh └── zig-uninstall.sh ├── build-scripts ├── aarch64-linux-musl-zig-cc ├── aarch64-macos-gnu-zig-cc ├── arm-linux-gnu-zig-cc ├── install_target.sh └── x86_64-linux-musl-zig-cc ├── cliff.toml ├── connector ├── cargo_template │ ├── .gitignore │ ├── Cargo.toml.liquid │ ├── Connector.toml │ ├── README.md │ ├── cargo-generate.toml │ ├── sample-config.yaml │ └── src │ │ ├── config.rs │ │ └── main.rs ├── docker │ ├── .gitignore │ ├── Dockerfile │ ├── Makefile │ ├── README.md │ └── connector.yaml ├── json-test-connector │ ├── Cargo.toml │ ├── Connector.toml │ ├── README.md │ ├── sample-config-v2.yaml │ ├── sample-config.yaml │ ├── sample-config2-v2.yaml │ └── src │ │ ├── main.rs │ │ └── source.rs └── sink-test-connector │ ├── Cargo.toml │ ├── Connector.toml │ ├── README.md │ ├── sample-config-v2.yaml │ ├── sample-config.yaml │ ├── secrets.txt │ └── src │ ├── main.rs │ └── sink.rs ├── crates ├── cargo-builder │ ├── Cargo.toml │ └── src │ │ ├── cargo.rs │ │ ├── lib.rs │ │ └── package.rs ├── cdk │ ├── Cargo.toml │ ├── build.rs │ ├── src │ │ ├── build.rs │ │ ├── cmd.rs │ │ ├── deploy.rs │ │ ├── generate.rs │ │ ├── hub.rs │ │ ├── main.rs │ │ ├── publish.rs │ │ ├── set_public.rs │ │ ├── test.rs │ │ └── utils.rs │ └── tests │ │ └── foo-0.1.0.ipkg ├── fluvio-auth │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ └── src │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── policy.rs │ │ ├── root.rs │ │ └── x509 │ │ ├── authenticator.rs │ │ ├── identity.rs │ │ ├── mod.rs │ │ └── request.rs ├── fluvio-benchmark │ ├── Cargo.toml │ ├── Makefile │ └── src │ │ ├── benchmark_driver.rs │ │ ├── bin │ │ └── fluvio-benchmark.rs │ │ ├── cli.rs │ │ ├── config │ │ ├── config_matrix.rs │ │ ├── cross.rs │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── producer_benchmark.rs │ │ ├── producer_worker.rs │ │ ├── stats_collector.rs │ │ └── utils.rs ├── fluvio-channel-cli │ ├── Cargo.toml │ ├── src │ │ ├── bin │ │ │ └── main.rs │ │ ├── cli │ │ │ ├── create.rs │ │ │ ├── delete.rs │ │ │ ├── list.rs │ │ │ ├── mod.rs │ │ │ └── switch.rs │ │ └── lib.rs │ └── tests │ │ ├── issue_2168_regression.rs │ │ └── pinned_version_channel.rs ├── fluvio-channel │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── fluvio-cli-common │ ├── Cargo.toml │ ├── build.rs │ └── src │ │ ├── error.rs │ │ ├── http.rs │ │ ├── install.rs │ │ ├── lib.rs │ │ ├── smartmodule.rs │ │ ├── user_input.rs │ │ └── version_cmd │ │ ├── basic.rs │ │ └── mod.rs ├── fluvio-cli │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── build.rs │ ├── src │ │ ├── bin │ │ │ └── main.rs │ │ ├── client │ │ │ ├── consume │ │ │ │ ├── mod.rs │ │ │ │ ├── record_format.rs │ │ │ │ └── table_format.rs │ │ │ ├── consumer │ │ │ │ ├── delete.rs │ │ │ │ ├── list.rs │ │ │ │ └── mod.rs │ │ │ ├── home │ │ │ │ ├── connect.rs │ │ │ │ ├── mod.rs │ │ │ │ └── status.rs │ │ │ ├── hub │ │ │ │ ├── connector │ │ │ │ │ └── mod.rs │ │ │ │ ├── mod.rs │ │ │ │ └── smartmodule │ │ │ │ │ ├── download.rs │ │ │ │ │ ├── list.rs │ │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── partition │ │ │ │ ├── list.rs │ │ │ │ └── mod.rs │ │ │ ├── produce │ │ │ │ ├── mod.rs │ │ │ │ └── stats │ │ │ │ │ ├── mod.rs │ │ │ │ │ └── report.rs │ │ │ ├── remote │ │ │ │ ├── export.rs │ │ │ │ ├── list.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── register.rs │ │ │ │ └── unregister.rs │ │ │ ├── smartmodule │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── list.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── test.rs │ │ │ │ └── watch.rs │ │ │ ├── smartmodule_invocation.rs │ │ │ ├── tableformat │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── list.rs │ │ │ │ └── mod.rs │ │ │ └── topic │ │ │ │ ├── add_mirror.rs │ │ │ │ ├── add_partition.rs │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── describe.rs │ │ │ │ ├── list.rs │ │ │ │ └── mod.rs │ │ ├── error.rs │ │ ├── install │ │ │ ├── mod.rs │ │ │ ├── opts.rs │ │ │ └── update.rs │ │ ├── lib.rs │ │ ├── metadata.rs │ │ ├── monitoring.rs │ │ ├── profile │ │ │ ├── add.rs │ │ │ ├── current.rs │ │ │ ├── delete_cluster.rs │ │ │ ├── delete_profile.rs │ │ │ ├── export.rs │ │ │ ├── list.rs │ │ │ ├── mod.rs │ │ │ ├── rename.rs │ │ │ ├── switch.rs │ │ │ └── sync │ │ │ │ ├── k8.rs │ │ │ │ ├── local.rs │ │ │ │ └── mod.rs │ │ ├── render.rs │ │ └── version.rs │ └── test-data │ │ ├── connectors │ │ ├── error-batchsize.yaml │ │ ├── error-compression.yaml │ │ ├── error-linger.yaml │ │ ├── error-version.yaml │ │ ├── full-config.yaml │ │ └── simple.yaml │ │ ├── derivedstream │ │ ├── left.yaml │ │ ├── right.yaml │ │ └── simple_join.yaml │ │ ├── group-cfg │ │ └── valid_config.json │ │ ├── input-files │ │ ├── icon1.png │ │ ├── icon2.png │ │ ├── json-file.json │ │ └── text-file.txt │ │ ├── test-config.yaml │ │ ├── test-tableformat-config.yaml │ │ └── topics │ │ ├── mirror_assignment.json │ │ ├── out_of_order.json │ │ ├── replica_assignment.json │ │ └── valid_assignment1.json ├── fluvio-cluster │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── Makefile │ ├── build.rs │ ├── src │ │ ├── charts │ │ │ ├── chart.rs │ │ │ ├── location.rs │ │ │ └── mod.rs │ │ ├── check │ │ │ ├── mod.rs │ │ │ └── render.rs │ │ ├── cli │ │ │ ├── check.rs │ │ │ ├── delete.rs │ │ │ ├── diagnostics.rs │ │ │ ├── error.rs │ │ │ ├── group │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── list.rs │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── resume.rs │ │ │ ├── shutdown.rs │ │ │ ├── spu │ │ │ │ ├── display.rs │ │ │ │ ├── list.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── register.rs │ │ │ │ └── unregister.rs │ │ │ ├── start │ │ │ │ ├── k8.rs │ │ │ │ ├── local.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── sys.rs │ │ │ │ └── tls.rs │ │ │ ├── status.rs │ │ │ ├── upgrade.rs │ │ │ └── util.rs │ │ ├── delete.rs │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── process │ │ │ └── mod.rs │ │ ├── progress.rs │ │ ├── render.rs │ │ ├── runtime │ │ │ ├── local │ │ │ │ ├── mod.rs │ │ │ │ ├── sc.rs │ │ │ │ └── spu.rs │ │ │ └── mod.rs │ │ └── start │ │ │ ├── common.rs │ │ │ ├── k8.rs │ │ │ ├── local.rs │ │ │ └── mod.rs │ └── test-data │ │ └── metadata.json ├── fluvio-compression │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ └── src │ │ ├── error.rs │ │ ├── gzip.rs │ │ ├── lib.rs │ │ ├── lz4.rs │ │ ├── snappy.rs │ │ └── zstd.rs ├── fluvio-connector-common │ ├── Cargo.toml │ ├── src │ │ ├── config.rs │ │ ├── consumer.rs │ │ ├── lib.rs │ │ ├── monitoring.rs │ │ ├── producer.rs │ │ └── smartmodule.rs │ └── ui-test │ │ ├── mod.rs │ │ └── ui │ │ ├── config_input_self.rs │ │ ├── config_input_self.stderr │ │ ├── config_use_reserved_name_fluvio.rs │ │ ├── config_use_reserved_name_fluvio.stderr │ │ ├── config_use_reserved_name_transforms.rs │ │ ├── config_use_reserved_name_transforms.stderr │ │ ├── invalid_config_type.rs │ │ ├── invalid_config_type.stderr │ │ ├── not_async.rs │ │ ├── not_async.stderr │ │ ├── struct_without_config.rs │ │ ├── struct_without_config.stderr │ │ ├── wrong_direction.rs │ │ ├── wrong_direction.stderr │ │ ├── wrong_input_len.rs │ │ ├── wrong_input_len.stderr │ │ ├── wrong_rust_type.rs │ │ └── wrong_rust_type.stderr ├── fluvio-connector-deployer │ ├── Cargo.toml │ └── src │ │ ├── lib.rs │ │ └── local.rs ├── fluvio-connector-derive │ ├── Cargo.toml │ └── src │ │ ├── ast.rs │ │ ├── generator.rs │ │ └── lib.rs ├── fluvio-connector-package │ ├── Cargo.toml │ ├── src │ │ ├── config │ │ │ ├── bytesize_serde.rs │ │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── metadata.rs │ │ ├── render │ │ │ ├── context.rs │ │ │ ├── mod.rs │ │ │ └── syntax.rs │ │ └── secret.rs │ ├── test-data │ │ └── connectors │ │ │ ├── all-partitions.yaml │ │ │ ├── error-batchsize.yaml │ │ │ ├── error-compression.yaml │ │ │ ├── error-invalid-api-version.yaml │ │ │ ├── error-linger.yaml │ │ │ ├── error-secret-starts-with-number.yaml │ │ │ ├── error-secret-with-spaces.yaml │ │ │ ├── error-version.yaml │ │ │ ├── full-config-v2.yaml │ │ │ ├── full-config.yaml │ │ │ ├── many-partitions.yaml │ │ │ ├── simple.yaml │ │ │ ├── with_transform.yaml │ │ │ ├── with_transform_invalid.yaml │ │ │ └── with_transform_many.yaml │ └── tests │ │ ├── Connector.toml │ │ ├── mod.rs │ │ ├── sample-http.yaml │ │ └── sample-mqtt.yaml ├── fluvio-controlplane-metadata │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── src │ │ ├── lib.rs │ │ ├── message │ │ │ ├── messages.rs │ │ │ ├── mod.rs │ │ │ └── msg_type.rs │ │ ├── mirror │ │ │ ├── k8.rs │ │ │ ├── mod.rs │ │ │ ├── spec.rs │ │ │ └── status.rs │ │ ├── mirroring │ │ │ └── mod.rs │ │ ├── partition │ │ │ ├── k8.rs │ │ │ ├── mod.rs │ │ │ ├── spec.rs │ │ │ └── status.rs │ │ ├── smartmodule │ │ │ ├── k8.rs │ │ │ ├── mod.rs │ │ │ ├── package.rs │ │ │ ├── params.rs │ │ │ ├── spec.rs │ │ │ ├── spec_v1.rs │ │ │ └── status.rs │ │ ├── spg │ │ │ ├── k8 │ │ │ │ ├── mod.rs │ │ │ │ └── spec.rs │ │ │ ├── mod.rs │ │ │ ├── spec.rs │ │ │ ├── status.rs │ │ │ └── store.rs │ │ ├── spu │ │ │ ├── k8.rs │ │ │ ├── mod.rs │ │ │ ├── spec.rs │ │ │ └── status.rs │ │ ├── tableformat │ │ │ ├── k8 │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── spec.rs │ │ │ └── status.rs │ │ └── topic │ │ │ ├── config.rs │ │ │ ├── deduplication.rs │ │ │ ├── k8.rs │ │ │ ├── mod.rs │ │ │ ├── spec.rs │ │ │ ├── status.rs │ │ │ └── update.rs │ └── tests │ │ ├── SmartModule.toml │ │ ├── SmartModule.yaml │ │ ├── k8_mirror_v1.json │ │ ├── k8_mirror_v1.yaml │ │ ├── k8_partition_mirror_v1.json │ │ ├── k8_topic_mirror_down_v1.json │ │ ├── k8_topic_mirror_down_v2.json │ │ ├── sm_k8_v1.yaml │ │ ├── sm_k8_v2.json │ │ ├── sm_k8_v2.yaml │ │ ├── topic_assignment.json │ │ └── topic_computed.yaml ├── fluvio-controlplane │ ├── Cargo.toml │ ├── LICENSE-APACHE │ └── src │ │ ├── lib.rs │ │ ├── message │ │ ├── mod.rs │ │ └── replica_msg.rs │ │ ├── replica.rs │ │ ├── requests │ │ ├── mod.rs │ │ └── request.rs │ │ ├── sc_api │ │ ├── api.rs │ │ ├── mod.rs │ │ ├── register_spu.rs │ │ ├── remove.rs │ │ ├── update_lrs.rs │ │ ├── update_mirror.rs │ │ └── update_partition.rs │ │ └── spu_api │ │ ├── api.rs │ │ ├── mod.rs │ │ ├── update_mirror.rs │ │ ├── update_replica.rs │ │ ├── update_smartmodule.rs │ │ └── update_spu.rs ├── fluvio-extension-common │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ └── src │ │ ├── common │ │ ├── hex_dump.rs │ │ ├── mod.rs │ │ └── time.rs │ │ ├── installation.rs │ │ ├── lib.rs │ │ ├── output │ │ ├── describe.rs │ │ ├── mod.rs │ │ ├── serde.rs │ │ └── table.rs │ │ └── tls.rs ├── fluvio-hub-protocol │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── constants.rs │ │ ├── errors.rs │ │ ├── infinyon_tok.rs │ │ ├── lib.rs │ │ └── package_meta.rs ├── fluvio-hub-util │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── cmd │ │ │ ├── download.rs │ │ │ ├── list.rs │ │ │ └── mod.rs │ │ ├── fvm │ │ │ ├── api │ │ │ │ ├── client.rs │ │ │ │ ├── download.rs │ │ │ │ └── mod.rs │ │ │ └── mod.rs │ │ ├── htclient.rs │ │ ├── hubaccess.rs │ │ ├── keymgmt.rs │ │ ├── lib.rs │ │ ├── package.rs │ │ ├── package_meta_ext.rs │ │ └── utils.rs │ └── tests │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── apackage │ │ ├── module.wasm │ │ ├── package-meta-manifest-missing-file.yaml │ │ ├── package-meta-v0.1.yaml │ │ ├── package-meta-v0.2-owner.yaml │ │ ├── package-meta-v0.2-public.yaml │ │ ├── package-meta-v0.2-visbad.yaml │ │ ├── package-meta-v0.3-notags.yaml │ │ ├── package-meta-v0.3-notags2.yaml │ │ ├── package-meta-v0.3-targets.yaml │ │ └── package-meta.yaml │ │ ├── hub_package_meta_rw_test.yaml │ │ ├── id_ed25519 │ │ ├── id_ed25519.pub │ │ ├── sm_need_more_data.wasm │ │ ├── sm_ok.wasm │ │ ├── static-example-0.0.1.ipkg │ │ ├── static-example-keypair.pem │ │ └── static-example-pubkey.pem ├── fluvio-kv-storage │ ├── Cargo.toml │ └── src │ │ ├── lib.rs │ │ └── log.rs ├── fluvio-package-index │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── build.rs │ └── src │ │ ├── error.rs │ │ ├── http.rs │ │ ├── lib.rs │ │ ├── package.rs │ │ ├── package_id.rs │ │ ├── tags.rs │ │ ├── target.rs │ │ └── version.rs ├── fluvio-protocol-derive │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── rustfmt.toml │ └── src │ │ ├── api.rs │ │ ├── ast │ │ ├── container.rs │ │ ├── enum.rs │ │ ├── mod.rs │ │ ├── prop.rs │ │ └── struct.rs │ │ ├── de.rs │ │ ├── lib.rs │ │ ├── ser.rs │ │ └── util.rs ├── fluvio-protocol │ ├── CODE-OF-CONDUCT.md │ ├── CONTRIBUTING.md │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── Makefile │ ├── README.md │ ├── api-test │ │ └── api.rs │ ├── benches │ │ └── bench.rs │ ├── derive-test │ │ ├── mod.rs │ │ └── ui │ │ │ ├── fail │ │ │ ├── fails_missing_encode_discriminant_and_fluvio_tag_not_provided.rs │ │ │ ├── fails_missing_encode_discriminant_and_fluvio_tag_not_provided.stderr │ │ │ ├── fails_missing_fluvio_annotation_and_discriminant_on_enums.rs │ │ │ ├── fails_missing_fluvio_annotation_and_discriminant_on_enums.stderr │ │ │ ├── fails_missing_fluvio_annotation_with_tag_on_enums.rs │ │ │ └── fails_missing_fluvio_annotation_with_tag_on_enums.stderr │ │ │ └── pass │ │ │ ├── works_with_discriminant_on_enums.rs │ │ │ ├── works_with_fluvio_tag_and_discriminant_on_enums.rs │ │ │ └── works_with_fluvio_tag_on_enums.rs │ ├── fixtures │ │ └── smartmodule.wasm │ ├── send-b-client.sh │ ├── src │ │ ├── api │ │ │ ├── mod.rs │ │ │ ├── request.rs │ │ │ └── response.rs │ │ ├── codec │ │ │ └── mod.rs │ │ ├── core │ │ │ ├── bytebuf.rs │ │ │ ├── decoder.rs │ │ │ ├── encoder.rs │ │ │ ├── mod.rs │ │ │ ├── varint.rs │ │ │ └── zerocopy.rs │ │ ├── fixture.rs │ │ ├── lib.rs │ │ ├── link │ │ │ ├── error_code.rs │ │ │ ├── mod.rs │ │ │ ├── smartmodule.rs │ │ │ └── versions.rs │ │ ├── record │ │ │ ├── batch.rs │ │ │ ├── data.rs │ │ │ ├── mod.rs │ │ │ └── replica.rs │ │ └── store.rs │ ├── test-client.sh │ ├── tests │ │ ├── btreemap.rs │ │ ├── decode.rs │ │ ├── default.rs │ │ ├── encode.rs │ │ ├── enum.rs │ │ ├── generic.rs │ │ ├── option.rs │ │ ├── parse.rs │ │ ├── test_old_record_empty_key.bin │ │ ├── tuple_struct.rs │ │ └── version.rs │ └── ui-tests │ │ ├── pass_derive_decode.rs │ │ ├── pass_derive_decode_encode_with_custom_results.rs │ │ └── pass_derive_encode.rs ├── fluvio-run │ ├── Cargo.toml │ ├── build.rs │ └── src │ │ ├── bin │ │ └── main.rs │ │ ├── error.rs │ │ └── lib.rs ├── fluvio-sc-schema │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ └── src │ │ ├── apis.rs │ │ ├── customspu │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── mirror │ │ └── mod.rs │ │ ├── mirroring │ │ └── mod.rs │ │ ├── objects │ │ ├── classic.rs │ │ ├── create.rs │ │ ├── delete.rs │ │ ├── list.rs │ │ ├── metadata.rs │ │ ├── mod.rs │ │ ├── test.rs │ │ ├── update.rs │ │ └── watch.rs │ │ ├── partition │ │ └── mod.rs │ │ ├── remote_file.rs │ │ ├── request.rs │ │ ├── response.rs │ │ ├── shared.rs │ │ ├── smartmodule │ │ ├── client.rs │ │ └── mod.rs │ │ ├── spg │ │ └── mod.rs │ │ ├── spu │ │ └── mod.rs │ │ ├── tableformat │ │ └── mod.rs │ │ ├── topic │ │ └── mod.rs │ │ └── versions.rs ├── fluvio-sc │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── build.rs │ ├── src │ │ ├── bin │ │ │ └── main.rs │ │ ├── cli.rs │ │ ├── config │ │ │ ├── mod.rs │ │ │ └── sc_config.rs │ │ ├── controllers │ │ │ ├── mirroring │ │ │ │ ├── controller.rs │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── partitions │ │ │ │ ├── Test.MD │ │ │ │ ├── controller.rs │ │ │ │ ├── mod.rs │ │ │ │ └── reducer.rs │ │ │ ├── scheduler │ │ │ │ ├── mod.rs │ │ │ │ └── partition.rs │ │ │ ├── spus │ │ │ │ ├── actions.rs │ │ │ │ ├── controller.rs │ │ │ │ └── mod.rs │ │ │ └── topics │ │ │ │ ├── actions.rs │ │ │ │ ├── controller.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── policy.rs │ │ │ │ └── reducer.rs │ │ ├── core │ │ │ ├── context.rs │ │ │ └── mod.rs │ │ ├── error.rs │ │ ├── init.rs │ │ ├── k8 │ │ │ ├── controllers │ │ │ │ ├── mod.rs │ │ │ │ ├── spg_stateful.rs │ │ │ │ ├── spu_controller.rs │ │ │ │ └── spu_service.rs │ │ │ ├── fixture.rs │ │ │ ├── mod.rs │ │ │ └── objects │ │ │ │ ├── mod.rs │ │ │ │ ├── spg_group.rs │ │ │ │ ├── spg_service.rs │ │ │ │ ├── spu_k8_config.rs │ │ │ │ ├── spu_service.rs │ │ │ │ └── statefulset.rs │ │ ├── lib.rs │ │ ├── services │ │ │ ├── auth │ │ │ │ ├── basic.rs │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── private_api │ │ │ │ ├── mod.rs │ │ │ │ └── private_server.rs │ │ │ └── public_api │ │ │ │ ├── api_version.rs │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── derivedstream │ │ │ │ ├── fetch.rs │ │ │ │ └── mod.rs │ │ │ │ ├── list.rs │ │ │ │ ├── mirror │ │ │ │ ├── list.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── register.rs │ │ │ │ └── unregister.rs │ │ │ │ ├── mirroring │ │ │ │ ├── connect.rs │ │ │ │ └── mod.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── partition │ │ │ │ └── mod.rs │ │ │ │ ├── public_server.rs │ │ │ │ ├── smartmodule │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── list.rs │ │ │ │ └── mod.rs │ │ │ │ ├── spg │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── fetch.rs │ │ │ │ └── mod.rs │ │ │ │ ├── spu │ │ │ │ ├── fetch.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── register_custom_spus_req.rs │ │ │ │ └── unregister_custom_spus_req.rs │ │ │ │ ├── tableformat │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ └── mod.rs │ │ │ │ ├── topic │ │ │ │ ├── create.rs │ │ │ │ ├── delete.rs │ │ │ │ ├── fetch.rs │ │ │ │ ├── mod.rs │ │ │ │ └── update │ │ │ │ │ ├── add_mirror.rs │ │ │ │ │ ├── add_partition.rs │ │ │ │ │ └── mod.rs │ │ │ │ ├── update.rs │ │ │ │ └── watch.rs │ │ ├── start.rs │ │ ├── stores │ │ │ ├── mod.rs │ │ │ ├── partition │ │ │ │ ├── mod.rs │ │ │ │ ├── policy.rs │ │ │ │ └── store.rs │ │ │ ├── smartmodule.rs │ │ │ ├── spg.rs │ │ │ ├── spu │ │ │ │ ├── mod.rs │ │ │ │ └── store.rs │ │ │ ├── tableformat.rs │ │ │ └── topic │ │ │ │ ├── mod.rs │ │ │ │ └── store.rs │ │ └── tests │ │ │ └── TESTS.md │ └── test-data │ │ ├── auth_config │ │ ├── policy-instance.json │ │ ├── policy.json │ │ └── scopes.json │ │ └── config │ │ ├── sc_invalid.toml │ │ ├── sc_min.toml │ │ ├── sc_server.toml │ │ └── sc_tls.toml ├── fluvio-service │ ├── Cargo.toml │ ├── LICENSE-APACHE │ └── src │ │ ├── lib.rs │ │ ├── server.rs │ │ └── test_request.rs ├── fluvio-smartengine │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── src │ │ ├── engine │ │ │ ├── config.rs │ │ │ ├── error.rs │ │ │ ├── fixture.rs │ │ │ ├── metrics.rs │ │ │ ├── mod.rs │ │ │ └── wasmtime │ │ │ │ ├── engine.rs │ │ │ │ ├── init.rs │ │ │ │ ├── instance.rs │ │ │ │ ├── limiter.rs │ │ │ │ ├── look_back.rs │ │ │ │ ├── memory.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── state.rs │ │ │ │ └── transforms │ │ │ │ ├── aggregate.rs │ │ │ │ ├── array_map.rs │ │ │ │ ├── filter.rs │ │ │ │ ├── filter_map.rs │ │ │ │ ├── map.rs │ │ │ │ ├── mod.rs │ │ │ │ └── simple_transform.rs │ │ ├── lib.rs │ │ └── transformation.rs │ └── testdata │ │ └── transformation │ │ ├── empty.yaml │ │ └── full.yaml ├── fluvio-smartmodule-derive │ ├── Cargo.toml │ └── src │ │ ├── ast.rs │ │ ├── generator │ │ ├── aggregate.rs │ │ ├── array_map.rs │ │ ├── filter.rs │ │ ├── filter_map.rs │ │ ├── init.rs │ │ ├── look_back.rs │ │ ├── map.rs │ │ ├── mod.rs │ │ ├── opt.rs │ │ └── transform.rs │ │ ├── lib.rs │ │ └── util.rs ├── fluvio-smartmodule │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── benches │ │ └── bench.rs │ ├── fixtures │ │ └── smartmodule.wasm │ ├── src │ │ ├── error.rs │ │ ├── input.rs │ │ ├── lib.rs │ │ ├── memory.rs │ │ └── output.rs │ └── tests │ │ ├── opt.rs │ │ └── parse.rs ├── fluvio-socket │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── build.rs │ ├── certs │ │ ├── Makefile │ │ ├── cert.conf │ │ ├── certs │ │ │ ├── ca.crt │ │ │ ├── ca.key │ │ │ ├── ca.srl │ │ │ ├── client.crt │ │ │ ├── client.csr │ │ │ ├── client.key │ │ │ ├── client.pfx │ │ │ ├── server.crt │ │ │ ├── server.csr │ │ │ ├── server.key │ │ │ └── server.pfx │ │ ├── nginx.conf │ │ ├── openssl-ca.cnf │ │ └── openssl.cnf │ ├── src │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── multiplexing.rs │ │ ├── sink.rs │ │ ├── socket.rs │ │ ├── stream.rs │ │ ├── stream_socket.rs │ │ ├── test_request.rs │ │ └── versioned.rs │ └── tests │ │ ├── enum.rs │ │ └── test.txt ├── fluvio-spu-schema │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── src │ │ ├── client │ │ │ ├── api.rs │ │ │ ├── api_key.rs │ │ │ ├── mod.rs │ │ │ └── offset.rs │ │ ├── fetch │ │ │ ├── mod.rs │ │ │ ├── request.rs │ │ │ └── response.rs │ │ ├── file.rs │ │ ├── isolation.rs │ │ ├── lib.rs │ │ ├── produce │ │ │ ├── mod.rs │ │ │ ├── request.rs │ │ │ └── response.rs │ │ └── server │ │ │ ├── api.rs │ │ │ ├── api_key.rs │ │ │ ├── consumer.rs │ │ │ ├── consumer_offset.rs │ │ │ ├── fetch_offset.rs │ │ │ ├── mirror.rs │ │ │ ├── mod.rs │ │ │ ├── smartmodule.rs │ │ │ ├── stream_fetch.rs │ │ │ └── update_offset.rs │ └── tests │ │ ├── file_fetch.rs │ │ ├── produce.rs │ │ ├── produce_request.bin │ │ └── test_old_record_empty_key.bin ├── fluvio-spu │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── Makefile │ ├── README.md │ ├── build.rs │ └── src │ │ ├── config │ │ ├── cli.rs │ │ ├── mod.rs │ │ └── spu_config.rs │ │ ├── control_plane │ │ ├── action.rs │ │ ├── dispatcher.rs │ │ ├── message_sink.rs │ │ └── mod.rs │ │ ├── core │ │ ├── global_context.rs │ │ ├── leader_client.rs │ │ ├── metrics.rs │ │ ├── mirror.rs │ │ ├── mod.rs │ │ ├── replica │ │ │ ├── metadata.rs │ │ │ └── mod.rs │ │ ├── smartmodule │ │ │ ├── metadata.rs │ │ │ └── mod.rs │ │ ├── spus │ │ │ ├── metadata.rs │ │ │ └── mod.rs │ │ ├── storage │ │ │ └── mod.rs │ │ └── store.rs │ │ ├── error.rs │ │ ├── kv │ │ ├── consumer.rs │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── main.rs │ │ ├── mirroring │ │ ├── home │ │ │ ├── api_key.rs │ │ │ ├── connection.rs │ │ │ ├── home_api.rs │ │ │ ├── mod.rs │ │ │ ├── sync.rs │ │ │ └── update_offsets.rs │ │ ├── mod.rs │ │ ├── remote │ │ │ ├── api_key.rs │ │ │ ├── controller.rs │ │ │ ├── mod.rs │ │ │ ├── remote_api.rs │ │ │ ├── sync.rs │ │ │ └── update_offsets.rs │ │ └── test │ │ │ ├── fixture.rs │ │ │ ├── integration.rs │ │ │ └── mod.rs │ │ ├── monitoring.rs │ │ ├── replication │ │ ├── follower │ │ │ ├── api_key.rs │ │ │ ├── controller.rs │ │ │ ├── mod.rs │ │ │ ├── peer_api.rs │ │ │ ├── reject_request.rs │ │ │ ├── state.rs │ │ │ └── sync.rs │ │ ├── leader │ │ │ ├── actions.rs │ │ │ ├── api_key.rs │ │ │ ├── connection.rs │ │ │ ├── kv.rs │ │ │ ├── leaders_state.rs │ │ │ ├── mod.rs │ │ │ ├── peer_api.rs │ │ │ ├── replica_state.rs │ │ │ ├── spu.rs │ │ │ └── update_offsets.rs │ │ ├── mod.rs │ │ └── test.rs │ │ ├── services │ │ ├── auth │ │ │ └── mod.rs │ │ ├── internal │ │ │ ├── api.rs │ │ │ ├── fetch_consumer_offset_handler.rs │ │ │ ├── fetch_consumer_offset_request.rs │ │ │ ├── fetch_stream_request.rs │ │ │ ├── mod.rs │ │ │ ├── service_impl.rs │ │ │ ├── update_consumer_offset_handler.rs │ │ │ └── update_consumer_offset_request.rs │ │ ├── mod.rs │ │ └── public │ │ │ ├── api_versions.rs │ │ │ ├── conn_context.rs │ │ │ ├── consumer_handler.rs │ │ │ ├── fetch_handler.rs │ │ │ ├── mod.rs │ │ │ ├── offset_request.rs │ │ │ ├── offset_update.rs │ │ │ ├── produce_handler.rs │ │ │ ├── stream_fetch.rs │ │ │ └── tests │ │ │ ├── consumer_offset.rs │ │ │ ├── mod.rs │ │ │ ├── produce.rs │ │ │ ├── stream_fetch.rs │ │ │ └── test_data │ │ │ └── filter_missing_attribute.wasm │ │ ├── smartengine │ │ ├── batch.rs │ │ ├── chain.rs │ │ ├── context.rs │ │ ├── file_batch.rs │ │ ├── mod.rs │ │ └── produce_batch.rs │ │ ├── start.rs │ │ └── storage │ │ └── mod.rs ├── fluvio-storage │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── Makefile │ ├── README.md │ ├── src │ │ ├── batch.rs │ │ ├── batch_header.rs │ │ ├── bin │ │ │ └── cli.rs │ │ ├── checkpoint.rs │ │ ├── cleaner.rs │ │ ├── config.rs │ │ ├── error.rs │ │ ├── file.rs │ │ ├── fixture.rs │ │ ├── index.rs │ │ ├── iterators.rs │ │ ├── lib.rs │ │ ├── mut_index.rs │ │ ├── mut_records.rs │ │ ├── records.rs │ │ ├── replica.rs │ │ ├── segment.rs │ │ ├── segments.rs │ │ ├── util.rs │ │ └── validator.rs │ └── tests │ │ └── replica_test.rs ├── fluvio-stream-dispatcher │ ├── Cargo.toml │ ├── LICENSE-APACHE │ └── src │ │ ├── actions.rs │ │ ├── dispatcher │ │ ├── metadata.rs │ │ └── mod.rs │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── metadata │ │ ├── k8.rs │ │ ├── local.rs │ │ └── mod.rs │ │ └── store │ │ └── mod.rs ├── fluvio-stream-model │ ├── Cargo.toml │ ├── LICENSE-APACHE │ └── src │ │ ├── core.rs │ │ ├── epoch │ │ ├── dual_epoch_map.rs │ │ ├── epoch_map.rs │ │ └── mod.rs │ │ ├── lib.rs │ │ └── store │ │ ├── actions.rs │ │ ├── concurrent_hashmap.rs │ │ ├── dual_store.rs │ │ ├── event.rs │ │ ├── filter.rs │ │ ├── k8.rs │ │ ├── metadata.rs │ │ └── mod.rs ├── fluvio-test-case-derive │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── src │ │ └── lib.rs │ ├── tests │ │ └── parse.rs │ └── ui-tests │ │ ├── fail_enum.rs │ │ ├── pass_basic_empty.rs │ │ └── pass_basic_not_empty.rs ├── fluvio-test-derive │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── src │ │ └── lib.rs │ ├── tests │ │ └── parse.rs │ └── ui-tests │ │ ├── fail_async.rs │ │ ├── fail_cluster_type.rs │ │ ├── fail_invalid_key.rs │ │ ├── fail_min_spu_type.rs │ │ ├── fail_multi.rs │ │ ├── fail_name.rs │ │ ├── fail_timeout.rs │ │ ├── fail_topic_type.rs │ │ ├── pass_async.rs │ │ ├── pass_cluster_type.rs │ │ ├── pass_min_spu.rs │ │ ├── pass_multi.rs │ │ ├── pass_name.rs │ │ ├── pass_no_macro_args.rs │ │ ├── pass_timeout.rs │ │ └── pass_topic.rs ├── fluvio-test-util │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── lib.rs │ ├── setup │ │ ├── environment │ │ │ ├── k8.rs │ │ │ ├── local.rs │ │ │ └── mod.rs │ │ └── mod.rs │ ├── test_meta │ │ ├── derive_attr.rs │ │ ├── environment.rs │ │ ├── fork.rs │ │ ├── mod.rs │ │ ├── test_result.rs │ │ └── test_timer.rs │ ├── test_runner │ │ ├── mod.rs │ │ ├── test_driver │ │ │ └── mod.rs │ │ └── test_meta.rs │ └── tls.rs ├── fluvio-test │ ├── Cargo.toml │ ├── LICENSE-APACHE │ └── src │ │ ├── lib.rs │ │ ├── main.rs │ │ └── tests │ │ ├── batching │ │ └── mod.rs │ │ ├── concurrent │ │ ├── consumer.rs │ │ ├── mod.rs │ │ ├── producer.rs │ │ └── util.rs │ │ ├── consumer.rs │ │ ├── consumer_offsets │ │ ├── auto │ │ │ ├── flush.rs │ │ │ └── mod.rs │ │ ├── manual │ │ │ ├── beginning.rs │ │ │ ├── end.rs │ │ │ ├── from_absolute.rs │ │ │ ├── from_beginning.rs │ │ │ ├── from_end.rs │ │ │ └── mod.rs │ │ ├── mod.rs │ │ ├── none │ │ │ └── mod.rs │ │ ├── option.rs │ │ └── utils.rs │ │ ├── data_generator │ │ ├── mod.rs │ │ └── producer.rs │ │ ├── election │ │ └── mod.rs │ │ ├── expected_fail.rs │ │ ├── expected_fail_join_fail_first.rs │ │ ├── expected_fail_join_success_first.rs │ │ ├── expected_pass.rs │ │ ├── expected_timeout.rs │ │ ├── longevity │ │ ├── consumer.rs │ │ ├── mod.rs │ │ └── producer.rs │ │ ├── mod.rs │ │ ├── multiple_partitions │ │ ├── consumer.rs │ │ ├── mod.rs │ │ └── producer.rs │ │ ├── producer.rs │ │ ├── producer_fail │ │ └── mod.rs │ │ ├── reconnection │ │ └── mod.rs │ │ ├── self_test │ │ └── mod.rs │ │ ├── smoke │ │ ├── consume.rs │ │ ├── listener.rs │ │ ├── message.rs │ │ ├── mod.rs │ │ ├── offsets.rs │ │ └── produce.rs │ │ ├── stats.rs │ │ └── stress.rs ├── fluvio-types │ ├── Cargo.toml │ ├── LICENSE-APACHE │ └── src │ │ ├── compression.rs │ │ ├── config_file.rs │ │ ├── defaults.rs │ │ ├── event.rs │ │ ├── lib.rs │ │ ├── macros.rs │ │ └── partition.rs ├── fluvio-version-manager │ ├── .gitignore │ ├── Cargo.toml │ ├── LICENSE │ ├── build.rs │ ├── fixtures │ │ └── version │ │ │ ├── 0.10.14 │ │ │ ├── fluvio │ │ │ └── manifest.json │ │ │ ├── 0.10.15 │ │ │ ├── fluvio │ │ │ └── manifest.json │ │ │ └── stable │ │ │ ├── fluvio │ │ │ └── manifest.json │ ├── src │ │ ├── command │ │ │ ├── current.rs │ │ │ ├── install.rs │ │ │ ├── itself │ │ │ │ ├── install.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── uninstall.rs │ │ │ │ └── update.rs │ │ │ ├── list.rs │ │ │ ├── mod.rs │ │ │ ├── switch.rs │ │ │ ├── uninstall.rs │ │ │ ├── update.rs │ │ │ └── version.rs │ │ ├── common │ │ │ ├── executable.rs │ │ │ ├── manifest.rs │ │ │ ├── mod.rs │ │ │ ├── notify.rs │ │ │ ├── settings.rs │ │ │ ├── update_manager.rs │ │ │ ├── version_directory.rs │ │ │ ├── version_installer.rs │ │ │ └── workdir.rs │ │ └── main.rs │ └── tests │ │ ├── install-fvm.sh │ │ └── readme.md ├── fluvio │ ├── Cargo.toml │ ├── LICENSE-APACHE │ ├── README.md │ ├── VERSION │ ├── build.rs │ ├── src │ │ ├── admin.rs │ │ ├── config │ │ │ ├── cluster.rs │ │ │ ├── config.rs │ │ │ ├── mod.rs │ │ │ └── tls.rs │ │ ├── consumer │ │ │ ├── config.rs │ │ │ ├── mod.rs │ │ │ ├── offset.rs │ │ │ ├── retry.rs │ │ │ └── stream.rs │ │ ├── error.rs │ │ ├── fluvio.rs │ │ ├── lib.rs │ │ ├── metrics │ │ │ └── mod.rs │ │ ├── offset.rs │ │ ├── producer │ │ │ ├── accumulator.rs │ │ │ ├── config.rs │ │ │ ├── error.rs │ │ │ ├── event.rs │ │ │ ├── memory_batch.rs │ │ │ ├── mod.rs │ │ │ ├── output.rs │ │ │ ├── partition_producer.rs │ │ │ ├── partitioning.rs │ │ │ └── record.rs │ │ ├── spu.rs │ │ └── sync │ │ │ ├── controller.rs │ │ │ ├── mod.rs │ │ │ └── store.rs │ └── test-data │ │ ├── profiles │ │ ├── config.toml │ │ ├── default.toml │ │ ├── invalid.toml │ │ └── updatable_config.toml │ │ └── tls │ │ ├── file.toml │ │ ├── inline.toml │ │ └── no-verf.toml └── smartmodule-development-kit │ ├── Cargo.toml │ ├── LICENSE-APACHE │ └── src │ ├── build.rs │ ├── clean.rs │ ├── cmd.rs │ ├── generate.rs │ ├── hub.rs │ ├── init.rs │ ├── load.rs │ ├── main.rs │ ├── publish.rs │ ├── set_public.rs │ └── test.rs ├── deny.toml ├── dev-tools ├── PUBLISH.md ├── README.md ├── log │ ├── debug-sc-all │ ├── debug-sc-client │ ├── debug-sc-min │ ├── debug-sc-min-k8 │ ├── debug-sc-server │ ├── debug-spu-all │ ├── debug-spu-min │ ├── test-sc-min │ └── test-spu-min ├── sc-pod-log.sh └── smartmodule.Dockerfile ├── examples ├── 00-produce │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── 01-produce-batch │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── 01-produce-key-value │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── 02-consume │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── 03-echo │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── 04-admin-watch │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── docker-compose │ ├── Dockerfile │ ├── README.md │ └── docker-compose.yml ├── partitioning-simple │ ├── Cargo.toml │ └── src │ │ └── lib.rs └── tests │ ├── build.sh │ ├── run.sh │ ├── test-echo.sh │ └── test-simple.sh ├── k8-util ├── DEVELOP.md ├── cluster │ ├── cluster-type.sh │ ├── kind.sh │ ├── kind.yaml │ ├── reset-docker.sh │ ├── reset-k3d.sh │ ├── reset-kind.sh │ ├── reset-minikube.sh │ └── start-minikube-mac.sh ├── dashboard │ ├── README.md │ ├── metric-server.yaml │ ├── token.sh │ └── user.yaml ├── docker │ ├── build.sh │ └── fluvio.Dockerfile ├── helm │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── fluvio-app │ │ ├── Chart.yaml │ │ ├── templates │ │ │ ├── fluvio-connector-config.yaml │ │ │ ├── role-binding.yaml │ │ │ ├── role.yaml │ │ │ ├── sc-deployment.yaml │ │ │ ├── sc-internal.yaml │ │ │ ├── sc-public.yaml │ │ │ ├── service.yaml │ │ │ └── spu-k8-config.yaml │ │ └── values.yaml │ └── fluvio-sys │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ ├── crd_mirror.yaml │ │ ├── crd_partition.yaml │ │ ├── crd_smart_module.yaml │ │ ├── crd_spg.yaml │ │ ├── crd_spu.yaml │ │ ├── crd_tableformat.yaml │ │ └── crd_topic.yaml │ │ └── values.yaml ├── samples │ └── crd │ │ ├── group1.yaml │ │ ├── group1_debug.yaml │ │ ├── group2.yaml │ │ ├── group3.yaml │ │ ├── init.sh │ │ ├── partition.yaml │ │ ├── spu_5001.yaml │ │ ├── spu_5002.yaml │ │ ├── spu_5003.yaml │ │ ├── start_5001.sh │ │ ├── topic1.yaml │ │ ├── topic2.yaml │ │ ├── topic3.yaml │ │ ├── topic4.yaml │ │ ├── topic5.yaml │ │ ├── topic5_custom.yaml │ │ └── topic6_custom.yaml └── util │ ├── README.md │ ├── busybox.yaml │ ├── deploy_dashboard.sh │ ├── dns.sh │ ├── run_dashboard.sh │ └── snippet.md ├── makefiles ├── build.mk ├── check.mk ├── release.mk └── test.mk ├── release-tools ├── check-crate-version │ ├── .gitignore │ ├── Cargo.toml │ ├── publish-list.toml │ ├── src │ │ ├── download.rs │ │ └── main.rs │ └── toml-diff │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── src │ │ ├── display.rs │ │ ├── lib.rs │ │ └── test.rs │ │ └── test_data │ │ ├── array_add_a.toml │ │ ├── array_add_b.toml │ │ ├── array_delete_a.toml │ │ ├── array_delete_b.toml │ │ ├── array_delete_one_a.toml │ │ ├── array_delete_one_b.toml │ │ ├── array_reorder_a.toml │ │ ├── array_reorder_b.toml │ │ ├── arrays_a.toml │ │ ├── arrays_b.toml │ │ ├── control_a.toml │ │ ├── control_b.toml │ │ ├── nested_tables_a.toml │ │ ├── nested_tables_b.toml │ │ ├── strings_a.toml │ │ ├── strings_b.toml │ │ ├── tables_a.toml │ │ └── tables_b.toml ├── check-publish-crates.sh ├── publish-crates.sh ├── publish-list └── test-crate-version.sh ├── rfc ├── add-partition.md ├── materialize_view.md ├── offset-management.md └── produce-message-size.md ├── rust-toolchain.toml ├── rustfmt.toml ├── smartmodule ├── cargo_template │ ├── .gitignore │ ├── Cargo.toml.liquid │ ├── README.md │ ├── SmartModule.toml │ ├── cargo-generate.toml │ ├── rust-toolchain.toml │ └── src │ │ └── lib.rs ├── examples │ ├── .cargo │ │ └── config.toml │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ ├── Makefile │ ├── aggregate-average │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── aggregate-init │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── SmartModule.toml │ │ ├── src │ │ │ └── lib.rs │ │ └── transforms.yaml │ ├── aggregate-json │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── aggregate-sum │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── aggregate │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── aggregate_with_timestamp │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── array_map_json_array │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── array_map_json_array_with_timestamp │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── array_map_json_object │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── array_map_json_reddit │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_hashset │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── SmartModule.toml │ │ ├── src │ │ │ └── lib.rs │ │ └── transforms.yaml │ ├── filter_init │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_json │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_look_back │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── SmartModule.toml │ │ ├── src │ │ │ └── lib.rs │ │ └── transforms.yaml │ ├── filter_look_back_with_timestamps │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── SmartModule.toml │ │ ├── src │ │ │ └── lib.rs │ │ └── transforms.yaml │ ├── filter_map │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_odd │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_odd_key │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_regex │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_with_param │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── filter_with_param_v1 │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── map │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── map_double │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── map_json │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ ├── map_regex │ │ ├── Cargo.toml │ │ └── src │ │ │ └── lib.rs │ └── map_with_timestamp │ │ ├── Cargo.toml │ │ └── src │ │ └── lib.rs └── regex-filter │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ ├── Makefile │ ├── README.md │ ├── SmartModule.toml │ └── src │ └── lib.rs ├── tests ├── .gitignore ├── Makefile ├── PERF.md ├── README.md ├── TEST_CASE.md ├── cli │ ├── cdk_smoke_tests │ │ ├── cdk-basic.bats │ │ ├── cdk-graceful-shutdown.bats │ │ ├── cdk-multi-deploy.bats │ │ └── cdk-multi-partition-consumer.bats │ ├── cli-backward-compatibility.bash │ ├── cli-platform-cross-version.bats │ ├── fluvio_read_only │ │ └── basic.bats │ ├── fluvio_smoke_tests │ │ ├── add-partitions.bats │ │ ├── consume-batch-size.bats │ │ ├── consumer-offset.bats │ │ ├── e2e-basic.bats │ │ ├── e2e-file-with-separator.bats │ │ ├── e2e-segment-partitions.bats │ │ ├── e2e-smartmodule-basic.bats │ │ ├── non-concurrent │ │ │ ├── cluster-delete.bats │ │ │ └── local-resume.bats │ │ ├── produce-batch-size.bats │ │ ├── produce-error.bats │ │ ├── producer-smartmodule.bats │ │ ├── profile-export.bats │ │ ├── smartmodule-basic.bats │ │ ├── system-topic-basic.bats │ │ ├── table-format-basic.bats │ │ └── topic-basic.bats │ ├── fvm_smoke_tests │ │ └── fvm_basic.bats │ ├── mirroring_smoke_tests │ │ ├── e2e │ │ │ └── fluvio-core.bats │ │ ├── export.bats │ │ ├── mirror-topic-reverse.bats │ │ ├── mirror-topic.bats │ │ └── remote.bats │ ├── partition_test │ │ └── multiple_partitions.bats │ ├── smdk_smoke_tests │ │ └── smdk-basic.bats │ └── test_helper │ │ ├── .gitignore │ │ ├── fluvio_dev.bash │ │ ├── test-table-format-config.yml │ │ └── tools_check.bash ├── fluvio-validate-release.sh ├── fluvio-values.yaml ├── local-resume-test.sh ├── longevity-producer.sh ├── produce-large.sh ├── test-table-format-config.yaml └── upgrade-test.sh └── tls ├── Makefile ├── cert.conf ├── certs ├── ca.crt ├── ca.key ├── ca.srl ├── client-root.crt ├── client-root.csr ├── client-root.key ├── client-user1.crt ├── client-user1.csr ├── client-user1.key ├── server.crt ├── server.csr └── server.key └── openssl.cnf /.github/PULL_REQUEST_TEMPLATE/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # What does this PR do? 2 | 3 | A brief description of the change being made in this pull request. 4 | 5 | # Motivation 6 | 7 | What inspired you to submit this pull request? 8 | 9 | # Related issues 10 | 11 | A list of issues closed by or relevant to this PR. 12 | 13 | - Closes #. 14 | 15 | # Additional Notes 16 | 17 | Is there anything else you'd like us to know while reviewing this? 18 | 19 | # Checklist 20 | 21 | - [ ] Created or updated all relevant documentation 22 | - [ ] Added unit tests to relevant added or changed code 23 | - [ ] Bumped required crate version numbers 24 | - [ ] Updated CHANGELOG.md 25 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/release_template.md: -------------------------------------------------------------------------------- 1 | # Release complete 2 | 3 | Closes # 4 | 5 | * [ ] Increment patch version in [`VERSION`](https://github.com/infinyon/fluvio/blob/master/VERSION) file 6 | * [ ] Add a new heading in [`CHANGELOG`](https://github.com/infinyon/fluvio/blob/master/CHANGELOG.md) with for the next release (Use heading level 2) with UNRELEASED for the date 7 | -------------------------------------------------------------------------------- /.github/actions/dl-smdk/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Download smdk' 2 | description: 'download smdk artfact from build_primary_binaries' 3 | runs: 4 | using: "composite" 5 | steps: 6 | - name: Download artifact - smdk 7 | uses: actions/download-artifact@v4 8 | with: 9 | name: smdk-x86_64-unknown-linux-musl 10 | path: ~/bin 11 | - name: mark smdk 12 | shell: bash 13 | run: | 14 | chmod +x ~/bin/smdk 15 | echo "~/bin" >> $GITHUB_PATH 16 | -------------------------------------------------------------------------------- /.github/assets/fluvio-event-streaming.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/.github/assets/fluvio-event-streaming.gif -------------------------------------------------------------------------------- /.github/workflows/lint_pr.yml: -------------------------------------------------------------------------------- 1 | name: "Lint PR" 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | main: 12 | name: Validate PR title 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: amannn/action-semantic-pull-request@v4 16 | env: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | with: 19 | ignoreLabels: | 20 | bot 21 | ignore-semantic-pull-request 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | **/*.rs.bk 3 | .DS_Store 4 | .docker-cargo 5 | target-docker 6 | .idea/ 7 | VERSION 8 | .scripts 9 | .env 10 | .vscode/ 11 | vendor/ 12 | *.tmp 13 | 14 | # Test Connectors Logs 15 | connector/**/*.log 16 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.17.4-dev-1 -------------------------------------------------------------------------------- /actions/ci-replace-helm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | echo "Operating System: $OS" 4 | case $OS in 5 | 'Linux') PLATFORM=linux-amd64 6 | ;; 7 | 'ubuntu-latest') PLATFORM=linux-amd64 8 | ;; 9 | 'macOS-latest') PLATFORM=darwin-amd64 10 | ;; 11 | *) echo "invalid os" 12 | exit 13 | ;; 14 | esac 15 | 16 | cd $(mktemp -d) 17 | curl -sSL https://get.helm.sh/helm-${HELM_VERSION}-${PLATFORM}.tar.gz | tar zx 18 | helm_bin=$(which helm) 19 | sudo rm -f $helm_bin 20 | sudo mv ${PLATFORM}/helm $helm_bin 21 | chmod +x $helm_bin -------------------------------------------------------------------------------- /actions/post-release-changelog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # update CHANGELOG.md with unreleased changes since last tag 3 | git cliff --unreleased -p CHANGELOG.md -------------------------------------------------------------------------------- /actions/sccache-terminate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # set up sccache 3 | set -e 4 | MATRIX_OS=${1} 5 | 6 | echo "Starting SCCACHE" 7 | sccache --stop-server || true 8 | 9 | 10 | -------------------------------------------------------------------------------- /actions/view-changelog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # run to view changelog updates 3 | # run with argument "modify" to prepend changelog updates to CHANGELOG.md 4 | 5 | set -e 6 | 7 | MODIFY="${1:-no}" 8 | 9 | LAST_TAG=$(gh release list --limit 1 --exclude-pre-releases | cut -w -f1) 10 | CURR_TAG=$(cat VERSION) 11 | echo git cliff ${LAST_TAG}..HEAD -t ${CURR_TAG} 12 | 13 | if [ "$MODIFY" = "modify" ]; then 14 | git cliff ${LAST_TAG}..HEAD -t ${CURR_TAG} -p CHANGELOG.md 15 | else 16 | git cliff ${LAST_TAG}..HEAD -t ${CURR_TAG} | less 17 | fi 18 | -------------------------------------------------------------------------------- /actions/zig-uninstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Install Zig 3 | # This is optimized for installing on GitHub Actions 4 | # To run in Apple silicon, `ARCH=aarch64 zig-install` 5 | set -e 6 | echo "removing zig" 7 | sudo rm -rf /usr/local/bin/zig 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /build-scripts/aarch64-linux-musl-zig-cc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | new_array=() 4 | 5 | for value in "$@" 6 | do 7 | [[ $value != *self-contained/*crt* ]] && new_array+=($value) 8 | done 9 | 10 | $FLUVIO_BUILD_ZIG cc "${new_array[@]}" -target aarch64-linux-musl 11 | -------------------------------------------------------------------------------- /build-scripts/aarch64-macos-gnu-zig-cc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | new_array=() 4 | 5 | for value in "$@" 6 | do 7 | [[ $value != *self-contained/*crt* ]] && new_array+=($value) 8 | done 9 | 10 | $FLUVIO_BUILD_ZIG cc "${new_array[@]}" -target aarch64-macos-gnu 11 | -------------------------------------------------------------------------------- /build-scripts/arm-linux-gnu-zig-cc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $FLUVIO_BUILD_ZIG cc $(echo $@ | sed s/\-latomic\ //g) -target arm-linux-gnueabihf 4 | -------------------------------------------------------------------------------- /build-scripts/install_target.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | if [ "$TARGET" = "armv7-unknown-linux-gnueabihf" ] || [ "$TARGET" = "arm-unknown-linux-gnueabihf" ]; then 5 | cargo install --locked bindgen-cli 6 | fi 7 | 8 | if [ -n "$TARGET" ]; then 9 | echo "loading target $TARGET" 10 | rustup target add $TARGET 11 | fi 12 | -------------------------------------------------------------------------------- /build-scripts/x86_64-linux-musl-zig-cc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | new_array=() 4 | 5 | for value in "$@" 6 | do 7 | [[ $value != *self-contained/*crt* ]] && new_array+=($value) 8 | done 9 | 10 | $FLUVIO_BUILD_ZIG cc "${new_array[@]}" -target x86_64-linux-musl -------------------------------------------------------------------------------- /connector/cargo_template/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.hub 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /connector/cargo_template/Cargo.toml.liquid: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "{{project-name}}" 3 | version = "0.1.0" 4 | authors = ["{{authors}}"] 5 | edition = "2024" 6 | 7 | [workspace] 8 | 9 | [dependencies] 10 | {% if connector-type == "sink" %}futures = { version = "0.3", default-features = false }{% endif %} 11 | serde = { version = "1.0", default-features = false, features = ["derive"]} 12 | 13 | fluvio = { git = "https://github.com/infinyon/fluvio", rev = "{{fluvio-cargo-dependency-hash}}" } 14 | fluvio-connector-common = { git = "https://github.com/infinyon/fluvio", rev = "{{fluvio-cargo-dependency-hash}}", features = ["derive"] } 15 | -------------------------------------------------------------------------------- /connector/cargo_template/Connector.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "{{project-name}}" 3 | group = "{{project-group | kebab_case}}" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | fluvio = "0.10.0" 7 | description = "{{project-description}}" 8 | license = "Apache-2.0" 9 | visibility = {% if connector-public %}"public"{% else %}"private"{% endif %} 10 | 11 | [direction] 12 | {% if connector-type == "source" %} source = true {% else %} dest = true {% endif %} 13 | 14 | [deployment] 15 | binary = "{{project-name}}" 16 | 17 | [custom.properties.foo] 18 | title = "Foo" 19 | description = "Foo" 20 | type = "string" 21 | -------------------------------------------------------------------------------- /connector/cargo_template/README.md: -------------------------------------------------------------------------------- 1 | # {{project-name}} 2 | -------------------------------------------------------------------------------- /connector/cargo_template/cargo-generate.toml: -------------------------------------------------------------------------------- 1 | [placeholders.fluvio-cargo-dependency-hash] 2 | type = "string" 3 | prompt = "Value of commit rev of `fluvio` dependency in Cargo.toml" 4 | 5 | [placeholders.project-group] 6 | type = "string" 7 | prompt = "Please set a group name" 8 | 9 | [placeholders.connector-type] 10 | type = "string" 11 | prompt = "Which type of Connector would you like [source/sink]?" 12 | choices = ["source", "sink"] 13 | default = "source" 14 | 15 | [placeholders.connector-public] 16 | type = "bool" 17 | prompt = "Will your Connector be public?" 18 | default = false 19 | -------------------------------------------------------------------------------- /connector/cargo_template/sample-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-{{project-name}}-test-connector 5 | type: {{project-name}}-{% if connector-type == "source" %}source{% else %}sink{% endif %} 6 | topic: test-{{project-name}}-topic 7 | custom: 8 | foo: bar 9 | -------------------------------------------------------------------------------- /connector/cargo_template/src/config.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(config)] 4 | #[derive(Debug)] 5 | pub(crate) struct CustomConfig { 6 | #[allow(dead_code)] 7 | foo: String, 8 | } 9 | -------------------------------------------------------------------------------- /connector/docker/.gitignore: -------------------------------------------------------------------------------- 1 | fluvio_profile.* 2 | -------------------------------------------------------------------------------- /connector/docker/Makefile: -------------------------------------------------------------------------------- 1 | ORG=myorg 2 | IMAGE=http-source 3 | VERSION=0.3.8 4 | 5 | copy-profile: 6 | fluvio profile export > fluvio_profile.toml 7 | 8 | build: copy-profile 9 | docker build --progress=plain -t $(ORG)/$(IMAGE):$(VERSION) . 10 | 11 | run: build 12 | docker run --network host --rm $(ORG)/$(IMAGE):$(VERSION) -------------------------------------------------------------------------------- /connector/docker/connector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.3.8 4 | name: http-quotes 5 | type: http-source 6 | topic: quotes 7 | http: 8 | endpoint: https://demo-data.infinyon.com/api/quote 9 | interval: 2s -------------------------------------------------------------------------------- /connector/json-test-connector/Connector.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "json-test-connector" 3 | group = "fluvio" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | fluvio = "0.10.0" 7 | description = "Generate JSON generator" 8 | license = "Apache-2.0" 9 | 10 | 11 | [direction] 12 | source = true 13 | 14 | [deployment] 15 | binary = "json-test-connector" 16 | 17 | [custom] 18 | name = "json" 19 | required = ["template", "interval"] 20 | 21 | [custom.properties.template] 22 | title = "template" 23 | description = "JSON template" 24 | type = "string" 25 | 26 | [custom.properties.interval] 27 | title = "Interval" 28 | description = "Interval of producing value" 29 | type = "integer" 30 | 31 | -------------------------------------------------------------------------------- /connector/json-test-connector/README.md: -------------------------------------------------------------------------------- 1 | # json-test-connector 2 | -------------------------------------------------------------------------------- /connector/json-test-connector/sample-config-v2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.2.0 2 | meta: 3 | version: 0.1.0 4 | name: my-json-test-connector 5 | type: json-test-source 6 | topic: 7 | version: 0.1.0 8 | meta: 9 | name: test-full-topic 10 | partition: 11 | count: 1 12 | max-size: 4.0 KB 13 | replication: 1 14 | ignore-rack-assignment: true 15 | retention: 16 | time: 2m 17 | segment-size: 2.0 KB 18 | compression: 19 | type: Lz4 20 | json: 21 | interval: 5 22 | timeout: 15 23 | template: '{"template":"test"}' 24 | 25 | -------------------------------------------------------------------------------- /connector/json-test-connector/sample-config.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | version: 0.1.0 3 | name: my-json-test-connector 4 | type: json-test-source 5 | topic: test-topic 6 | json: 7 | interval: 5 8 | timeout: 15 9 | template: '{"template":"test"}' 10 | -------------------------------------------------------------------------------- /connector/json-test-connector/sample-config2-v2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.2.0 2 | meta: 3 | version: 0.1.0 4 | name: my-json-test-connector2 5 | type: json-test-source 6 | topic: 7 | version: 0.1.0 8 | meta: 9 | name: test-full-topic 10 | partition: 11 | count: 1 12 | max-size: 4.0 KB 13 | replication: 1 14 | ignore-rack-assignment: true 15 | retention: 16 | time: 2m 17 | segment-size: 2.0 KB 18 | compression: 19 | type: Lz4 20 | json: 21 | interval: 5 22 | timeout: 15 23 | template: '{"template":"test"}' 24 | 25 | -------------------------------------------------------------------------------- /connector/sink-test-connector/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sink-test-connector" 3 | description = "Connector that reads data from a topic and prints to stdout" 4 | version = "0.0.0" 5 | publish = false 6 | repository.workspace = true 7 | edition.workspace = true 8 | license.workspace = true 9 | authors.workspace = true 10 | 11 | [dependencies] 12 | async-trait = { workspace = true } 13 | futures = { workspace = true } 14 | anyhow = { workspace = true } 15 | serde = { default-features = false, features = ["derive"], workspace = true } 16 | 17 | fluvio = { path = "../../crates/fluvio/", features = ["smartengine"]} 18 | fluvio-connector-common = { path = "../../crates/fluvio-connector-common/", features = ["derive"] } 19 | 20 | -------------------------------------------------------------------------------- /connector/sink-test-connector/Connector.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sink-test-connector" 3 | group = "fluvio" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | fluvio = "0.10.0" 7 | description = "Test Sink Connector" 8 | license = "Apache-2.0" 9 | 10 | [direction] 11 | dest = true 12 | 13 | [deployment] 14 | binary = "sink-test-connector" 15 | -------------------------------------------------------------------------------- /connector/sink-test-connector/README.md: -------------------------------------------------------------------------------- 1 | # sink-test-connector 2 | -------------------------------------------------------------------------------- /connector/sink-test-connector/sample-config.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | version: 0.1.0 3 | name: my-sink-test-connector 4 | type: test-sink 5 | topic: test-topic 6 | secrets: 7 | - name: TEST_API_KEY 8 | - name: TEST_API_CLIENT_ID 9 | consumer: 10 | id: "sink-test-connector" 11 | offset: 12 | strategy: manual 13 | custom: 14 | api_key: 15 | secret: 16 | name: TEST_API_KEY 17 | client_id: ${{ secrets.TEST_API_CLIENT_ID }} 18 | transforms: 19 | - uses: infinyon/jolt@0.1.0 20 | with: 21 | spec: 22 | - operation: default 23 | spec: 24 | data: 25 | source: "sink-test-connector" 26 | -------------------------------------------------------------------------------- /connector/sink-test-connector/secrets.txt: -------------------------------------------------------------------------------- 1 | TEST_API_KEY=test value 2 | TEST_API_CLIENT_ID=client1 -------------------------------------------------------------------------------- /crates/cargo-builder/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cargo-builder" 3 | description = "Build Cargo project programmability" 4 | version = "0.0.0" 5 | publish = false 6 | repository.workspace = true 7 | edition.workspace = true 8 | license.workspace = true 9 | authors.workspace = true 10 | 11 | 12 | 13 | [dependencies] 14 | anyhow = { workspace = true } 15 | derive_builder = { workspace = true } 16 | cargo_metadata = "0.18.0" 17 | -------------------------------------------------------------------------------- /crates/cargo-builder/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod cargo; 2 | pub mod package; 3 | 4 | const WASM_TARGET: &str = "wasm32-wasip1"; 5 | -------------------------------------------------------------------------------- /crates/cdk/build.rs: -------------------------------------------------------------------------------- 1 | use std::process::Command; 2 | 3 | fn main() { 4 | // Fetch current git hash to print version output 5 | let git_version_output = Command::new("git") 6 | .args(["rev-parse", "HEAD"]) 7 | .output() 8 | .expect("should run 'git rev-parse HEAD' to get git hash"); 9 | let git_hash = String::from_utf8(git_version_output.stdout) 10 | .expect("should read 'git' stdout to find hash"); 11 | // Assign the git hash to the compile-time GIT_HASH env variable (to use with env!()) 12 | println!("cargo:rustc-env=GIT_HASH={git_hash}"); 13 | } 14 | -------------------------------------------------------------------------------- /crates/cdk/src/main.rs: -------------------------------------------------------------------------------- 1 | mod cmd; 2 | mod generate; 3 | mod build; 4 | mod test; 5 | mod deploy; 6 | mod publish; 7 | mod set_public; 8 | 9 | pub(crate) mod utils; 10 | mod hub; 11 | 12 | fn main() -> anyhow::Result<()> { 13 | use clap::Parser; 14 | 15 | use cmd::CdkCommand; 16 | 17 | fluvio_future::subscriber::init_tracer(None); 18 | 19 | let root = CdkCommand::parse(); 20 | root.process()?; 21 | 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /crates/cdk/tests/foo-0.1.0.ipkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/cdk/tests/foo-0.1.0.ipkg -------------------------------------------------------------------------------- /crates/fluvio-auth/README.md: -------------------------------------------------------------------------------- 1 | # flv-auth 2 | 3 | Fluvio Authorization Framework 4 | 5 | ## License 6 | 7 | This project is licensed under the [Apache license](LICENSE-APACHE). 8 | 9 | ### Contribution 10 | 11 | Unless you explicitly state otherwise, any contribution intentionally submitted 12 | for inclusion in Fluvio by you, shall be licensed as Apache, without any additional 13 | terms or conditions. -------------------------------------------------------------------------------- /crates/fluvio-auth/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::io::Error as IoError; 2 | use thiserror::Error; 3 | 4 | /// Possible errors from Auth 5 | #[derive(Error, Debug)] 6 | pub enum AuthError { 7 | #[error("IoError: {0}")] 8 | IoError(#[from] IoError), 9 | } 10 | 11 | impl From for IoError { 12 | fn from(e: AuthError) -> Self { 13 | match &e { 14 | AuthError::IoError(source) => IoError::new(source.kind(), e), 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /crates/fluvio-auth/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod policy; 2 | mod error; 3 | 4 | pub mod root; 5 | pub mod x509; 6 | 7 | pub use policy::*; 8 | pub use error::AuthError; 9 | -------------------------------------------------------------------------------- /crates/fluvio-auth/src/x509/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(unix)] 2 | mod authenticator; 3 | mod identity; 4 | mod request; 5 | 6 | #[cfg(unix)] 7 | pub use authenticator::*; 8 | pub use identity::*; 9 | -------------------------------------------------------------------------------- /crates/fluvio-benchmark/Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | cargo build --bin fluvio-benchmark --release -------------------------------------------------------------------------------- /crates/fluvio-benchmark/src/bin/fluvio-benchmark.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use fluvio_benchmark::cli::{BenchmarkOpt, run_benchmarks}; 4 | use fluvio_future::task::run_block_on; 5 | 6 | fn main() -> Result<()> { 7 | fluvio_future::subscriber::init_logger(); 8 | let args = BenchmarkOpt::parse(); 9 | 10 | run_block_on(run_benchmarks(args)) 11 | } 12 | -------------------------------------------------------------------------------- /crates/fluvio-benchmark/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod cli; 2 | pub mod config; 3 | pub mod producer_worker; 4 | pub mod stats_collector; 5 | pub mod benchmark_driver; 6 | pub mod producer_benchmark; 7 | pub mod utils; 8 | -------------------------------------------------------------------------------- /crates/fluvio-cli-common/src/error.rs: -------------------------------------------------------------------------------- 1 | use semver::Version; 2 | use fluvio_index::{PackageId, Target}; 3 | 4 | #[derive(thiserror::Error, Debug)] 5 | #[error("Package {package} is not published at version {version} for target {target}")] 6 | pub struct PackageNotFound { 7 | pub package: PackageId, 8 | pub version: Version, 9 | pub target: Target, 10 | } 11 | 12 | #[derive(thiserror::Error, Debug)] 13 | pub enum HttpError { 14 | #[error("Invalid input: {0}")] 15 | InvalidInput(String), 16 | } 17 | -------------------------------------------------------------------------------- /crates/fluvio-cli-common/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod http; 2 | pub mod install; 3 | pub mod error; 4 | 5 | #[cfg(feature = "file-records")] 6 | pub mod user_input; 7 | 8 | #[cfg(feature = "version-cmd")] 9 | pub mod version_cmd; 10 | 11 | #[cfg(feature = "smartmodule-test")] 12 | pub mod smartmodule; 13 | 14 | // Environment vars for Channels 15 | pub const FLUVIO_RELEASE_CHANNEL: &str = "FLUVIO_RELEASE_CHANNEL"; 16 | pub const FLUVIO_EXTENSIONS_DIR: &str = "FLUVIO_EXTENSIONS_DIR"; 17 | pub const FLUVIO_IMAGE_TAG_STRATEGY: &str = "FLUVIO_IMAGE_TAG_STRATEGY"; 18 | pub const FLUVIO_ALWAYS_CHECK_UPDATES: &str = "FLUVIO_ALWAYS_CHECK_UPDATES"; 19 | -------------------------------------------------------------------------------- /crates/fluvio-cli/src/client/produce/stats/mod.rs: -------------------------------------------------------------------------------- 1 | mod report; 2 | pub use report::{start_csv_report, write_csv_dataframe, format_current_stats, producer_summary}; 3 | 4 | pub(crate) use std::io::Error as IoError; 5 | pub(crate) use std::io::Write; 6 | pub(crate) use fluvio::stats::ClientStatsDataCollect; 7 | pub(crate) use tracing::debug; 8 | pub(crate) use indicatif::ProgressBar; 9 | pub(crate) use std::io::{ErrorKind, self}; 10 | pub(crate) use crossterm::tty::IsTty; 11 | -------------------------------------------------------------------------------- /crates/fluvio-cli/src/install/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod opts; 2 | pub mod update; 3 | -------------------------------------------------------------------------------- /crates/fluvio-cli/src/profile/current.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use anyhow::Result; 3 | 4 | use fluvio::config::ConfigFile; 5 | 6 | #[derive(Debug, Parser)] 7 | pub struct CurrentOpt {} 8 | 9 | impl CurrentOpt { 10 | pub fn process(self) -> Result<()> { 11 | match ConfigFile::load(None) { 12 | Ok(config_file) => { 13 | if let Some(profile) = config_file.config().current_profile_name() { 14 | println!("{profile}"); 15 | } else { 16 | println!("no current profile set"); 17 | } 18 | } 19 | Err(_) => println!("no profile can be found"), 20 | } 21 | 22 | Ok(()) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/connectors/error-batchsize.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | name: my-test-mqtt 3 | type: mqtt 4 | topic: my-mqtt 5 | create_topic: false 6 | direction: source 7 | producer: 8 | batch-size: 1aoeu 9 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/connectors/error-compression.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | name: my-test-mqtt 3 | type: mqtt 4 | topic: my-mqtt 5 | create_topic: false 6 | direction: source 7 | producer: 8 | compression: gzipaoeu 9 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/connectors/error-linger.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | name: my-test-mqtt 3 | type: mqtt 4 | topic: my-mqtt 5 | create_topic: false 6 | direction: source 7 | producer: 8 | linger: 1 9 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/connectors/error-version.yaml: -------------------------------------------------------------------------------- 1 | name: my-test-mqtt 2 | type: mqtt 3 | topic: my-mqtt 4 | create_topic: false 5 | direction: source 6 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/connectors/full-config.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | name: my-test-mqtt 3 | type: mqtt 4 | topic: my-mqtt 5 | create_topic: false 6 | direction: source 7 | parameters: 8 | param_1: "mqtt.hsl.fi" 9 | param_2: 10 | - "foo:baz" 11 | - "bar" 12 | param_3: 13 | foo: bar 14 | bar: 10.0 15 | linger.ms: 10 16 | param_4: true 17 | param_5: 10.0 18 | param_6: 19 | - -10 20 | - -10.0 21 | 22 | secrets: 23 | foo: bar 24 | producer: 25 | linger: 1ms 26 | batch-size: '44.0 MB' 27 | compression: gzip 28 | consumer: 29 | partition: 10 30 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/connectors/simple.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | name: my-test-mqtt 3 | type: mqtt 4 | topic: my-mqtt 5 | create_topic: false 6 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/derivedstream/left.yaml: -------------------------------------------------------------------------------- 1 | name: sjoin 2 | input: 3 | topic: 4 | name: left 5 | steps: 6 | - join: 7 | module: join 8 | right: 9 | derivedstream: 10 | name: rdouble -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/derivedstream/right.yaml: -------------------------------------------------------------------------------- 1 | name: rdouble 2 | input: 3 | topic: 4 | name: right 5 | steps: 6 | - map: 7 | module: double 8 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/derivedstream/simple_join.yaml: -------------------------------------------------------------------------------- 1 | name: join1 2 | input: 3 | topic: 4 | name: left 5 | steps: 6 | - join: 7 | module: join 8 | right: 9 | topic: 10 | name: right -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/group-cfg/valid_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage": { 3 | "logDir": "/tmp/fluvio", 4 | "size": "2Gi" 5 | }, 6 | "replication": { 7 | "inSyncReplicaMin ": 5 8 | }, 9 | "env": [ 10 | { 11 | "name": "ENV-VAR1", 12 | "value": "one" 13 | }, 14 | { 15 | "name": "ENV-VAR2", 16 | "value": "two" 17 | } 18 | ] 19 | } -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/input-files/icon1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-cli/test-data/input-files/icon1.png -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/input-files/icon2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-cli/test-data/input-files/icon2.png -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/input-files/json-file.json: -------------------------------------------------------------------------------- 1 | { 2 | "partitions": [ 3 | { 4 | "id": 0, 5 | "replicas": [ 6 | 5001, 7 | 5002, 8 | 5003 9 | ] 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/input-files/text-file.txt: -------------------------------------------------------------------------------- 1 | Lorem Ipsum is simply dummy text. 2 | Lorem Ipsum has been the industry's standard since 1500's. 3 | It has survived over five centuries. 4 | Still here today. -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/test-config.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | name: my-test-mqtt 3 | type: mqtt 4 | topic: my-mqtt 5 | create_topic: false 6 | direction: source 7 | parameters: 8 | mqtt-url: "mqtt.hsl.fi" 9 | mqtt-topic: "/hfp/v2/journey/#" 10 | fluvio-topic: "my-mqtt" 11 | secrets: 12 | foo: bar 13 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/test-tableformat-config.yaml: -------------------------------------------------------------------------------- 1 | name: testtable 2 | inputFormat: "JSON" 3 | columns: 4 | - keyPath: "key2" 5 | display: true 6 | - keyPath: "key1" 7 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/topics/mirror_assignment.json: -------------------------------------------------------------------------------- 1 | [ 2 | "boat1", "boat2" 3 | ] 4 | -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/topics/out_of_order.json: -------------------------------------------------------------------------------- 1 | { 2 | "partitions": [ 3 | { 4 | "id": 0, 5 | "replicas": [ 6 | 5001, 7 | 5002, 8 | 5003 9 | ] 10 | }, 11 | { 12 | "id": 2, 13 | "replicas": [ 14 | 5003, 15 | 5001, 16 | 5002 17 | ] 18 | }, 19 | { 20 | "id": 1, 21 | "replicas": [ 22 | 5002, 23 | 5003, 24 | 5001 25 | ] 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/topics/replica_assignment.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 0, 4 | "replicas": [ 5 | 5001, 6 | 5002, 7 | 5003 8 | ] 9 | }, 10 | { 11 | "id": 1, 12 | "replicas": [ 13 | 5002, 14 | 5003, 15 | 5001 16 | ] 17 | }, 18 | { 19 | "id": 2, 20 | "replicas": [ 21 | 5003, 22 | 5001, 23 | 5002 24 | ] 25 | } 26 | ] -------------------------------------------------------------------------------- /crates/fluvio-cli/test-data/topics/valid_assignment1.json: -------------------------------------------------------------------------------- 1 | { 2 | "partitions": [ 3 | { 4 | "id": 0, 5 | "replicas": [ 6 | 5001 7 | ] 8 | } 9 | ] 10 | } -------------------------------------------------------------------------------- /crates/fluvio-cluster/Makefile: -------------------------------------------------------------------------------- 1 | install: 2 | make -C ../../k8-util/helm package -------------------------------------------------------------------------------- /crates/fluvio-cluster/src/check/render.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | 3 | use futures_util::StreamExt; 4 | use async_channel::Receiver; 5 | use crate::{ 6 | CheckResult, CheckResults, CheckStatus, CheckSuggestion, 7 | render::{ProgressRenderedText, ProgressRenderer}, 8 | }; 9 | 10 | const ISSUE_URL: &str = "https://github.com/infinyon/fluvio/issues/new/choose"; 11 | -------------------------------------------------------------------------------- /crates/fluvio-cluster/src/start/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod k8; 2 | pub mod local; 3 | mod common; 4 | 5 | mod constants { 6 | 7 | use std::env; 8 | 9 | use once_cell::sync::Lazy; 10 | 11 | /// maximum time waiting for SC and SPU to provision 12 | pub static MAX_PROVISION_TIME_SEC: Lazy = Lazy::new(|| { 13 | let var_value = env::var("FLV_CLUSTER_PROVISION_TIMEOUT").unwrap_or_default(); 14 | var_value.parse().unwrap_or(300) 15 | }); 16 | } 17 | -------------------------------------------------------------------------------- /crates/fluvio-cluster/test-data/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "topics": [ 3 | { 4 | "apiVersion": "fluvio.infinyon.com/v2", 5 | "kind": "Topic", 6 | "metadata": { 7 | "name": "my-topic" 8 | }, 9 | "spec": { 10 | "replicas": { 11 | "computed": { 12 | "partitions": 1, 13 | "replicationFactor": 1, 14 | "ignoreRackAssignment": false 15 | } 16 | } 17 | } 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /crates/fluvio-compression/README.md: -------------------------------------------------------------------------------- 1 | # Fluvio Compression 2 | 3 | Library with handlers to compress and uncompress data in the fluvio protocol. 4 | 5 | In fluvio, compression is done in producer side, then consumers and SPU when it is using SmartModules, uncompress the data using the compression information that is in the attributes of the batch. 6 | 7 | Currently, the supported compressions codecs are None (default), Gzip, Snappy, Zstd and LZ4. 8 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/mod.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn ui() { 3 | let t = trybuild::TestCases::new(); 4 | t.compile_fail("ui-test/ui/wrong_direction.rs"); 5 | t.compile_fail("ui-test/ui/wrong_input_len.rs"); 6 | t.compile_fail("ui-test/ui/wrong_rust_type.rs"); 7 | t.compile_fail("ui-test/ui/not_async.rs"); 8 | t.compile_fail("ui-test/ui/invalid_config_type.rs"); 9 | t.compile_fail("ui-test/ui/config_input_self.rs"); 10 | t.compile_fail("ui-test/ui/config_use_reserved_name_fluvio.rs"); 11 | t.compile_fail("ui-test/ui/config_use_reserved_name_transforms.rs"); 12 | t.compile_fail("ui-test/ui/struct_without_config.rs"); 13 | } 14 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/config_input_self.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | async fn start_fn(self, producer: ()) {} 5 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/config_input_self.stderr: -------------------------------------------------------------------------------- 1 | error: config input argument must not be self 2 | --> ui-test/ui/config_input_self.rs:4:19 3 | | 4 | 4 | async fn start_fn(self, producer: ()) {} 5 | | ^^^^ 6 | 7 | error[E0601]: `main` function not found in crate `$CRATE` 8 | --> ui-test/ui/config_input_self.rs:4:41 9 | | 10 | 4 | async fn start_fn(self, producer: ()) {} 11 | | ^ consider adding a `main` function to `$DIR/ui-test/ui/config_input_self.rs` 12 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/config_use_reserved_name_fluvio.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | async fn start_fn(config: CustomConfig, producer: ()) {} 5 | 6 | #[connector(config, name = "meta")] 7 | struct CustomConfig {} 8 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/config_use_reserved_name_transforms.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | async fn start_fn(config: CustomConfig, producer: ()) {} 5 | 6 | #[connector(config, name = "transforms")] 7 | struct CustomConfig {} 8 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/invalid_config_type.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | async fn start_fn(config: &[i32], producer: ()) {} 5 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/invalid_config_type.stderr: -------------------------------------------------------------------------------- 1 | error: config type must valid path of owned type 2 | --> ui-test/ui/invalid_config_type.rs:4:19 3 | | 4 | 4 | async fn start_fn(config: &[i32], producer: ()) {} 5 | | ^^^^^^ 6 | 7 | error[E0601]: `main` function not found in crate `$CRATE` 8 | --> ui-test/ui/invalid_config_type.rs:4:51 9 | | 10 | 4 | async fn start_fn(config: &[i32], producer: ()) {} 11 | | ^ consider adding a `main` function to `$DIR/ui-test/ui/invalid_config_type.rs` 12 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/not_async.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | fn start_fn() {} 5 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/not_async.stderr: -------------------------------------------------------------------------------- 1 | error: Connector function must be async 2 | --> ui-test/ui/not_async.rs:4:1 3 | | 4 | 4 | fn start_fn() {} 5 | | ^^ 6 | 7 | error[E0601]: `main` function not found in crate `$CRATE` 8 | --> ui-test/ui/not_async.rs:4:17 9 | | 10 | 4 | fn start_fn() {} 11 | | ^ consider adding a `main` function to `$DIR/ui-test/ui/not_async.rs` 12 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/struct_without_config.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | async fn start_fn(config: CustomConfig, producer: ()) {} 5 | 6 | #[connector] 7 | struct CustomConfig {} 8 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/wrong_direction.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(wrong)] 4 | fn start_fn() {} 5 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/wrong_direction.stderr: -------------------------------------------------------------------------------- 1 | error: Missing Connector type. Supported: '#[connector(source)]' and '#[connector(sink)]' 2 | --> ui-test/ui/wrong_direction.rs:3:13 3 | | 4 | 3 | #[connector(wrong)] 5 | | ^^^^^ 6 | 7 | error[E0601]: `main` function not found in crate `$CRATE` 8 | --> ui-test/ui/wrong_direction.rs:4:17 9 | | 10 | 4 | fn start_fn() {} 11 | | ^ consider adding a `main` function to `$DIR/ui-test/ui/wrong_direction.rs` 12 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/wrong_input_len.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | async fn start_fn() {} 5 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/wrong_input_len.stderr: -------------------------------------------------------------------------------- 1 | error: Connector function must have two input arguments 2 | --> ui-test/ui/wrong_input_len.rs:4:1 3 | | 4 | 4 | async fn start_fn() {} 5 | | ^^^^^ 6 | 7 | error[E0601]: `main` function not found in crate `$CRATE` 8 | --> ui-test/ui/wrong_input_len.rs:4:23 9 | | 10 | 4 | async fn start_fn() {} 11 | | ^ consider adding a `main` function to `$DIR/ui-test/ui/wrong_input_len.rs` 12 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/wrong_rust_type.rs: -------------------------------------------------------------------------------- 1 | use fluvio_connector_common::connector; 2 | 3 | #[connector(source)] 4 | mod start {} 5 | -------------------------------------------------------------------------------- /crates/fluvio-connector-common/ui-test/ui/wrong_rust_type.stderr: -------------------------------------------------------------------------------- 1 | error: macro supports only functions and structs 2 | --> ui-test/ui/wrong_rust_type.rs:4:1 3 | | 4 | 4 | mod start {} 5 | | ^^^ 6 | 7 | error[E0601]: `main` function not found in crate `$CRATE` 8 | --> ui-test/ui/wrong_rust_type.rs:4:13 9 | | 10 | 4 | mod start {} 11 | | ^ consider adding a `main` function to `$DIR/ui-test/ui/wrong_rust_type.rs` 12 | -------------------------------------------------------------------------------- /crates/fluvio-connector-deployer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-connector-deployer" 3 | description = "Deployer Fluvio Connector" 4 | version = "0.0.0" 5 | repository.workspace = true 6 | edition.workspace = true 7 | license.workspace = true 8 | authors.workspace = true 9 | 10 | 11 | 12 | [dependencies] 13 | tracing = { workspace = true } 14 | anyhow = { workspace = true } 15 | clap = { workspace = true, features = ["std", "derive", "help", "usage", "error-context", "env", "wrap_help", "suggestions"], default-features = false } 16 | derive_builder = { workspace = true } 17 | enum-display = { workspace = true } 18 | 19 | fluvio-connector-package = { workspace = true } 20 | -------------------------------------------------------------------------------- /crates/fluvio-connector-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-connector-derive" 3 | description = "Fluvio Connector procedural macro" 4 | version = "0.0.0" 5 | repository.workspace = true 6 | edition.workspace = true 7 | license.workspace = true 8 | authors.workspace = true 9 | 10 | [lib] 11 | proc-macro = true 12 | 13 | [dependencies] 14 | syn = { workspace = true, features = ["full"] } 15 | quote = { workspace = true } 16 | proc-macro2 = { workspace = true } -------------------------------------------------------------------------------- /crates/fluvio-connector-package/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod metadata; 2 | pub mod config; 3 | pub mod secret; 4 | mod render; 5 | 6 | pub use render::render_config_str; 7 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/all-partitions.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.2.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: 7 | version: 0.1.0 8 | meta: 9 | name: test-topic 10 | consumer: 11 | partition: all 12 | max_bytes: "1 MB" 13 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/error-batchsize.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | producer: 9 | batch-size: 1aoeu 10 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/error-compression.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | producer: 9 | compression: gzipaoeu 10 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/error-invalid-api-version.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/error-linger.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | producer: 9 | linger: "1" 10 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/error-secret-starts-with-number.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | secrets: 9 | - name: 1secret 10 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/error-secret-with-spaces.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | secrets: 9 | - name: secret name 10 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/error-version.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | name: my-test-mqtt 4 | type: mqtt 5 | topic: my-mqtt 6 | create_topic: false 7 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/full-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | producer: 9 | linger: 1ms 10 | batch-size: "44.0 MB" 11 | compression: gzip 12 | consumer: 13 | partition: 10 14 | max_bytes: "1 MB" 15 | secrets: 16 | - name: secret1 17 | transforms: 18 | - uses: infinyon/json-sql 19 | with: 20 | mapping: 21 | table: "topic_message" 22 | param: param_value 23 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/many-partitions.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.2.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: 7 | version: 0.1.0 8 | meta: 9 | name: test-topic 10 | consumer: 11 | partition: 12 | - 0 13 | - 1 14 | max_bytes: "1 MB" 15 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/simple.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.1.0 4 | name: my-test-mqtt 5 | type: mqtt 6 | topic: my-mqtt 7 | create_topic: false 8 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/with_transform.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | version: 0.1.0 3 | name: my-test-mqtt 4 | type: mqtt 5 | topic: my-mqtt 6 | create_topic: false 7 | transforms: 8 | - uses: infinyon/sql 9 | lookback: 10 | last: 11 | 100 12 | age: 1h 13 | with: 14 | mapping: 15 | table: "topic_message" 16 | map-columns: 17 | "device_id": 18 | json-key: "device.device_id" 19 | value: 20 | type: "int" 21 | default: 0 22 | required: true 23 | "record": 24 | json-key: "$" 25 | value: 26 | type: "jsonb" 27 | required: true 28 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/with_transform_invalid.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 0.1.0 2 | meta: 3 | version: 0.3.3 4 | name: cat-facts 5 | type: http-source 6 | topic: cat-facts 7 | create-topic: true 8 | http: 9 | endpoint: "https://catfact.ninja/fact" 10 | interval: 10s 11 | transforms: 12 | - uses: infinyon/jolt@0.1.0 13 | with: 14 | - spec: 15 | - operation: default 16 | spec: 17 | source: "http-connector" 18 | - operation: remove 19 | spec: 20 | length: "" -------------------------------------------------------------------------------- /crates/fluvio-connector-package/test-data/connectors/with_transform_many.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | version: 0.1.0 3 | name: my-test-mqtt 4 | type: mqtt 5 | topic: my-mqtt 6 | create_topic: false 7 | transforms: 8 | - uses: infinyon/json-sql 9 | with: 10 | mapping: 11 | table: "topic_message" 12 | - uses: infinyon/avro-sql 13 | - uses: infinyon/regex-filter 14 | with: 15 | regex: "\\w" 16 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/tests/Connector.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "json-test-connector" 3 | group = "fluvio" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | fluvio = "0.10.0" 7 | description = "Generate JSON generator" 8 | license = "Apache-2.0" 9 | visibility = "public" 10 | 11 | [direction] 12 | source = true 13 | 14 | [deployment] 15 | binary = "json-test-connector" 16 | 17 | [secret.password] 18 | type = "env" 19 | 20 | [secret.my_cert] 21 | type = "file" 22 | mount = "/mydata/secret1" 23 | 24 | [custom] 25 | required = ["template"] 26 | 27 | [custom.properties.template] 28 | title = "template" 29 | description = "JSON template" 30 | type = "string" 31 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/tests/sample-http.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | version: latest 3 | name: cat-facts 4 | type: http-source 5 | topic: cat-facts 6 | create-topic: true 7 | http: 8 | endpoint: "https://catfact.ninja/fact" 9 | interval: 10s 10 | 11 | -------------------------------------------------------------------------------- /crates/fluvio-connector-package/tests/sample-mqtt.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | version: latest 3 | name: my-mqtt-connector 4 | type: mqtt-source 5 | topic: mqtt-topic 6 | create-topic: true 7 | mqtt: 8 | url: "mqtt://test.mosquitto.org/" 9 | topic: "mqtt-to-fluvio" 10 | client_id: "my_mqtt" 11 | timeout: 12 | secs: 30 13 | nanos: 0 14 | payload_output_type: json 15 | 16 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/README.md: -------------------------------------------------------------------------------- 1 | # flv-metadata 2 | 3 | Fluvio metadata 4 | 5 | ## License 6 | 7 | This project is licensed under the [Apache license](LICENSE-APACHE). 8 | 9 | ### Contribution 10 | 11 | Unless you explicitly state otherwise, any contribution intentionally submitted 12 | for inclusion in Fluvio by you, shall be licensed as Apache, without any additional 13 | terms or conditions. 14 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/src/message/mod.rs: -------------------------------------------------------------------------------- 1 | mod messages; 2 | mod msg_type; 3 | 4 | pub use messages::*; 5 | pub use msg_type::*; 6 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/src/smartmodule/status.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # SmartModule Status 3 | //! 4 | //! SmartModule Status metadata information cached locally. 5 | //! 6 | use std::fmt; 7 | 8 | use fluvio_protocol::{Encoder, Decoder}; 9 | 10 | #[derive(Default, Decoder, Encoder, Debug, Clone, Eq, PartialEq)] 11 | #[cfg_attr( 12 | feature = "use_serde", 13 | derive(serde::Serialize, serde::Deserialize), 14 | serde(rename_all = "camelCase") 15 | )] 16 | pub struct SmartModuleStatus; 17 | 18 | impl fmt::Display for SmartModuleStatus { 19 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 20 | write!(f, "SmartModuleStatus") 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/src/spg/k8/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Cluster 3 | //! 4 | //! Interface to the Cluster metadata in K8 key value store 5 | //! 6 | mod spec; 7 | 8 | pub use self::spec::*; 9 | 10 | mod ext { 11 | use crate::k8_types::Status as K8Status; 12 | use crate::spg::SpuGroupStatus; 13 | 14 | /// implement k8 status for spu group status because they are same 15 | impl K8Status for SpuGroupStatus {} 16 | } 17 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/src/spg/store.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Spu Group 3 | //! 4 | 5 | use crate::store::*; 6 | 7 | use super::*; 8 | 9 | pub type SpuGroupMetadata = MetadataStoreObject; 10 | 11 | pub type SpuGroupLocalStore = LocalStore; 12 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/src/spu/k8.rs: -------------------------------------------------------------------------------- 1 | use crate::k8_types::{Crd, GROUP, V1, CrdNames, Spec, Status, DefaultHeader}; 2 | 3 | use super::SpuStatus; 4 | use super::SpuSpec; 5 | 6 | const SPU_API: Crd = Crd { 7 | group: GROUP, 8 | version: V1, 9 | names: CrdNames { 10 | kind: "Spu", 11 | plural: "spus", 12 | singular: "spu", 13 | }, 14 | }; 15 | 16 | impl Spec for SpuSpec { 17 | type Status = SpuStatus; 18 | type Header = DefaultHeader; 19 | 20 | fn metadata() -> &'static Crd { 21 | &SPU_API 22 | } 23 | } 24 | 25 | impl Status for SpuStatus {} 26 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/SmartModule.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "MyCustomModule" 3 | group = "infinyon" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | description = "My Custom module" 7 | license = "Apache-2.0" 8 | repository = "https://github.com/infinyon/fluvio" 9 | 10 | 11 | 12 | [[params]] 13 | name = "multiplier" 14 | description = "multiply input" 15 | 16 | 17 | [[params]] 18 | name = "scaler" 19 | description = "scaling factor" 20 | optional = true 21 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/SmartModule.yaml: -------------------------------------------------------------------------------- 1 | name: "MyCustomModule" 2 | description: "My Custom module" 3 | group: "InfinyOn" 4 | version: 0.1.0 5 | api_version: 0.1.0 6 | params: 7 | multiplier: 8 | description: "Use for multiplying value" 9 | required: true -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/k8_mirror_v1.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "fluvio.infinyon.com/v1", 3 | "kind": "Mirror", 4 | "metadata": { 5 | "creationTimestamp": "2023-08-16T17:36:58Z", 6 | "generation": 1, 7 | "name": "offshore", 8 | "namespace": "default", 9 | "resourceVersion": "237352", 10 | "uid": "0115f1f9-a17b-446a-8b9d-45ac67a4627b" 11 | }, 12 | "spec": { 13 | "mirrorType": { 14 | "remote": { 15 | "id": "offshore-edge-1" 16 | } 17 | } 18 | } 19 | } 20 | 21 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/k8_mirror_v1.yaml: -------------------------------------------------------------------------------- 1 | # full V2 Kubernets CRD 2 | # 3 | apiVersion: fluvio.infinyon.com/v1 4 | kind: Mirror 5 | metadata: 6 | name: onshore 7 | spec: 8 | mirrorType: 9 | home: 10 | id: onshore-core-1 11 | remoteId: offshore-edge-1 12 | publicEndpoint: fluvio.com:9000 13 | 14 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/sm_k8_v1.yaml: -------------------------------------------------------------------------------- 1 | # full V1 Kubernets CRD 2 | apiVersion: fluvio.infinyon.com/v1 3 | kind: SmartModule 4 | metadata: 5 | name: module1 6 | namespace: default 7 | spec: 8 | input_kind: Stream 9 | output_kind: Stream 10 | wasm: 11 | format: BINARY 12 | payload: H4sIAAAAAAAA/+y9B2AVVfY/PjN35r15LXlpJJB -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/sm_k8_v2.yaml: -------------------------------------------------------------------------------- 1 | # full V2 Kubernets CRD 2 | # 3 | apiVersion: fluvio.infinyon.com/v2 4 | kind: SmartModule 5 | metadata: 6 | name: module1 7 | namespace: default 8 | spec: 9 | meta: 10 | package: 11 | name: module1 12 | group: infinyon 13 | version: 0.1.0 14 | apiVersion: 0.1.0 15 | description: "This is a test module" 16 | params: 17 | - name: multipler 18 | description: "multipler" 19 | - name: scaler 20 | description: "used for scaling" 21 | optional: true 22 | wasm: 23 | format: BINARY 24 | # Many hands make light work (not real WASM but for testing) 25 | payload: TWFueSBoYW5kcyBtYWtlIGxpZ2h0IHdvcmsu -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/topic_assignment.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "fluvio.infinyon.com/v2", 3 | "kind": "Topic", 4 | "metadata": { 5 | "name": "test3", 6 | "namespace": "default" 7 | }, 8 | "spec": { 9 | "compressionType": "Any", 10 | "replicas": { 11 | "assigned": [ 12 | { 13 | "id": 0, 14 | "replicas": [ 15 | 5001, 16 | 5002 17 | ] 18 | } 19 | ] 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /crates/fluvio-controlplane-metadata/tests/topic_computed.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: fluvio.infinyon.com/v2 2 | kind: Topic 3 | metadata: 4 | name: test3 5 | namespace: default 6 | spec: 7 | compressionType: Any 8 | replicas: 9 | computed: 10 | ignoreRackAssignment: false 11 | partitions: 1 12 | replicationFactor: 1 13 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-controlplane" 3 | description = "API for Fluvio Control Plane" 4 | version = "0.0.0" 5 | publish = false 6 | repository.workspace = true 7 | edition.workspace = true 8 | license.workspace = true 9 | authors.workspace = true 10 | 11 | [lib] 12 | name = "fluvio_controlplane" 13 | path = "src/lib.rs" 14 | 15 | [dependencies] 16 | tracing = { workspace = true } 17 | 18 | # Fluvio dependencies 19 | fluvio-types = { workspace = true } 20 | fluvio-controlplane-metadata = { workspace = true } 21 | fluvio-protocol = { workspace = true, features = ["api"]} 22 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod sc_api; 2 | pub mod spu_api; 3 | pub mod replica; 4 | pub mod message; 5 | pub mod requests; 6 | 7 | pub use alias::*; 8 | mod alias { 9 | use fluvio_controlplane_metadata::{store::MetadataStoreObject, partition::PartitionSpec}; 10 | 11 | pub type PartitionMetadata = MetadataStoreObject; 12 | } 13 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/src/message/mod.rs: -------------------------------------------------------------------------------- 1 | mod replica_msg; 2 | 3 | pub use self::replica_msg::{ReplicaMsgs, ReplicaMsg}; 4 | pub use self::smartmodule_msg::{SmartModuleMsgs, SmartModuleMsg}; 5 | 6 | pub use spu_msg::*; 7 | 8 | mod spu_msg { 9 | use fluvio_controlplane_metadata::{spu::SpuSpec, message::Message}; 10 | 11 | pub type SpuMsg = Message; 12 | } 13 | 14 | mod smartmodule_msg { 15 | use fluvio_controlplane_metadata::message::{Message, Messages}; 16 | 17 | use crate::spu_api::update_smartmodule::SmartModule; 18 | 19 | pub type SmartModuleMsg = Message; 20 | pub type SmartModuleMsgs = Messages; 21 | } 22 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/src/message/replica_msg.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Replica Messages 3 | //! 4 | //! Replicas are sent from SC to all live SPUs that participate in this replica group. 5 | //! This message is sent for any changes in the live replica group. 6 | //! 7 | //! UPDATE/DEL operation is computed at sender by comparing KV notification with 8 | //! internal metadata cache. Receiver translates UPDATE operations into an ADD/DEL 9 | //! operation the comparing message with internal metadata. 10 | //! 11 | 12 | use fluvio_controlplane_metadata::message::{Message, Messages}; 13 | 14 | use crate::replica::Replica; 15 | 16 | pub type ReplicaMsg = Message; 17 | pub type ReplicaMsgs = Messages; 18 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/src/requests/mod.rs: -------------------------------------------------------------------------------- 1 | mod request; 2 | pub use self::request::*; 3 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/src/sc_api/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | pub mod register_spu; 3 | pub mod remove; 4 | pub mod update_lrs; 5 | pub mod update_mirror; 6 | pub mod update_partition; 7 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/src/spu_api/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | pub mod update_replica; 3 | pub mod update_smartmodule; 4 | pub mod update_spu; 5 | pub mod update_mirror; 6 | -------------------------------------------------------------------------------- /crates/fluvio-controlplane/src/spu_api/update_spu.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::api::Request; 2 | use fluvio_protocol::Decoder; 3 | use fluvio_protocol::Encoder; 4 | use fluvio_controlplane_metadata::spu::SpuSpec; 5 | 6 | use crate::requests::ControlPlaneRequest; 7 | 8 | use super::api::InternalSpuApi; 9 | 10 | pub type UpdateSpuRequest = ControlPlaneRequest; 11 | 12 | impl Request for UpdateSpuRequest { 13 | const API_KEY: u16 = InternalSpuApi::UpdateSpu as u16; 14 | type Response = UpdateSpuResponse; 15 | } 16 | 17 | #[derive(Decoder, Encoder, Default, Debug)] 18 | pub struct UpdateSpuResponse {} 19 | -------------------------------------------------------------------------------- /crates/fluvio-extension-common/README.md: -------------------------------------------------------------------------------- 1 | # Fluvio Consumer Extension 2 | 3 | 4 | -------------------------------------------------------------------------------- /crates/fluvio-extension-common/src/common/mod.rs: -------------------------------------------------------------------------------- 1 | mod hex_dump; 2 | 3 | pub mod time; 4 | 5 | use clap::Parser; 6 | 7 | pub use self::hex_dump::*; 8 | 9 | use crate::output::OutputType; 10 | 11 | #[derive(Debug, Parser, Default, Clone)] 12 | pub struct OutputFormat { 13 | /// Output 14 | #[arg( 15 | default_value_t, 16 | short = 'O', 17 | long = "output", 18 | value_name = "type", 19 | value_enum, 20 | ignore_case = true 21 | )] 22 | pub format: OutputType, 23 | } 24 | -------------------------------------------------------------------------------- /crates/fluvio-extension-common/src/common/time.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use chrono::{DateTime, Utc}; 3 | use timeago::Formatter; 4 | 5 | pub fn time_elapsed(date: DateTime) -> Result { 6 | let now = Utc::now(); 7 | let time_fmt = Formatter::new(); 8 | let dur = now.signed_duration_since(date).to_std()?; 9 | 10 | Ok(time_fmt.convert(dur)) 11 | } 12 | -------------------------------------------------------------------------------- /crates/fluvio-hub-protocol/README.md: -------------------------------------------------------------------------------- 1 | # fluvio-hub-util-protocol 2 | Includes reusable types from the Package Hub 3 | -------------------------------------------------------------------------------- /crates/fluvio-hub-protocol/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod errors; 2 | mod package_meta; 3 | 4 | pub mod constants; 5 | pub mod infinyon_tok; 6 | 7 | pub use errors::{Result, HubError}; 8 | pub use package_meta::{PackageMeta, PkgTag, PkgVisibility}; 9 | pub use package_meta::{validate_allowedchars, validate_noleading_punct}; 10 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | tests/*.tar.gz 3 | tests/*.tar 4 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/README.md: -------------------------------------------------------------------------------- 1 | 2 | Hub package util crate for access to smartmodule hub packages. 3 | 4 | The package-x.y.z.ipkg contains 5 | * package-meta file 6 | * manifest.tar.gz file containg smartmodule build files 7 | * Signature files 8 | 9 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/src/fvm/api/mod.rs: -------------------------------------------------------------------------------- 1 | mod client; 2 | mod download; 3 | 4 | pub use client::Client; 5 | pub use download::Download; 6 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/.gitignore: -------------------------------------------------------------------------------- 1 | example-0.0.1.ipkg 2 | hubutil_package_sign-keypair.pem 3 | hubutil_package_sign-pubkey.pem 4 | apackage/example-0.0.1.tar 5 | key_test.pem 6 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/Makefile: -------------------------------------------------------------------------------- 1 | 2 | # generate ssample id_ed25519 and id_ed25519.pub 3 | gen-keys: 4 | ssh-keygen -t ed25519 -C "sshkey@example.com" -f ./id_ed25519 -P "" 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/module.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-hub-util/tests/apackage/module.wasm -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-manifest-missing-file.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.1" 3 | name: example 4 | version: "0.0" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | manifest: 9 | - tests/apackage/module.wasm 10 | - doesnotexist.txt 11 | 12 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-v0.1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.1" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | manifest: 9 | - tests/apackage/module.wasm 10 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-v0.2-owner.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.2" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | private: true 9 | manifest: 10 | - tests/apackage/module.wasm 11 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-v0.2-public.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.2" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | private: false 9 | manifest: 10 | - tests/apackage/module.wasm 11 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-v0.2-visbad.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.2" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | visibility: not-a-keyword 9 | manifest: 10 | - tests/apackage/module.wasm 11 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-v0.3-notags.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.3" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | visibility: private 9 | manifest: 10 | - tests/apackage/module.wasm 11 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-v0.3-notags2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.3" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | visibility: private 9 | manifest: 10 | - tests/apackage/module.wasm 11 | tags: 12 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta-v0.3-targets.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.3" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | visibility: private 9 | manifest: 10 | - tests/apackage/module.wasm 11 | tags: 12 | - tag: arch 13 | value: aarch64-apple-darwin 14 | - tag: arch 15 | value: aarch64-unknown-linux-musl 16 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/apackage/package-meta.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | package_format_version: "0.3" 3 | name: example 4 | version: "0.0.1" 5 | group: infinyon 6 | description: Describe the module here 7 | license: e.g. Apache2 8 | visibility: private 9 | manifest: 10 | - module.wasm 11 | tags: 12 | - tag: arch 13 | value: aarch64-unknown-linux-gnu 14 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/hub_package_meta_rw_test.yaml: -------------------------------------------------------------------------------- 1 | package_format_version: '0.3' 2 | name: example 3 | version: 0.0.1 4 | group: infinyon 5 | description: Describe the module here 6 | license: e.g. Apache2 7 | manifest: 8 | - module.wasm 9 | repository_url: null 10 | tags: null 11 | visibility: private 12 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/id_ed25519: -------------------------------------------------------------------------------- 1 | -----BEGIN OPENSSH PRIVATE KEY----- 2 | b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW 3 | QyNTUxOQAAACAAhnhh1r7fxOtJPxS7g7ApAKywCGuJb8JNcIN5ySBiZQAAAJi5pAUtuaQF 4 | LQAAAAtzc2gtZWQyNTUxOQAAACAAhnhh1r7fxOtJPxS7g7ApAKywCGuJb8JNcIN5ySBiZQ 5 | AAAEBJXKjYGK5yjXn6zwfXMCbf8A8AgjVcIOA+Rg99MuerggCGeGHWvt/E60k/FLuDsCkA 6 | rLAIa4lvwk1wg3nJIGJlAAAAEnNzaGtleUBleGFtcGxlLmNvbQECAw== 7 | -----END OPENSSH PRIVATE KEY----- 8 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/id_ed25519.pub: -------------------------------------------------------------------------------- 1 | ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIACGeGHWvt/E60k/FLuDsCkArLAIa4lvwk1wg3nJIGJl sshkey@example.com 2 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/sm_need_more_data.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-hub-util/tests/sm_need_more_data.wasm -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/sm_ok.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-hub-util/tests/sm_ok.wasm -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/static-example-0.0.1.ipkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-hub-util/tests/static-example-0.0.1.ipkg -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/static-example-keypair.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | 8YtAe44SlUTDm2E81hSu4JiojbX2s00H0G9p76FIJFg= 3 | -----END PRIVATE KEY----- 4 | -------------------------------------------------------------------------------- /crates/fluvio-hub-util/tests/static-example-pubkey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | rCvytjVSS4GI9L93pLifYCrpCzuZyc8xSrN+2ENWISE= 3 | -----END PUBLIC KEY----- 4 | -------------------------------------------------------------------------------- /crates/fluvio-kv-storage/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-kv-storage" 3 | description = "Fluvio Key-Value Storage" 4 | version = "0.0.0" 5 | publish = false 6 | repository.workspace = true 7 | edition.workspace = true 8 | license.workspace = true 9 | authors.workspace = true 10 | 11 | [dependencies] 12 | anyhow = { workspace = true } 13 | tracing = { workspace = true } 14 | futures-util = { workspace = true } 15 | 16 | 17 | 18 | fluvio-protocol = { workspace = true } 19 | 20 | [dev-dependencies] 21 | fluvio-future = { workspace = true,features = ["fixture"] } 22 | -------------------------------------------------------------------------------- /crates/fluvio-kv-storage/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod log; 2 | 3 | use std::hash::Hash; 4 | 5 | use anyhow::Result; 6 | use fluvio_protocol::{Encoder, Decoder}; 7 | 8 | pub use log::{LogBasedKVStorage, Log}; 9 | 10 | #[allow(async_fn_in_trait)] 11 | pub trait KVStorage 12 | where 13 | K: Encoder + Decoder + Hash + Eq + Clone, 14 | V: Encoder + Decoder + Clone, 15 | { 16 | async fn get(&self, key: &K) -> Result>; 17 | 18 | async fn delete(&mut self, key: &K) -> Result<()>; 19 | 20 | async fn put(&mut self, key: impl Into, value: impl Into) -> Result<()>; 21 | 22 | async fn entries(&self) -> Result>; 23 | } 24 | -------------------------------------------------------------------------------- /crates/fluvio-package-index/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!( 3 | "cargo:rustc-env=PACKAGE_TARGET={}", 4 | std::env::var("TARGET").unwrap() 5 | ); 6 | } 7 | -------------------------------------------------------------------------------- /crates/fluvio-protocol-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-protocol-derive" 3 | description = "Procedure macro to encode/decode fluvio protocol" 4 | categories = ["encoding"] 5 | version.workspace = true 6 | repository.workspace = true 7 | edition.workspace = true 8 | license.workspace = true 9 | authors.workspace = true 10 | 11 | [lib] 12 | proc-macro = true 13 | doctest = false 14 | 15 | [dependencies] 16 | proc-macro2 = { workspace = true } 17 | quote = { workspace = true } 18 | tracing = { workspace = true } 19 | 20 | [dependencies.syn] 21 | version = "1.0.0" 22 | features = ["full"] 23 | 24 | 25 | -------------------------------------------------------------------------------- /crates/fluvio-protocol-derive/README.md: -------------------------------------------------------------------------------- 1 | # fluvio-protocol-derive 2 | 3 | Procedure macro to derive encode/decode fluvio protocol 4 | 5 | 6 | ## License 7 | 8 | This project is licensed under the [Apache license](LICENSE-APACHE). 9 | 10 | ### Contribution 11 | 12 | Unless you explicitly state otherwise, any contribution intentionally submitted 13 | for inclusion in Fluvio by you, shall be licensed as Apache, without any additional 14 | terms or conditions. -------------------------------------------------------------------------------- /crates/fluvio-protocol-derive/rustfmt.toml: -------------------------------------------------------------------------------- 1 | use_field_init_shorthand = true 2 | use_try_shorthand = true 3 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/Makefile: -------------------------------------------------------------------------------- 1 | RUSTV = stable 2 | TARGET_FLAG=$(if $(TARGET),--target $(TARGET),) 3 | 4 | build: 5 | cargo build --all-features $(TARGET_FLAG) 6 | 7 | test-all: test-crates 8 | 9 | test-crates: 10 | cargo test $(TARGET_FLAG) --all-features 11 | 12 | install-fmt: 13 | rustup component add rustfmt --toolchain $(RUSTV) 14 | 15 | check-fmt: install-fmt 16 | cargo +$(RUSTV) fmt -- --check 17 | 18 | 19 | install-clippy: 20 | rustup component add clippy --toolchain $(RUSTV) 21 | 22 | check-clippy: install-clippy 23 | cargo +$(RUSTV) clippy --all-targets --all-features -- -D warnings 24 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/mod.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn ui() { 3 | let t = trybuild::TestCases::new(); 4 | 5 | t.compile_fail("derive-test/ui/fail/*.rs"); 6 | t.pass("derive-test/ui/pass/*.rs"); 7 | } 8 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/fail/fails_missing_encode_discriminant_and_fluvio_tag_not_provided.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Decoder, Encoder}; 2 | 3 | #[derive(Clone, Default, Encoder, Decoder)] 4 | pub enum SmartModuleInvocationWasm { 5 | #[default] 6 | Predefined = 0, 7 | AdHoc = 1, 8 | } 9 | 10 | fn main() {} 11 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/fail/fails_missing_encode_discriminant_and_fluvio_tag_not_provided.stderr: -------------------------------------------------------------------------------- 1 | error: You must provide `fluvio(encode_discriminant)` if `fluvio(tag)` is not provided 2 | --> derive-test/ui/fail/fails_missing_encode_discriminant_and_fluvio_tag_not_provided.rs:5:5 3 | | 4 | 5 | #[default] 5 | | ^ 6 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/fail/fails_missing_fluvio_annotation_and_discriminant_on_enums.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Decoder, Encoder}; 2 | 3 | #[derive(Clone, Default, Encoder, Decoder)] 4 | pub enum SmartModuleInvocationWasm { 5 | #[default] 6 | Predefined, 7 | AdHoc, 8 | } 9 | 10 | fn main() {} 11 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/fail/fails_missing_fluvio_annotation_and_discriminant_on_enums.stderr: -------------------------------------------------------------------------------- 1 | error: You must either provide a `tag` or a discriminant for enum types deriving Encode/Decode 2 | --> derive-test/ui/fail/fails_missing_fluvio_annotation_and_discriminant_on_enums.rs:5:5 3 | | 4 | 5 | #[default] 5 | | ^ 6 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/fail/fails_missing_fluvio_annotation_with_tag_on_enums.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Decoder, Encoder}; 2 | 3 | #[derive(Clone, Default, Encoder, Decoder)] 4 | pub enum SmartModuleInvocationWasm { 5 | #[default] 6 | #[fluvio(min_version = 1)] 7 | Predefined, 8 | #[fluvio(min_version = 2)] 9 | AdHoc, 10 | } 11 | 12 | fn main() {} 13 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/fail/fails_missing_fluvio_annotation_with_tag_on_enums.stderr: -------------------------------------------------------------------------------- 1 | error: You must either provide a `tag` or a discriminant for enum types deriving Encode/Decode 2 | --> derive-test/ui/fail/fails_missing_fluvio_annotation_with_tag_on_enums.rs:5:5 3 | | 4 | 5 | #[default] 5 | | ^ 6 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/pass/works_with_discriminant_on_enums.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Decoder, Encoder}; 2 | 3 | #[derive(Clone, Default, Encoder, Decoder)] 4 | #[fluvio(encode_discriminant)] 5 | pub enum SmartModuleInvocationWasm { 6 | #[default] 7 | Predefined = 0, 8 | AdHoc = 1, 9 | } 10 | 11 | fn main() {} 12 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/pass/works_with_fluvio_tag_and_discriminant_on_enums.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Decoder, Encoder}; 2 | 3 | #[derive(Clone, Default, Encoder, Decoder)] 4 | pub enum SmartModuleInvocationWasm { 5 | #[default] 6 | #[fluvio(tag = 0)] 7 | Predefined = 0, 8 | #[fluvio(tag = 1)] 9 | AdHoc = 1, 10 | } 11 | 12 | fn main() {} 13 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/derive-test/ui/pass/works_with_fluvio_tag_on_enums.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Decoder, Encoder}; 2 | 3 | #[derive(Clone, Default, Encoder, Decoder)] 4 | pub enum SmartModuleInvocationWasm { 5 | #[default] 6 | #[fluvio(tag = 0)] 7 | Predefined, 8 | #[fluvio(tag = 1)] 9 | AdHoc, 10 | } 11 | 12 | fn main() {} 13 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/fixtures/smartmodule.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-protocol/fixtures/smartmodule.wasm -------------------------------------------------------------------------------- /crates/fluvio-protocol/send-b-client.sh: -------------------------------------------------------------------------------- 1 | # /bin/bash 2 | # send out simple message 3 | msg="$1" 4 | port=${2:-9092} 5 | xxd -r -p $msg - | nc -w 1 localhost $port | xxd -p -c 4 6 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/src/link/mod.rs: -------------------------------------------------------------------------------- 1 | mod error_code; 2 | pub mod smartmodule; 3 | pub mod versions; 4 | 5 | pub use error_code::*; 6 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/src/record/mod.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::len_without_is_empty)] 2 | #![allow(clippy::should_implement_trait)] 3 | 4 | mod data; 5 | pub use self::data::*; 6 | 7 | mod batch; 8 | mod replica; 9 | pub use batch::*; 10 | pub use replica::*; 11 | 12 | pub type Offset = i64; 13 | pub type Size = u32; 14 | pub type Size64 = u64; 15 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/test-client.sh: -------------------------------------------------------------------------------- 1 | # /bin/bash 2 | # send out simple message 3 | msg="$1" 4 | echo "$msg" | nc -w 1 localhost 9092 -------------------------------------------------------------------------------- /crates/fluvio-protocol/tests/default.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::FluvioDefault; 2 | 3 | #[derive(FluvioDefault, Debug)] 4 | struct TestRecord { 5 | _value: i8, 6 | _value2: i8, 7 | #[fluvio(default = "4")] 8 | value3: i8, 9 | #[fluvio(default = "-1")] 10 | value4: i16, 11 | } 12 | 13 | #[test] 14 | fn test_default() { 15 | let record = TestRecord::default(); 16 | assert_eq!(record.value3, 4); 17 | assert_eq!(record.value4, -1); 18 | } 19 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/tests/parse.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn derive_ui() { 3 | let t = trybuild::TestCases::new(); 4 | 5 | t.pass("ui-tests/pass_*.rs"); 6 | t.compile_fail("ui-tests/fail_*.rs"); 7 | } 8 | -------------------------------------------------------------------------------- /crates/fluvio-protocol/tests/test_old_record_empty_key.bin: -------------------------------------------------------------------------------- 1 | VVVVVVVVVV -------------------------------------------------------------------------------- /crates/fluvio-protocol/ui-tests/pass_derive_decode_encode_with_custom_results.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Encoder, Decoder}; 2 | 3 | pub type Result = std::result::Result<(), ()>; 4 | 5 | fn main() {} 6 | 7 | #[derive(Default, Decoder, Encoder)] 8 | struct PassTupleStruct(String); 9 | -------------------------------------------------------------------------------- /crates/fluvio-run/src/bin/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use fluvio_run::RunCmd; 3 | 4 | fn main() -> Result<(), Box> { 5 | let cmd: RunCmd = RunCmd::parse(); 6 | 7 | fluvio_future::subscriber::init_tracer(None); 8 | 9 | cmd.process()?; 10 | Ok(()) 11 | } 12 | -------------------------------------------------------------------------------- /crates/fluvio-run/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::io::Error as IoError; 2 | 3 | pub type Result = std::result::Result; 4 | 5 | #[derive(thiserror::Error, Debug)] 6 | pub enum RunnerError { 7 | #[error(transparent)] 8 | IoError(#[from] IoError), 9 | #[error("Unknown error: {0}")] 10 | Other(String), 11 | } 12 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/README.md: -------------------------------------------------------------------------------- 1 | # flv-api-sc 2 | 3 | Fluvio API for SC 4 | 5 | ## License 6 | 7 | This project is licensed under the [Apache license](LICENSE-APACHE). 8 | 9 | ### Contribution 10 | 11 | Unless you explicitly state otherwise, any contribution intentionally submitted 12 | for inclusion in Fluvio by you, shall be licensed as Apache, without any additional 13 | terms or conditions. 14 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/src/customspu/mod.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::spu::{CustomSpuSpec, CustomSpuKey}; 2 | 3 | use crate::AdminSpec; 4 | use crate::CreatableAdminSpec; 5 | use crate::DeletableAdminSpec; 6 | use crate::UpdatableAdminSpec; 7 | 8 | impl AdminSpec for CustomSpuSpec {} 9 | 10 | impl CreatableAdminSpec for CustomSpuSpec {} 11 | 12 | impl DeletableAdminSpec for CustomSpuSpec { 13 | type DeleteKey = CustomSpuKey; 14 | } 15 | 16 | impl UpdatableAdminSpec for CustomSpuSpec { 17 | type UpdateKey = CustomSpuKey; 18 | type UpdateAction = String; 19 | } 20 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/src/mirror/mod.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::mirror::*; 2 | 3 | use crate::{AdminSpec, CreatableAdminSpec, DeletableAdminSpec}; 4 | 5 | impl AdminSpec for MirrorSpec {} 6 | 7 | impl CreatableAdminSpec for MirrorSpec {} 8 | 9 | impl DeletableAdminSpec for MirrorSpec { 10 | type DeleteKey = String; 11 | } 12 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/src/objects/mod.rs: -------------------------------------------------------------------------------- 1 | mod create; 2 | mod delete; 3 | mod update; 4 | mod list; 5 | mod watch; 6 | mod metadata; 7 | 8 | // backward compatibility with classic protocol. this should go away once we deprecate classic 9 | pub mod classic; 10 | 11 | pub use create::*; 12 | pub use update::*; 13 | pub use delete::*; 14 | pub use list::*; 15 | pub use watch::*; 16 | pub use metadata::*; 17 | 18 | pub(crate) const COMMON_VERSION: i16 = 19; // from now, we use a single version for all objects 19 | pub(crate) const DYN_OBJ: i16 = 11; // version indicate dynamic object 20 | 21 | #[cfg(test)] 22 | mod test; 23 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/src/partition/mod.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::partition::*; 2 | 3 | mod convert { 4 | 5 | use crate::{AdminSpec}; 6 | use super::*; 7 | 8 | impl AdminSpec for PartitionSpec {} 9 | } 10 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/src/spg/mod.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::spg::*; 2 | 3 | mod convert { 4 | 5 | use crate::{AdminSpec, CreatableAdminSpec, DeletableAdminSpec, UpdatableAdminSpec}; 6 | use super::SpuGroupSpec; 7 | 8 | impl AdminSpec for SpuGroupSpec {} 9 | 10 | impl CreatableAdminSpec for SpuGroupSpec {} 11 | 12 | impl DeletableAdminSpec for SpuGroupSpec { 13 | type DeleteKey = String; 14 | } 15 | 16 | impl UpdatableAdminSpec for SpuGroupSpec { 17 | type UpdateKey = String; 18 | type UpdateAction = String; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/src/spu/mod.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::spu::{SpuSpec}; 2 | 3 | use crate::{AdminSpec}; 4 | 5 | impl AdminSpec for SpuSpec {} 6 | -------------------------------------------------------------------------------- /crates/fluvio-sc-schema/src/tableformat/mod.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::tableformat::*; 2 | 3 | mod convert { 4 | 5 | use crate::{CreatableAdminSpec, DeletableAdminSpec, UpdatableAdminSpec}; 6 | 7 | use crate::AdminSpec; 8 | use super::TableFormatSpec; 9 | 10 | impl AdminSpec for TableFormatSpec {} 11 | 12 | impl CreatableAdminSpec for TableFormatSpec {} 13 | 14 | impl DeletableAdminSpec for TableFormatSpec { 15 | type DeleteKey = String; 16 | } 17 | 18 | impl UpdatableAdminSpec for TableFormatSpec { 19 | type UpdateKey = String; 20 | type UpdateAction = String; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crates/fluvio-sc/README.md: -------------------------------------------------------------------------------- 1 | ## Run SC in development mode 2 | ``` 3 | ./target/debug/sc-server 4 | ``` 5 | 6 | ## Run SC with various debug flags 7 | 8 | To use different debug flags use: 9 | ``` 10 | ./dev-tools/log/debug-ctrl-all 11 | ./dev-tools/log/debug-ctrl-min 12 | ./dev-tools/log/debug-ctrl-client 13 | ./dev-tools/log/debug-ctrl-controller 14 | ``` 15 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/bin/main.rs: -------------------------------------------------------------------------------- 1 | #[global_allocator] 2 | static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; 3 | 4 | use clap::Parser; 5 | 6 | use fluvio_sc::cli::ScOpt; 7 | use fluvio_sc::start::main_loop; 8 | 9 | fn main() { 10 | fluvio_future::subscriber::init_tracer(None); 11 | 12 | let opt = ScOpt::parse(); 13 | main_loop(opt); 14 | } 15 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/config/mod.rs: -------------------------------------------------------------------------------- 1 | mod sc_config; 2 | 3 | pub use self::sc_config::ScConfig; 4 | pub use self::sc_config::ScConfigBuilder; 5 | pub use self::sc_config::DEFAULT_NAMESPACE; 6 | 7 | macro_rules! whitelist { 8 | ($config:expr,$name:expr,$start:expr) => { 9 | if $config.enabled($name) { 10 | $start; 11 | } 12 | }; 13 | } 14 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/controllers/mirroring/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod controller; 2 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/controllers/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod partitions; 2 | pub(crate) mod spus; 3 | pub(crate) mod topics; 4 | pub(crate) mod scheduler; 5 | pub(crate) mod mirroring; 6 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/controllers/partitions/mod.rs: -------------------------------------------------------------------------------- 1 | mod controller; 2 | mod reducer; 3 | 4 | pub use self::controller::*; 5 | pub use common::*; 6 | 7 | mod common { 8 | 9 | use fluvio_controlplane_metadata::{partition::PartitionSpec, store::k8::K8MetaItem}; 10 | use crate::stores::actions::WSAction; 11 | 12 | pub type PartitionWSAction = WSAction; 13 | } 14 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/controllers/scheduler/mod.rs: -------------------------------------------------------------------------------- 1 | mod partition; 2 | 3 | pub(crate) use partition::*; 4 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/controllers/spus/actions.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # SPU Actions 3 | //! 4 | //! 5 | 6 | use fluvio_types::SpuId; 7 | 8 | /// action for SPU controller to take 9 | #[derive(Debug, Eq, PartialEq, Clone)] 10 | pub struct SpuAction { 11 | pub id: SpuId, 12 | pub status: bool, 13 | } 14 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/controllers/spus/mod.rs: -------------------------------------------------------------------------------- 1 | mod actions; 2 | mod controller; 3 | 4 | pub use self::controller::*; 5 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/controllers/topics/mod.rs: -------------------------------------------------------------------------------- 1 | mod actions; 2 | mod reducer; 3 | pub(crate) mod controller; 4 | pub(crate) mod policy; 5 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/core/mod.rs: -------------------------------------------------------------------------------- 1 | mod context; 2 | pub use self::context::*; 3 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/k8/mod.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Initialization routines for Streaming Coordinator (SC) 3 | //! 4 | //! All processing engines are hooked-up here. Channels are created and split between sencders 5 | //! and receivers. 6 | //! 7 | 8 | pub(crate) mod controllers; 9 | mod objects; 10 | 11 | #[cfg(test)] 12 | mod fixture; 13 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/k8/objects/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod spg_group; 2 | pub mod spg_service; 3 | pub mod spu_k8_config; 4 | pub mod statefulset; 5 | pub mod spu_service; 6 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | pub mod config; 3 | pub mod k8; 4 | pub mod cli; 5 | pub mod core; 6 | pub mod start; 7 | 8 | pub mod stores; 9 | mod init; 10 | mod error; 11 | mod services; 12 | mod controllers; 13 | 14 | const VERSION: &str = include_str!("../../../VERSION"); 15 | 16 | pub mod dispatcher { 17 | pub use fluvio_stream_dispatcher::*; 18 | } 19 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/mod.rs: -------------------------------------------------------------------------------- 1 | // pub mod send_channels; 2 | mod public_api; 3 | mod private_api; 4 | 5 | pub mod auth; 6 | 7 | pub use public_api::start_public_server; 8 | pub use private_api::start_internal_server; 9 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/public_api/derivedstream/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/public_api/mirror/mod.rs: -------------------------------------------------------------------------------- 1 | mod register; 2 | mod unregister; 3 | mod list; 4 | 5 | pub use register::*; 6 | pub use unregister::*; 7 | pub use list::*; 8 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/public_api/smartmodule/mod.rs: -------------------------------------------------------------------------------- 1 | mod create; 2 | mod delete; 3 | mod list; 4 | 5 | pub(crate) use create::*; 6 | pub(crate) use delete::*; 7 | pub(crate) use list::*; 8 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/public_api/spg/mod.rs: -------------------------------------------------------------------------------- 1 | mod create; 2 | mod delete; 3 | mod fetch; 4 | 5 | pub use create::*; 6 | pub use fetch::*; 7 | pub use delete::*; 8 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/public_api/spu/mod.rs: -------------------------------------------------------------------------------- 1 | mod fetch; 2 | mod register_custom_spus_req; 3 | mod unregister_custom_spus_req; 4 | 5 | pub use fetch::*; 6 | pub use register_custom_spus_req::*; 7 | pub use unregister_custom_spus_req::*; 8 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/public_api/tableformat/mod.rs: -------------------------------------------------------------------------------- 1 | mod create; 2 | mod delete; 3 | 4 | pub use create::*; 5 | pub use delete::*; 6 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/services/public_api/topic/mod.rs: -------------------------------------------------------------------------------- 1 | mod create; 2 | mod delete; 3 | mod fetch; 4 | pub mod update; 5 | 6 | pub(crate) use create::*; 7 | pub(crate) use delete::*; 8 | pub(crate) use fetch::*; 9 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/stores/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod spu; 2 | pub mod topic; 3 | pub mod partition; 4 | pub mod spg; 5 | pub mod smartmodule; 6 | pub mod tableformat; 7 | 8 | pub use crate::dispatcher::store::*; 9 | 10 | pub mod actions { 11 | pub use crate::dispatcher::actions::*; 12 | } 13 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/stores/partition/mod.rs: -------------------------------------------------------------------------------- 1 | use fluvio_controlplane::PartitionMetadata; 2 | pub use fluvio_controlplane_metadata::partition::*; 3 | pub use fluvio_controlplane_metadata::store::k8::K8MetaItem; 4 | 5 | pub type PartitionAdminMd = PartitionMetadata; 6 | pub type PartitionAdminStore = PartitionLocalStore; 7 | 8 | mod policy; 9 | mod store; 10 | 11 | pub use store::*; 12 | pub use policy::*; 13 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/stores/smartmodule.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::smartmodule::*; 2 | pub use fluvio_controlplane_metadata::store::k8::K8MetaItem; 3 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/stores/spg.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::spg::*; 2 | pub use fluvio_controlplane_metadata::spg::store::*; 3 | pub use fluvio_controlplane_metadata::store::k8::K8MetaItem; 4 | 5 | pub type SpgAdminMd = SpuGroupMetadata; 6 | pub type SpgAdminStore = SpuGroupLocalStore; 7 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/stores/tableformat.rs: -------------------------------------------------------------------------------- 1 | pub use fluvio_controlplane_metadata::tableformat::*; 2 | pub use fluvio_controlplane_metadata::store::k8::K8MetaItem; 3 | -------------------------------------------------------------------------------- /crates/fluvio-sc/src/stores/topic/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod store; 2 | 3 | pub use store::*; 4 | pub use fluvio_controlplane_metadata::topic::*; 5 | pub use fluvio_controlplane_metadata::store::k8::K8MetaItem; 6 | 7 | pub type TopicAdminStore = TopicLocalStore; 8 | pub type TopicAdminMd = TopicMetadata; 9 | -------------------------------------------------------------------------------- /crates/fluvio-sc/test-data/auth_config/scopes.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": ["Root"], 3 | "user1": ["Default"] 4 | } -------------------------------------------------------------------------------- /crates/fluvio-sc/test-data/config/sc_invalid.toml: -------------------------------------------------------------------------------- 1 | unknown = "value" 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /crates/fluvio-sc/test-data/config/sc_min.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | 3 | [sc] 4 | id = 500 5 | 6 | [bind_public] 7 | host = "127.0.0.1" 8 | port = 9999 -------------------------------------------------------------------------------- /crates/fluvio-sc/test-data/config/sc_server.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | 3 | [sc] 4 | id = 500 5 | 6 | [bind_public] 7 | host = "127.0.0.1" 8 | port = 9999 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /crates/fluvio-sc/test-data/config/sc_tls.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | 3 | 4 | [bind_public] 5 | host = "127.0.0.1" 6 | port = 9999 7 | 8 | [tls] 9 | server_cert = "test-data/server.crt" 10 | server_key = "test-data/server.key" 11 | enable_client_cert = true 12 | ca_cert = "test-data/ca.crt" -------------------------------------------------------------------------------- /crates/fluvio-smartengine/README.md: -------------------------------------------------------------------------------- 1 | Fluvio SmartModule execution engine -------------------------------------------------------------------------------- /crates/fluvio-smartengine/src/engine/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(thiserror::Error, Debug)] 2 | pub enum EngineError { 3 | #[error("No valid smartmodule found")] 4 | UnknownSmartModule, 5 | #[error("Failed to instantiate: {0}")] 6 | Instantiate(anyhow::Error), 7 | #[error("Requested memory {requested}b exceeded max allowed {max}b")] 8 | StoreMemoryExceeded { 9 | current: usize, 10 | requested: usize, 11 | max: usize, 12 | }, 13 | } 14 | -------------------------------------------------------------------------------- /crates/fluvio-smartengine/src/engine/mod.rs: -------------------------------------------------------------------------------- 1 | //! SmartModule configuration 2 | 3 | mod config; 4 | mod error; 5 | mod wasmtime; 6 | 7 | #[cfg(test)] 8 | mod fixture; 9 | 10 | pub mod metrics; 11 | 12 | pub use error::EngineError; 13 | pub use config::{ 14 | SmartModuleConfig, SmartModuleConfigBuilder, SmartModuleConfigBuilderError, 15 | SmartModuleInitialData, Lookback, DEFAULT_SMARTENGINE_VERSION, 16 | }; 17 | 18 | pub type WasmSlice = (i32, i32, u32); 19 | 20 | /// SmartEngine Version 21 | pub type Version = i16; 22 | 23 | pub use self::wasmtime::{SmartEngine, SmartModuleChainBuilder, SmartModuleChainInstance}; 24 | -------------------------------------------------------------------------------- /crates/fluvio-smartengine/src/engine/wasmtime/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod memory; 2 | pub(crate) mod transforms; 3 | pub(crate) mod init; 4 | pub(crate) mod state; 5 | pub(crate) mod engine; 6 | pub(crate) mod instance; 7 | pub(crate) mod look_back; 8 | pub(crate) mod limiter; 9 | pub use engine::{SmartEngine, SmartModuleChainBuilder, SmartModuleChainInstance}; 10 | 11 | use super::*; 12 | -------------------------------------------------------------------------------- /crates/fluvio-smartengine/src/lib.rs: -------------------------------------------------------------------------------- 1 | cfg_if::cfg_if! { 2 | if #[cfg(feature = "engine")] { 3 | mod engine; 4 | pub use engine::*; 5 | } 6 | } 7 | 8 | #[cfg(feature = "transformation")] 9 | pub mod transformation; 10 | -------------------------------------------------------------------------------- /crates/fluvio-smartengine/testdata/transformation/empty.yaml: -------------------------------------------------------------------------------- 1 | transforms: [] -------------------------------------------------------------------------------- /crates/fluvio-smartmodule-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-derive" 3 | description = "Fluvio SmartModule WASM procedural macro" 4 | categories = ["wasm", "database", "encoding"] 5 | keywords = ["streaming", "stream", "wasm", "fluvio"] 6 | version.workspace = true 7 | repository.workspace = true 8 | edition.workspace = true 9 | license.workspace = true 10 | authors.workspace = true 11 | 12 | [lib] 13 | proc-macro = true 14 | 15 | [dependencies] 16 | syn = { workspace = true, features = ["full"] } 17 | quote = { workspace = true } 18 | proc-macro2 = { workspace = true } 19 | -------------------------------------------------------------------------------- /crates/fluvio-smartmodule-derive/src/util.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::Span; 2 | 3 | use crate::SmartModuleKind; 4 | 5 | pub(crate) fn generate_ident(kind: &SmartModuleKind) -> syn::Ident { 6 | syn::Ident::new(&kind.to_string(), Span::call_site()) 7 | } 8 | -------------------------------------------------------------------------------- /crates/fluvio-smartmodule/fixtures/smartmodule.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-smartmodule/fixtures/smartmodule.wasm -------------------------------------------------------------------------------- /crates/fluvio-smartmodule/tests/parse.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn derive_ui() { 3 | let t = trybuild::TestCases::check_only(); 4 | 5 | t.pass("ui-tests/pass_*.rs"); 6 | t.compile_fail("ui-tests/fail_*.rs"); 7 | } 8 | -------------------------------------------------------------------------------- /crates/fluvio-socket/README.md: -------------------------------------------------------------------------------- 1 | # fluvio-protocol 2 | 3 | Provide TCP socket wrapper for Fluvio protocol 4 | 5 | # build fluvio send utility 6 | 7 | build fluvio-send utility 8 | 9 | ```cargo build --all-features``` 10 | 11 | invoking fluvio-send utility to send api request sample data 12 | 13 | ```../target/debug/fluvio-send data/apirequest.bin``` 14 | 15 | ## License 16 | 17 | This project is licensed under the [Apache license](LICENSE-APACHE). 18 | 19 | ### Contribution 20 | 21 | Unless you explicitly state otherwise, any contribution intentionally submitted 22 | for inclusion in Fluvio by you, shall be licensed as Apache, without any additional 23 | terms or conditions. 24 | -------------------------------------------------------------------------------- /crates/fluvio-socket/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().expect("Failed to acquire build-time information"); 3 | 4 | // Copy VERSION file. Do not fail e.g. when built via `cargo publish` 5 | if let Ok(verpath) = std::fs::canonicalize("../../VERSION") { 6 | if verpath.exists() { 7 | println!("cargo:rerun-if-changed=../../VERSION"); 8 | } 9 | } 10 | println!("cargo:rerun-if-changed=build.rs"); 11 | } 12 | -------------------------------------------------------------------------------- /crates/fluvio-socket/certs/cert.conf: -------------------------------------------------------------------------------- 1 | [req] 2 | default_bits = 4096 3 | prompt = no 4 | default_md = sha256 5 | req_extensions = req_ext 6 | distinguished_name = dn 7 | 8 | [ dn ] 9 | C=US 10 | ST=CA 11 | L=Sunnyvale 12 | O=End Point 13 | OU=Testing Domain 14 | emailAddress=test@fluvio.io 15 | CN = test.fluvio.io 16 | 17 | [ req_ext ] 18 | subjectAltName = @alt_names 19 | 20 | [ alt_names ] 21 | DNS.1 = localhost -------------------------------------------------------------------------------- /crates/fluvio-socket/certs/certs/ca.srl: -------------------------------------------------------------------------------- 1 | 7B42AED25FBCE3CD876E0C8139FE2B3B84D49B44 2 | -------------------------------------------------------------------------------- /crates/fluvio-socket/certs/certs/client.pfx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-socket/certs/certs/client.pfx -------------------------------------------------------------------------------- /crates/fluvio-socket/certs/certs/server.pfx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-socket/certs/certs/server.pfx -------------------------------------------------------------------------------- /crates/fluvio-socket/certs/openssl-ca.cnf: -------------------------------------------------------------------------------- 1 | ######################### Req Section ################## 2 | # THIS IS USED BY REQ COMMAND 3 | 4 | [req] 5 | default_bits = 4096 6 | prompt = no 7 | default_md = sha256 8 | req_extensions = req_ext 9 | distinguished_name = dn 10 | 11 | [ dn ] 12 | C=US 13 | ST=CA 14 | L=Sunnyvale 15 | O=Fluvio 16 | OU=Eng 17 | CN = fluvio.io 18 | 19 | [ req_ext ] 20 | # subjectAltName = @alt_names 21 | 22 | [ ca ] 23 | default_ca = CA 24 | -------------------------------------------------------------------------------- /crates/fluvio-socket/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::io::Error as IoError; 2 | 3 | #[derive(thiserror::Error, Debug)] 4 | pub enum SocketError { 5 | #[error("Socket io {msg}")] 6 | Io { source: IoError, msg: String }, 7 | #[error("Socket closed")] 8 | SocketClosed, 9 | #[error("Socket is stale")] 10 | SocketStale, 11 | } 12 | 13 | impl From for SocketError { 14 | fn from(err: IoError) -> Self { 15 | let msg = err.to_string(); 16 | SocketError::Io { source: err, msg } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /crates/fluvio-socket/tests/enum.rs: -------------------------------------------------------------------------------- 1 | use std::convert::TryInto; 2 | 3 | use fluvio_protocol::derive::{Decoder, Encoder}; 4 | 5 | #[repr(u16)] 6 | #[derive(Encoder, Decoder, Eq, PartialEq, Debug, Clone, Copy)] 7 | #[fluvio(encode_discriminant)] 8 | #[derive(Default)] 9 | pub enum TestKafkaApiEnum { 10 | #[default] 11 | Echo = 1000, 12 | Status = 1001, 13 | } 14 | 15 | #[test] 16 | fn test_conversion() { 17 | let key: u16 = 1000; 18 | let key_enum: TestKafkaApiEnum = key.try_into().expect("conversion"); 19 | assert_eq!(key_enum, TestKafkaApiEnum::Echo); 20 | } 21 | -------------------------------------------------------------------------------- /crates/fluvio-socket/tests/test.txt: -------------------------------------------------------------------------------- 1 | hello -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/README.md: -------------------------------------------------------------------------------- 1 | # flv-api-spu 2 | 3 | Fluvio API for SPU 4 | 5 | ## License 6 | 7 | This project is licensed under the [Apache license](LICENSE-APACHE). 8 | 9 | ### Contribution 10 | 11 | Unless you explicitly state otherwise, any contribution intentionally submitted 12 | for inclusion in Fluvio by you, shall be licensed as Apache, without any additional 13 | terms or conditions. 14 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/src/client/api_key.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Encoder, Decoder}; 2 | 3 | /// Api Key for Spu Client API (from server to client) 4 | #[repr(u16)] 5 | #[derive(Eq, PartialEq, Debug, Encoder, Decoder, Clone, Copy)] 6 | #[fluvio(encode_discriminant)] 7 | pub enum SpuClientApiKey { 8 | ReplicaOffsetUpdate = 1001, 9 | } 10 | 11 | impl Default for SpuClientApiKey { 12 | fn default() -> Self { 13 | Self::ReplicaOffsetUpdate 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/src/client/mod.rs: -------------------------------------------------------------------------------- 1 | mod api_key; 2 | mod api; 3 | pub mod offset; 4 | 5 | pub use self::api_key::*; 6 | pub use self::api::SpuClientRequest; 7 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/src/fetch/mod.rs: -------------------------------------------------------------------------------- 1 | mod request; 2 | mod response; 3 | 4 | pub use request::*; 5 | pub use response::*; 6 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/src/isolation.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Encoder, Decoder}; 2 | use serde::{Serialize, Deserialize}; 3 | 4 | #[derive(Debug, Encoder, Decoder, Clone, Copy, Eq, PartialEq, Serialize, Deserialize, Hash)] 5 | #[fluvio(encode_discriminant)] 6 | #[repr(u8)] 7 | #[derive(Default)] 8 | pub enum Isolation { 9 | #[default] 10 | ReadUncommitted = 0, 11 | ReadCommitted = 1, 12 | } 13 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::assign_op_pattern)] 2 | 3 | pub mod server; 4 | pub mod client; 5 | pub mod fetch; 6 | pub mod produce; 7 | mod isolation; 8 | 9 | #[cfg(feature = "file")] 10 | pub mod file; 11 | 12 | pub mod errors { 13 | pub use fluvio_protocol::link::ErrorCode; 14 | } 15 | 16 | pub use fluvio_protocol::link::versions::{ApiVersions, ApiVersionsRequest, ApiVersionsResponse}; 17 | pub use isolation::*; 18 | 19 | /// Default API version for all API 20 | pub const COMMON_VERSION: i16 = 25; 21 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/src/produce/mod.rs: -------------------------------------------------------------------------------- 1 | mod request; 2 | mod response; 3 | 4 | pub use request::*; 5 | pub use response::*; 6 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/src/server/mod.rs: -------------------------------------------------------------------------------- 1 | mod api_key; 2 | 3 | #[cfg(feature = "file")] 4 | mod api; 5 | pub mod smartmodule; 6 | pub mod fetch_offset; 7 | pub mod stream_fetch; 8 | pub mod update_offset; 9 | pub mod consumer_offset; 10 | pub mod mirror; 11 | 12 | pub use self::api_key::*; 13 | 14 | #[cfg(feature = "file")] 15 | pub use self::api::SpuServerRequest; 16 | -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/tests/produce_request.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-spu-schema/tests/produce_request.bin -------------------------------------------------------------------------------- /crates/fluvio-spu-schema/tests/test_old_record_empty_key.bin: -------------------------------------------------------------------------------- 1 | VVVVVVVVVV -------------------------------------------------------------------------------- /crates/fluvio-spu/Makefile: -------------------------------------------------------------------------------- 1 | FLUVIO_BIN=target/debug/fluvio 2 | SPU=2 3 | REPLICATION=2 4 | 5 | fluvio: 6 | cargo build --bin fluvio 7 | 8 | # make -f src/spu/Makefile test_replica 9 | test_replica: fluvio 10 | $(FLUVIO_BIN) cluster delete --force 11 | $(FLUVIO_BIN) cluster start --spu $(SPU) --local --develop --rust-log=fluvio_spu=debug 12 | $(FLUVIO_BIN) topic create test --replication $(REPLICATION) 13 | echo "hello world" | $(FLUVIO_BIN) produce test 14 | sleep 2 15 | $(FLUVIO_BIN) partition list 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /crates/fluvio-spu/README.md: -------------------------------------------------------------------------------- 1 | ## Running SPU in development mode 2 | 3 | To run SPU with default parameters. 4 | ``` 5 | ./target/debug/fluvio-spu 6 | ``` 7 | 8 | Run server with debug flag 9 | ``` 10 | RUST_LOG=debug ./target/debug/fluvio-spu 11 | ``` -------------------------------------------------------------------------------- /crates/fluvio-spu/src/config/mod.rs: -------------------------------------------------------------------------------- 1 | mod cli; 2 | mod spu_config; 3 | 4 | pub use self::cli::SpuOpt; 5 | 6 | pub use self::spu_config::{SpuConfig, ReplicationConfig}; 7 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/control_plane/action.rs: -------------------------------------------------------------------------------- 1 | use fluvio_controlplane_metadata::partition::ReplicaKey; 2 | 3 | #[allow(dead_code)] 4 | #[derive(Debug)] 5 | pub enum SupervisorCommand { 6 | ReplicaLeaderTerminated(ReplicaKey), 7 | } 8 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/control_plane/mod.rs: -------------------------------------------------------------------------------- 1 | //! Responsible for communication with SC 2 | 3 | mod dispatcher; 4 | mod action; 5 | mod message_sink; 6 | 7 | pub use dispatcher::ScDispatcher; 8 | 9 | pub use message_sink::*; 10 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/core/mirror.rs: -------------------------------------------------------------------------------- 1 | use fluvio_controlplane::spu_api::update_mirror::Mirror; 2 | use std::sync::Arc; 3 | 4 | use crate::core::Spec; 5 | use crate::core::LocalStore; 6 | 7 | pub type MirrorLocalStore = LocalStore; 8 | 9 | pub type SharedMirrorLocalStore = Arc; 10 | 11 | impl Spec for Mirror { 12 | const LABEL: &'static str = "Mirror"; 13 | 14 | type Key = String; 15 | 16 | fn key(&self) -> &Self::Key { 17 | &self.name 18 | } 19 | 20 | fn key_owned(&self) -> Self::Key { 21 | self.name.clone() 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/core/replica/metadata.rs: -------------------------------------------------------------------------------- 1 | use fluvio_controlplane::replica::Replica; 2 | use fluvio_protocol::record::ReplicaKey; 3 | 4 | use crate::core::Spec; 5 | use crate::core::LocalStore; 6 | 7 | impl Spec for Replica { 8 | const LABEL: &'static str = "Replica"; 9 | 10 | type Key = ReplicaKey; 11 | 12 | fn key(&self) -> &Self::Key { 13 | &self.id 14 | } 15 | 16 | fn key_owned(&self) -> Self::Key { 17 | self.id.clone() 18 | } 19 | } 20 | 21 | pub type ReplicaStore = LocalStore; 22 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/core/replica/mod.rs: -------------------------------------------------------------------------------- 1 | mod metadata; 2 | 3 | pub use self::metadata::ReplicaStore; 4 | 5 | use std::sync::Arc; 6 | 7 | pub type SharedReplicaLocalStore = Arc; 8 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/core/smartmodule/mod.rs: -------------------------------------------------------------------------------- 1 | mod metadata; 2 | 3 | pub use self::metadata::SmartModuleLocalStore; 4 | 5 | use std::sync::Arc; 6 | 7 | pub type SharedSmartModuleLocalStore = Arc; 8 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/core/spus/mod.rs: -------------------------------------------------------------------------------- 1 | mod metadata; 2 | 3 | pub use self::metadata::SpuLocalStore; 4 | 5 | use std::sync::Arc; 6 | 7 | pub type SharedSpuLocalStore = Arc; 8 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/core/storage/mod.rs: -------------------------------------------------------------------------------- 1 | use fluvio_storage::config::ConfigOption; 2 | use fluvio_storage::FileReplica; 3 | use fluvio_storage::StorageError; 4 | use fluvio_controlplane_metadata::partition::ReplicaKey; 5 | use fluvio_types::SpuId; 6 | 7 | use crate::config::Log; 8 | 9 | 10 | /* 11 | pub async fn clear_replica_storage(local_spu: SpuId, replica: &ReplicaKey, config: &Log) { 12 | let storage_config = config.as_storage_config(); 13 | let config = default_config(local_spu, &storage_config); 14 | FileReplica::clear(replica, &config).await 15 | } 16 | */ 17 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/error.rs: -------------------------------------------------------------------------------- 1 | use async_channel::SendError; 2 | use fluvio_types::PartitionError; 3 | use fluvio_storage::StorageError; 4 | use fluvio_socket::SocketError; 5 | 6 | #[derive(Debug, thiserror::Error)] 7 | pub enum InternalServerError { 8 | #[error("Storage error")] 9 | Storage(#[from] StorageError), 10 | #[error("Partition error")] 11 | Partition(#[from] PartitionError), 12 | #[error("Socket error")] 13 | Socket(#[from] SocketError), 14 | #[error("Channel send error")] 15 | Send(String), 16 | } 17 | 18 | impl From> for InternalServerError { 19 | fn from(error: SendError) -> Self { 20 | InternalServerError::Send(error.to_string()) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/kv/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod consumer; 2 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/main.rs: -------------------------------------------------------------------------------- 1 | #[global_allocator] 2 | static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; 3 | 4 | use clap::Parser; 5 | 6 | fn main() { 7 | fluvio_future::subscriber::init_tracer(None); 8 | 9 | let opt = fluvio_spu::SpuOpt::parse(); 10 | fluvio_spu::main_loop(opt); 11 | } 12 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/mirroring/home/api_key.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Encoder, Decoder}; 2 | 3 | #[repr(u16)] 4 | #[derive(Eq, PartialEq, Debug, Encoder, Decoder, Clone, Copy, Default)] 5 | #[fluvio(encode_discriminant)] 6 | pub enum MirrorHomeApiEnum { 7 | #[default] 8 | UpdateHomeOffset = 0, 9 | SyncRecords = 1, 10 | } 11 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/mirroring/home/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod connection; 2 | pub(crate) mod api_key; 3 | pub(crate) mod home_api; 4 | pub(crate) mod update_offsets; 5 | pub(crate) mod sync; 6 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/mirroring/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod remote; 2 | pub(crate) mod home; 3 | 4 | #[cfg(test)] 5 | mod test; 6 | 7 | const COMMON_MIRROR_VERSION: i16 = 1; 8 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/mirroring/remote/api_key.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Encoder, Decoder}; 2 | 3 | #[repr(u16)] 4 | #[derive(Eq, PartialEq, Debug, Encoder, Decoder, Clone, Copy, Default)] 5 | #[fluvio(encode_discriminant)] 6 | pub enum MirrorRemoteApiEnum { 7 | #[default] 8 | SyncRecords = 0, 9 | UpdateEdgeOffset = 1, 10 | } 11 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/mirroring/remote/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod controller; 2 | pub(crate) mod api_key; 3 | pub(crate) mod remote_api; 4 | pub(crate) mod sync; 5 | pub(crate) mod update_offsets; 6 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/mirroring/test/mod.rs: -------------------------------------------------------------------------------- 1 | mod integration; 2 | mod fixture; 3 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/replication/follower/api_key.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Encoder, Decoder}; 2 | 3 | #[repr(u16)] 4 | #[derive(Eq, PartialEq, Debug, Encoder, Decoder, Clone, Copy)] 5 | #[fluvio(encode_discriminant)] 6 | pub enum FollowerPeerApiEnum { 7 | SyncRecords = 0, 8 | RejectedOffsetRequest = 1, 9 | } 10 | 11 | impl Default for FollowerPeerApiEnum { 12 | fn default() -> Self { 13 | Self::SyncRecords 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/replication/follower/mod.rs: -------------------------------------------------------------------------------- 1 | mod state; 2 | mod api_key; 3 | mod peer_api; 4 | mod controller; 5 | mod reject_request; 6 | pub mod sync; 7 | 8 | pub use self::state::{FollowersState, SharedFollowersState, FollowerReplicaState}; 9 | pub use self::reject_request::RejectOffsetRequest; 10 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/replication/leader/api_key.rs: -------------------------------------------------------------------------------- 1 | use fluvio_protocol::{Encoder, Decoder}; 2 | 3 | #[repr(u16)] 4 | #[derive(Eq, PartialEq, Debug, Encoder, Decoder, Clone, Copy)] 5 | #[fluvio(encode_discriminant)] 6 | pub enum LeaderPeerApiEnum { 7 | UpdateOffsets = 0, 8 | } 9 | 10 | impl Default for LeaderPeerApiEnum { 11 | fn default() -> Self { 12 | Self::UpdateOffsets 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/replication/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod follower; 2 | pub(crate) mod leader; 3 | 4 | #[cfg(test)] 5 | pub(crate) mod test; 6 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/services/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod public; 2 | 3 | pub mod auth; 4 | pub mod internal; 5 | 6 | pub use self::internal::create_internal_server; 7 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/services/public/conn_context.rs: -------------------------------------------------------------------------------- 1 | use crate::services::public::StreamPublishers; 2 | 3 | #[derive(Debug)] 4 | pub(crate) struct ConnectionContext { 5 | stream_publishers: StreamPublishers, 6 | } 7 | 8 | impl ConnectionContext { 9 | pub(crate) fn new() -> Self { 10 | Self { 11 | stream_publishers: StreamPublishers::new(), 12 | } 13 | } 14 | 15 | pub(crate) fn stream_publishers(&self) -> &StreamPublishers { 16 | &self.stream_publishers 17 | } 18 | 19 | pub(crate) fn stream_publishers_mut(&mut self) -> &mut StreamPublishers { 20 | &mut self.stream_publishers 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crates/fluvio-spu/src/services/public/tests/test_data/filter_missing_attribute.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/fluvio-spu/src/services/public/tests/test_data/filter_missing_attribute.wasm -------------------------------------------------------------------------------- /crates/fluvio-storage/Makefile: -------------------------------------------------------------------------------- 1 | build-storage-cli: 2 | cargo build --release --features="cli fluvio-future/subscriber" -p fluvio-storage -------------------------------------------------------------------------------- /crates/fluvio-storage/README.md: -------------------------------------------------------------------------------- 1 | # Building storage cli 2 | 3 | From root of the project 4 | ``` 5 | make build-storage-cli 6 | ``` 7 | 8 | and set alias for storage-cli 9 | ``` 10 | alias scli=$(PWD)/target/release/storage-cli 11 | ``` 12 | 13 | 14 | # Running storage cli 15 | 16 | ``` 17 | scli log /tmp/fluvio/spu-logs-5001/t1-0/00000000000000000000.log 18 | ``` -------------------------------------------------------------------------------- /crates/fluvio-stream-dispatcher/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod store; 2 | pub mod dispatcher; 3 | pub mod actions; 4 | 5 | mod error; 6 | pub mod metadata; 7 | 8 | pub use error::StoreError; 9 | 10 | pub mod core { 11 | pub use fluvio_stream_model::core::*; 12 | } 13 | 14 | #[cfg(feature = "k8")] 15 | pub use fluvio_stream_model::k8_types; 16 | -------------------------------------------------------------------------------- /crates/fluvio-stream-model/src/epoch/mod.rs: -------------------------------------------------------------------------------- 1 | mod dual_epoch_map; 2 | mod epoch_map; 3 | 4 | pub use epoch_map::*; 5 | pub use dual_epoch_map::*; 6 | -------------------------------------------------------------------------------- /crates/fluvio-test-case-derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-test-case-derive" 3 | description = "Fluvio Test - TestCase derive macro" 4 | version = "0.0.0" 5 | publish = false 6 | repository.workspace = true 7 | edition.workspace = true 8 | license.workspace = true 9 | authors.workspace = true 10 | 11 | [lib] 12 | proc-macro = true 13 | 14 | [dependencies] 15 | proc-macro2 = { workspace = true } 16 | quote = { workspace = true } 17 | syn = { workspace = true, features = ["full"]} 18 | 19 | [dev-dependencies] 20 | trybuild = {git = "https://github.com/infinyon/trybuild", branch = "check_option"} 21 | -------------------------------------------------------------------------------- /crates/fluvio-test-case-derive/tests/parse.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn derive_ui() { 3 | let t = trybuild::TestCases::check_only(); 4 | 5 | t.pass("ui-tests/pass_*.rs"); 6 | t.compile_fail("ui-tests/fail_*.rs"); 7 | } 8 | -------------------------------------------------------------------------------- /crates/fluvio-test-case-derive/ui-tests/fail_enum.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_case_derive::MyTestCase; 2 | 3 | #[derive(MyTestCase)] 4 | pub enum AnyEnum {} 5 | 6 | fn main() { 7 | 8 | } -------------------------------------------------------------------------------- /crates/fluvio-test-case-derive/ui-tests/pass_basic_empty.rs: -------------------------------------------------------------------------------- 1 | 2 | use fluvio_test_case_derive::MyTestCase; 3 | 4 | #[derive(MyTestCase)] 5 | pub struct BareCaseTestOption {} 6 | 7 | fn main() { 8 | 9 | } -------------------------------------------------------------------------------- /crates/fluvio-test-case-derive/ui-tests/pass_basic_not_empty.rs: -------------------------------------------------------------------------------- 1 | 2 | use fluvio_test_case_derive::MyTestCase; 3 | 4 | #[derive(MyTestCase)] 5 | pub struct SimpleTestOption { 6 | num: u8, 7 | } 8 | 9 | fn main() { 10 | 11 | } -------------------------------------------------------------------------------- /crates/fluvio-test-derive/tests/parse.rs: -------------------------------------------------------------------------------- 1 | #[test] 2 | fn derive_ui() { 3 | let t = trybuild::TestCases::check_only(); 4 | 5 | t.pass("ui-tests/pass_*.rs"); 6 | t.compile_fail("ui-tests/fail_*.rs"); 7 | } 8 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/fail_async.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[allow(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | 5 | #[fluvio_test(async = 1)] 6 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) { 7 | } 8 | 9 | #[fluvio_test(async = a)] 10 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) { 11 | } 12 | 13 | #[fluvio_test(async = "true")] 14 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) { 15 | } 16 | 17 | #[fluvio_test(async = "false")] 18 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) { 19 | } 20 | 21 | fn main() { 22 | } 23 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/fail_cluster_type.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[allow(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | 5 | #[fluvio_test(cluster_type="not-a-type")] 6 | pub fn test1(mut test_driver: TestDriver, test_case: TestCase) { 7 | } 8 | 9 | // Wrong attr type (MetaList), but values would have been valid 10 | #[fluvio_test(cluster_type=["k8", "local"])] 11 | pub fn test2(mut test_driver: TestDriver, test_case: TestCase) { 12 | } 13 | 14 | 15 | fn main() { 16 | } 17 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/fail_invalid_key.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[allow(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | 5 | #[fluvio_test(fail)] 6 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) { 7 | } 8 | 9 | fn main() { 10 | } 11 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/fail_min_spu_type.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[allow(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | 5 | #[fluvio_test(min_spu = a)] 6 | pub fn test1(mut test_driver: TestDriver, test_case: TestCase) { 7 | } 8 | 9 | #[fluvio_test(min_spu = "1")] 10 | pub fn test2(mut test_driver: TestDriver, test_case: TestCase) { 11 | } 12 | 13 | fn main() { 14 | } 15 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/fail_multi.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[allow(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | 5 | #[fluvio_test(min_spu = a, topic = 2)] 6 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) { 7 | } 8 | 9 | fn main() { 10 | } 11 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/fail_name.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[allow(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | 5 | #[fluvio_test(name = 1)] 6 | pub fn test1(mut test_driver: TestDriver, test_case: TestCase) { 7 | } 8 | 9 | #[fluvio_test(name = unquoted)] 10 | pub fn test2(mut test_driver: TestDriver, test_case: TestCase) { 11 | } 12 | 13 | fn main() { 14 | } 15 | 16 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/fail_topic_type.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[allow(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | 5 | #[fluvio_test(topic = 1)] 6 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) { 7 | } 8 | 9 | fn main() { 10 | } 11 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/pass_min_spu.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[warn(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | use clap::Parser; 5 | use std::any::Any; 6 | use fluvio_test_util::test_meta::TestOption; 7 | 8 | #[derive(Debug, Clone, Parser, Default, PartialEq)] 9 | #[clap(name = "Fluvio Test Example")] 10 | pub struct RunTestOption {} 11 | 12 | impl TestOption for RunTestOption { 13 | fn as_any(&self) -> &dyn Any { 14 | self 15 | } 16 | } 17 | 18 | #[fluvio_test(min_spu = 2)] 19 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) {} 20 | 21 | fn main() {} 22 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/pass_multi.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[warn(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | use clap::Parser; 5 | use std::any::Any; 6 | use fluvio_test_util::test_meta::TestOption; 7 | 8 | #[derive(Debug, Clone, Parser, Default, PartialEq)] 9 | #[clap(name = "Fluvio Test Example")] 10 | pub struct RunTestOption {} 11 | 12 | impl TestOption for RunTestOption { 13 | fn as_any(&self) -> &dyn Any { 14 | self 15 | } 16 | } 17 | 18 | #[fluvio_test(min_spu = 2, topic = "test")] 19 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) {} 20 | 21 | fn main() {} 22 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/pass_name.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[warn(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | use clap::Parser; 5 | use std::any::Any; 6 | use fluvio_test_util::test_meta::TestOption; 7 | 8 | #[derive(Debug, Clone, Parser, Default, PartialEq)] 9 | #[clap(name = "Fluvio Test Example")] 10 | pub struct TestNameTestOption {} 11 | 12 | impl TestOption for TestNameTestOption { 13 | fn as_any(&self) -> &dyn Any { 14 | self 15 | } 16 | } 17 | 18 | #[fluvio_test(name = "test_name")] 19 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) {} 20 | 21 | fn main() {} 22 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/pass_no_macro_args.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[warn(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | use clap::Parser; 5 | use std::any::Any; 6 | use fluvio_test_util::test_meta::TestOption; 7 | 8 | #[derive(Debug, Clone, Parser, Default, PartialEq)] 9 | #[clap(name = "Fluvio Test Example")] 10 | pub struct RunTestOption {} 11 | 12 | impl TestOption for RunTestOption { 13 | fn as_any(&self) -> &dyn Any { 14 | self 15 | } 16 | } 17 | 18 | #[fluvio_test] 19 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) {} 20 | 21 | fn main() {} 22 | -------------------------------------------------------------------------------- /crates/fluvio-test-derive/ui-tests/pass_topic.rs: -------------------------------------------------------------------------------- 1 | use fluvio_test_derive::fluvio_test; 2 | #[warn(unused_imports)] 3 | use fluvio_test_util::test_meta::TestCase; 4 | use clap::Parser; 5 | use std::any::Any; 6 | use fluvio_test_util::test_meta::TestOption; 7 | 8 | #[derive(Debug, Clone, Parser, Default, PartialEq)] 9 | #[clap(name = "Fluvio Test Example")] 10 | pub struct RunTestOption {} 11 | 12 | impl TestOption for RunTestOption { 13 | fn as_any(&self) -> &dyn Any { 14 | self 15 | } 16 | } 17 | 18 | #[fluvio_test(topic = "test")] 19 | pub fn run(mut test_driver: TestDriver, test_case: TestCase) {} 20 | 21 | fn main() {} 22 | -------------------------------------------------------------------------------- /crates/fluvio-test-util/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod setup; 2 | pub mod test_runner; 3 | pub mod tls; 4 | 5 | pub mod test_meta; 6 | use once_cell::sync::Lazy; 7 | 8 | static VERSION: Lazy = Lazy::new(|| { 9 | let version = include_str!("../../VERSION"); 10 | match option_env!("FLUVIO_VERSION_SUFFIX") { 11 | Some(suffix) => format!("{version}-{suffix}"), 12 | None => version.to_string(), 13 | } 14 | }); 15 | -------------------------------------------------------------------------------- /crates/fluvio-test-util/test_meta/test_timer.rs: -------------------------------------------------------------------------------- 1 | use std::time::{Duration, Instant}; 2 | use std::fmt::Debug; 3 | 4 | #[derive(Debug, Clone)] 5 | pub struct TestTimer { 6 | pub start_time: Instant, 7 | pub duration: Option, 8 | } 9 | 10 | impl TestTimer { 11 | pub fn start() -> Self { 12 | TestTimer { 13 | start_time: Instant::now(), 14 | duration: None, 15 | } 16 | } 17 | 18 | pub fn stop(&mut self) { 19 | self.duration = Some(self.start_time.elapsed()); 20 | } 21 | 22 | pub fn duration(&self) -> Duration { 23 | self.duration.expect("Timer is still running") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /crates/fluvio-test-util/test_runner/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod test_meta; 2 | pub mod test_driver; 3 | -------------------------------------------------------------------------------- /crates/fluvio-test/src/tests/concurrent/util.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | use md5::Digest; 3 | 4 | type Record = Vec; 5 | 6 | pub fn rand_record() -> Record { 7 | let len: u16 = rand::random(); 8 | let record: Vec = (0..len).map(|_| rand::random::()).collect(); 9 | record 10 | } 11 | 12 | pub fn hash_messages(messages: &[String]) -> String { 13 | let mut hasher = md5::Md5::new(); 14 | for m in messages.iter() { 15 | hasher.update(m); 16 | } 17 | format!("{:X?}", hasher.finalize()) 18 | } 19 | 20 | pub fn hash_record(record: &[u8]) -> String { 21 | format!("{:X}", md5::Md5::digest(record)) 22 | } 23 | -------------------------------------------------------------------------------- /crates/fluvio-test/src/tests/multiple_partitions/producer.rs: -------------------------------------------------------------------------------- 1 | use fluvio::RecordKey; 2 | use fluvio_test_util::test_runner::test_driver::TestDriver; 3 | use fluvio_test_util::test_meta::environment::EnvDetail; 4 | 5 | use super::MyTestCase; 6 | 7 | pub async fn producer(test_driver: TestDriver, option: MyTestCase) { 8 | let producer = test_driver 9 | .create_producer(&option.environment.base_topic_name()) 10 | .await; 11 | 12 | let iterations: u16 = 10000; 13 | println!("Producing {iterations} records"); 14 | for i in 0..iterations { 15 | let record = i.to_string(); 16 | producer.send(RecordKey::NULL, record).await.unwrap(); 17 | } 18 | producer.flush().await.unwrap(); 19 | } 20 | -------------------------------------------------------------------------------- /crates/fluvio-test/src/tests/stress.rs: -------------------------------------------------------------------------------- 1 | use crate::TestOption; 2 | 3 | const MAX_COUNT: usize = 1000; 4 | 5 | /// in this, we fetch first, then send outs 1000 message and fetch back 6 | pub async fn stress_test(option: &TestOption) { 7 | 8 | println!("stress test with count: {}",MAX_COUNT); 9 | 10 | // first get batchs 11 | 12 | } -------------------------------------------------------------------------------- /crates/fluvio-types/src/compression.rs: -------------------------------------------------------------------------------- 1 | use schemars::JsonSchema; 2 | 3 | use serde::{Serialize, Deserialize}; 4 | 5 | #[derive(Clone, Debug, Deserialize, Eq, Serialize, PartialEq, JsonSchema)] 6 | #[serde(rename_all = "lowercase")] 7 | pub enum Compression { 8 | None, 9 | Gzip, 10 | Snappy, 11 | Lz4, 12 | Zstd, 13 | } 14 | -------------------------------------------------------------------------------- /crates/fluvio-version-manager/.gitignore: -------------------------------------------------------------------------------- 1 | !/fixtures/* 2 | -------------------------------------------------------------------------------- /crates/fluvio-version-manager/build.rs: -------------------------------------------------------------------------------- 1 | use std::env::var; 2 | 3 | fn main() { 4 | println!( 5 | "cargo:rustc-env=TARGET={}", 6 | var("TARGET").expect("The `TARGET` environment variable is not present") 7 | ); 8 | } 9 | -------------------------------------------------------------------------------- /crates/fluvio-version-manager/fixtures/version/0.10.14/fluvio: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Fluvio Test Executable" 3 | -------------------------------------------------------------------------------- /crates/fluvio-version-manager/fixtures/version/0.10.14/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "channel": { 3 | "tag": "0.10.14" 4 | }, 5 | "version": "0.10.14" 6 | } -------------------------------------------------------------------------------- /crates/fluvio-version-manager/fixtures/version/0.10.15/fluvio: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Fluvio Test Executable 0.10.15" 3 | -------------------------------------------------------------------------------- /crates/fluvio-version-manager/fixtures/version/0.10.15/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "channel": { 3 | "tag": "0.10.15" 4 | }, 5 | "version": "0.10.15" 6 | } -------------------------------------------------------------------------------- /crates/fluvio-version-manager/fixtures/version/stable/fluvio: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Fluvio Test Executable STABLE" 3 | -------------------------------------------------------------------------------- /crates/fluvio-version-manager/fixtures/version/stable/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "channel": "stable", 3 | "version": "0.10.16" 4 | } -------------------------------------------------------------------------------- /crates/fluvio-version-manager/src/command/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod current; 2 | pub mod install; 3 | pub mod itself; 4 | pub mod list; 5 | pub mod switch; 6 | pub mod uninstall; 7 | pub mod update; 8 | pub mod version; 9 | -------------------------------------------------------------------------------- /crates/fluvio-version-manager/tests/readme.md: -------------------------------------------------------------------------------- 1 | 2 | # install-fvm.sh Notes 3 | 4 | Trial download script for fvm 5 | 6 | 7 | -------------------------------------------------------------------------------- /crates/fluvio/VERSION: -------------------------------------------------------------------------------- 1 | ../../VERSION -------------------------------------------------------------------------------- /crates/fluvio/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | // Copy VERSION file. Do not fail e.g. when built via `cargo publish` 3 | if let Ok(verpath) = std::fs::canonicalize("./VERSION") { 4 | if verpath.exists() { 5 | println!("cargo:rerun-if-changed=./VERSION"); 6 | } 7 | } 8 | println!("cargo:rerun-if-changed=build.rs"); 9 | } 10 | -------------------------------------------------------------------------------- /crates/fluvio/src/config/mod.rs: -------------------------------------------------------------------------------- 1 | #[allow(clippy::module_inception)] 2 | mod config; 3 | mod tls; 4 | mod cluster; 5 | 6 | pub use config::*; 7 | pub use tls::*; 8 | pub use cluster::*; 9 | -------------------------------------------------------------------------------- /crates/fluvio/test-data/profiles/default.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | 3 | [sc] 4 | host = "127.0.0.1" 5 | port = 9033 6 | 7 | [spu] 8 | host = "127.0.0.1" 9 | port = 9034 10 | 11 | [kf] 12 | host = "127.0.0.1" 13 | port = 9093 14 | -------------------------------------------------------------------------------- /crates/fluvio/test-data/profiles/invalid.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | 3 | [sc] 4 | host = "127.0.0.1" 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /crates/fluvio/test-data/profiles/updatable_config.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | current_profile = "local" 3 | 4 | [profile.local] 5 | cluster = "updated" 6 | 7 | [cluster.updated] 8 | endpoint = "127.0.0.1" 9 | use_spu_local_address = false 10 | 11 | [cluster.updated.tls] 12 | tls_policy = "disabled" 13 | 14 | [cluster.updated.metadata.installation] 15 | type = "local" 16 | -------------------------------------------------------------------------------- /crates/fluvio/test-data/tls/file.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | current_profile = "local" 3 | [profile.local] 4 | cluster = "local" 5 | [cluster.local] 6 | addr = "localhost:9003" 7 | 8 | [cluster.local.tls] 9 | tls_policy = "verified" 10 | tls_source = "files" 11 | 12 | [cluster.local.tls.certs] 13 | key = "/tmp/client.key" 14 | cert = "/tmp/client.cert" 15 | ca_cert = "/tmp/ca.cert" 16 | domain = "my_domain" 17 | -------------------------------------------------------------------------------- /crates/fluvio/test-data/tls/inline.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | current_profile = "local" 3 | [profile.local] 4 | cluster = "local" 5 | [cluster.local] 6 | addr = "localhost:9003" 7 | 8 | [cluster.local.tls] 9 | tls_policy = "verified" 10 | tls_source = "inline" 11 | 12 | [cluster.local.tls.certs] 13 | key = "ABCDEFF" 14 | cert = "JJJJ" 15 | ca_cert = "XXXXX" 16 | domain = "my_domain" 17 | -------------------------------------------------------------------------------- /crates/fluvio/test-data/tls/no-verf.toml: -------------------------------------------------------------------------------- 1 | version = "1.0" 2 | current_profile = "local" 3 | [profile.local] 4 | cluster = "local" 5 | [cluster.local] 6 | addr = "localhost:9003" 7 | 8 | [cluster.local.tls] 9 | tls_policy = "disabled" 10 | -------------------------------------------------------------------------------- /crates/smartmodule-development-kit/src/clean.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Debug; 2 | 3 | use anyhow::Result; 4 | use clap::Parser; 5 | 6 | use cargo_builder::cargo::Cargo; 7 | 8 | /// Calls cargo to clean the current working directory. 9 | #[derive(Debug, Parser)] 10 | pub struct CleanCmd { 11 | /// Extra arguments to be passed to cargo 12 | #[arg(raw = true)] 13 | extra_arguments: Vec, 14 | } 15 | 16 | impl CleanCmd { 17 | pub(crate) fn process(self) -> Result<()> { 18 | let cargo = Cargo::clean() 19 | .extra_arguments(self.extra_arguments) 20 | .build()?; 21 | 22 | cargo.run() 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /crates/smartmodule-development-kit/src/init.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/infinyon/fluvio/cb52df8e36a28b8fc97881edad6eb3777dbb5494/crates/smartmodule-development-kit/src/init.rs -------------------------------------------------------------------------------- /dev-tools/PUBLISH.md: -------------------------------------------------------------------------------- 1 | # Steps to publish 2 | 3 | bump up top level `VERSION` 4 | 5 | run: `make updatee_version 6 | 7 | publish all crates 8 | 9 | # run CI workflow 10 | 11 | Run `Release` dispatch workflow -------------------------------------------------------------------------------- /dev-tools/log/debug-sc-all: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | RUST_LOG=kf=trace,fluvio=trace cargo run --bin fluvio-sc-k8 -- $@ 3 | -------------------------------------------------------------------------------- /dev-tools/log/debug-sc-client: -------------------------------------------------------------------------------- 1 | RUST_LOG=sc_core=debug,cluster_cache=debug,k8_client=trace,tokio_reactor=error target/debug/fluvio-sc-k8 2 | -------------------------------------------------------------------------------- /dev-tools/log/debug-sc-min: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | RUST_LOG=kf=debug,fluvio=debug cargo run --bin fluvio-sc-k8 -- $@ 3 | -------------------------------------------------------------------------------- /dev-tools/log/debug-sc-min-k8: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # debug sc with spu 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 4 | RELATIVE_PATH="$DIR"/../../target/debug 5 | COMMIT="$(git log -1 --pretty=format:"%H" )" 6 | SPU_IMAGE=infinyon/fluvio:"$COMMIT" 7 | echo $SPU_IMAGE 8 | RUST_LOG=kf=debug,sc=debug,flv=debug SPU_IMAGE="$SPU_IMAGE" "$RELATIVE_PATH"/fluvio-sc-k8 $@ 9 | -------------------------------------------------------------------------------- /dev-tools/log/debug-sc-server: -------------------------------------------------------------------------------- 1 | RUST_LOG=sc_core=trace,k8_config=debug,k8_client=debug,tokio_reactor=error target/debug/fluvio-sc-k8 2 | -------------------------------------------------------------------------------- /dev-tools/log/debug-spu-all: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ARG1=${1} 3 | ARG2=${2} 4 | ARG3=${3} 5 | 6 | RUST_LOG=kf=trace,fluvio=trace cargo run --bin fluvio-spu -- -i "$ARG1" -p 0.0.0.0:"$ARG2" -v 0.0.0.0:"$ARG3" 7 | -------------------------------------------------------------------------------- /dev-tools/log/debug-spu-min: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | RELATIVE_PATH="$DIR"/../../target/debug 4 | 5 | ARG1=${1} 6 | ARG2=${2} 7 | ARG3=${3} 8 | 9 | RUST_LOG=debug cargo run --bin fluvio-spu -- -i "$ARG1" -p 0.0.0.0:"$ARG2" -v 0.0.0.0:"$ARG3" 10 | -------------------------------------------------------------------------------- /dev-tools/log/test-sc-min: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ARG1=${1} 3 | RUST_LOG=sc_core=trace,future_helper=error,tokio_reactor=error cargo test "$ARG1" 4 | -------------------------------------------------------------------------------- /dev-tools/log/test-spu-min: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ARG1=${1} 3 | RUST_LOG=spu_server=trace,future_helper=error,tokio_reactor=error cargo test "$ARG1" 4 | -------------------------------------------------------------------------------- /dev-tools/sc-pod-log.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl logs "$@" `(kubectl get pod -l app=fluvio-sc -o jsonpath="{.items[0].metadata.name}")` -------------------------------------------------------------------------------- /examples/00-produce/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "produce" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [dependencies] 9 | tokio = { workspace = true } 10 | anyhow = { workspace = true } 11 | 12 | fluvio = { workspace = true } -------------------------------------------------------------------------------- /examples/01-produce-batch/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "produce-batch" 3 | version = "0.1.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | tokio = { workspace = true } 12 | anyhow = { workspace = true } 13 | 14 | fluvio = { workspace = true } -------------------------------------------------------------------------------- /examples/01-produce-key-value/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "produce-key-value" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | anyhow = { workspace = true } 12 | tokio = { workspace = true } 13 | tracing-subscriber = { workspace = true } 14 | 15 | fluvio = { workspace = true } 16 | -------------------------------------------------------------------------------- /examples/02-consume/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "consume" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | tokio = { workspace = true } 12 | futures-lite = { workspace = true } 13 | anyhow = { workspace = true } 14 | 15 | fluvio = { workspace = true } 16 | -------------------------------------------------------------------------------- /examples/03-echo/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "echo" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | futures = { workspace = true } 12 | tokio = { workspace = true } 13 | anyhow = { workspace = true } 14 | 15 | fluvio = { workspace = true } -------------------------------------------------------------------------------- /examples/04-admin-watch/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "admin" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | tokio = { workspace = true } 12 | futures = { workspace = true } 13 | 14 | 15 | fluvio = { workspace = true , features = ["unstable"] } -------------------------------------------------------------------------------- /examples/docker-compose/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | RUN apt-get update 4 | RUN apt-get install -y curl unzip 5 | RUN curl -fsS https://hub.infinyon.cloud/install/install.sh?ctx=dc | bash 6 | 7 | ENV PATH "$PATH:/root/.fluvio/bin" 8 | ENV PATH "$PATH:/root/.fvm/bin" 9 | -------------------------------------------------------------------------------- /examples/partitioning-simple/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "partitioning-simple" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [dependencies] 9 | anyhow = { workspace = true } 10 | fluvio = { workspace = true } -------------------------------------------------------------------------------- /examples/tests/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | RELEASE=$1 5 | 6 | cargo build $RELEASE --bin fluvio 7 | cargo build $RELEASE --bin produce 8 | cargo build $RELEASE --bin produce-key-value 9 | cargo build $RELEASE --bin consume 10 | cargo build $RELEASE --bin echo 11 | -------------------------------------------------------------------------------- /examples/tests/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | RELEASE=$1 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 6 | 7 | # Run tests 8 | "$DIR/test-simple.sh" "$RELEASE" 9 | "$DIR/test-echo.sh" "$RELEASE" 10 | -------------------------------------------------------------------------------- /examples/tests/test-echo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | RELEASE=$1 5 | 6 | cargo run $RELEASE --bin fluvio -- topic delete echo || true 7 | cargo run $RELEASE --bin fluvio -- topic create echo 8 | 9 | cargo run $RELEASE --bin echo 10 | -------------------------------------------------------------------------------- /k8-util/DEVELOP.md: -------------------------------------------------------------------------------- 1 | ## Trace K8 API 2 | 3 | Set up Local Proxy to minikube. It's easy to confingure for other K8 cluster 4 | as well: 5 | 6 | ```bash 7 | kubectl proxy --context=minikube 8 | Starting to serve on 127.0.0.1:8001 9 | ``` 10 | 11 | Add proxy cluster configuration to `~/.kube/config` 12 | 13 | ```yml 14 | clusters: 15 | - cluster: 16 | server: http://localhost:8001 17 | name: proxy 18 | 19 | contexts: 20 | - context: 21 | cluster: proxy 22 | user: minikube 23 | name: proxy 24 | ``` 25 | 26 | Use `tcpdump` to inspect: 27 | 28 | ```bash 29 | sudo tcpdump -i any port 8001 -A 30 | ``` 31 | -------------------------------------------------------------------------------- /k8-util/cluster/cluster-type.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # return type of cluster: minikube,k3d,aws 3 | set -e 4 | nodes=`kubectl get nodes -o=jsonpath='{.items[0].metadata.name}'` 5 | 6 | 7 | if echo ${nodes} | grep -q minikube; then 8 | echo 'minikube' 9 | elif echo ${nodes} | grep -q k3d; then 10 | echo "k3d" 11 | elif echo ${nodes} | grep -q control-plane; then 12 | echo "kind" 13 | elif echo ${nodes} | grep -q microk8s; then 14 | echo "microk8" 15 | elif echo ${nodes} | grep -q lima; then 16 | echo "lima" 17 | else 18 | echo "unknown" 19 | fi 20 | -------------------------------------------------------------------------------- /k8-util/cluster/kind.sh: -------------------------------------------------------------------------------- 1 | kind create cluster --config k8-util/cluster/kind.yaml 2 | make build_k8_image 3 | flvd cluster start --k8 --develop --proxy-addr 127.0.0.1 4 | -------------------------------------------------------------------------------- /k8-util/cluster/kind.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | nodes: 4 | - role: control-plane 5 | # port forward 80 on the host to 80 on this node 6 | extraPortMappings: 7 | - containerPort: 30003 #sc 8 | hostPort: 30003 9 | protocol: TCP 10 | - containerPort: 30004 #spu 1 11 | hostPort: 30004 12 | protocol: TCP 13 | - containerPort: 30005 #spu 2 14 | hostPort: 30005 15 | protocol: TCP 16 | - containerPort: 30006 #spu 3 17 | hostPort: 31005 18 | protocol: TCP 19 | -------------------------------------------------------------------------------- /k8-util/cluster/reset-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # restart docker if you have issue with installing helm chart on minikube running docker 3 | set -e 4 | sudo service restart docker -------------------------------------------------------------------------------- /k8-util/cluster/reset-k3d.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # delete and re-install k3d cluster ready for fluvio 3 | # this defaults to docker and assume you have have sudo access 4 | set -e 5 | 6 | K8_VERSION=${K8_VERSION:-v1.26.3} 7 | 8 | k3d cluster delete fluvio 9 | k3d cluster create fluvio --image rancher/k3s:${K8_VERSION}-k3s1 -------------------------------------------------------------------------------- /k8-util/cluster/reset-kind.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # delete and re-install minikube ready for fluvio 3 | # this defaults to docker and assume you have have sudo access 4 | set -e 5 | kind delete cluster 6 | kind create cluster -------------------------------------------------------------------------------- /k8-util/cluster/start-minikube-mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Hyperkit doesn't work for M1 4 | # It also is not reliable in Github Actions 5 | # Only reliable way to run is use of kind in Docker 6 | set -x 7 | brew install minikube 8 | brew install hyperkit 9 | minikube config set memory 1024 10 | minikube start --driver=hyperkit 11 | kubectl get nodes -------------------------------------------------------------------------------- /k8-util/dashboard/README.md: -------------------------------------------------------------------------------- 1 | # Deploy Components 2 | 3 | 4 | Deploy customized metric server to disable TLS certificate check for kind. 5 | 6 | ``` 7 | kubectl apply -f metric-server.yaml 8 | kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.4.0/aio/deploy/recommended.yaml 9 | kubectl apply -f user.yaml 10 | ``` 11 | 12 | # Start proxy 13 | 14 | Get token from running this script 15 | 16 | ``` 17 | $ ./token.sh 18 | ``` 19 | 20 | Then run proxy: 21 | ``` 22 | kubectl proxy 23 | ``` 24 | 25 | # Open dashboard on browser 26 | 27 | Open browser and go to: 28 | http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/ 29 | 30 | -------------------------------------------------------------------------------- /k8-util/dashboard/token.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get sa/admin-user -o jsonpath="{.secrets[0].name}") -------------------------------------------------------------------------------- /k8-util/dashboard/user.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: admin-user 5 | namespace: kubernetes-dashboard 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | kind: ClusterRoleBinding 9 | metadata: 10 | name: admin-user 11 | roleRef: 12 | apiGroup: rbac.authorization.k8s.io 13 | kind: ClusterRole 14 | name: cluster-admin 15 | subjects: 16 | - kind: ServiceAccount 17 | name: admin-user 18 | namespace: kubernetes-dashboard -------------------------------------------------------------------------------- /k8-util/docker/fluvio.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ARCH= 2 | FROM ${ARCH}alpine:3.14 3 | COPY fluvio-run fluvio-run 4 | -------------------------------------------------------------------------------- /k8-util/helm/.gitignore: -------------------------------------------------------------------------------- 1 | pkg_sys 2 | pkg_app -------------------------------------------------------------------------------- /k8-util/helm/fluvio-app/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: fluvio-app 3 | description: A Helm chart for Fluvio 4 | type: application 5 | version: 0.9.3 6 | -------------------------------------------------------------------------------- /k8-util/helm/fluvio-app/templates/role-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: {{ .Values.serviceAccount.name }} 5 | subjects: 6 | - kind: ServiceAccount 7 | name: {{ .Values.serviceAccount.name }} 8 | roleRef: 9 | kind: Role 10 | name: {{ .Values.serviceAccount.name }} 11 | apiGroup: rbac.authorization.k8s.io -------------------------------------------------------------------------------- /k8-util/helm/fluvio-app/templates/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: {{ .Values.serviceAccount.name }} 5 | rules: 6 | - apiGroups: [""] # "" indicates the core API group 7 | resources: 8 | - pods 9 | - services 10 | - statefulsets.apps 11 | - persistentvolumeclaims 12 | - persistentvolumes 13 | - replicasets 14 | - deployments 15 | - configmaps 16 | verbs: ["*"] 17 | - apiGroups: ["apps"] 18 | resources: ["*"] 19 | verbs: ["*"] 20 | - apiGroups: ["fluvio.infinyon.com"] 21 | resources: ["*"] 22 | verbs: ["*"] -------------------------------------------------------------------------------- /k8-util/helm/fluvio-app/templates/sc-internal.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: fluvio-sc-internal 5 | spec: 6 | selector: 7 | app: fluvio-sc 8 | ports: 9 | - protocol: TCP 10 | port: 9004 11 | targetPort: 9004 -------------------------------------------------------------------------------- /k8-util/helm/fluvio-app/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: fluvio -------------------------------------------------------------------------------- /k8-util/helm/fluvio-app/templates/spu-k8-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: spu-k8 5 | data: 6 | image: {{ .Values.image.registry }}/fluvio:{{ .Values.image.tag | default .Chart.AppVersion }} 7 | podSecurityContext: {{ .Values.podSecurityContext | toJson | quote }} 8 | spuPodConfig: {{ .Values.spuPod | toJson | quote }} 9 | lbServiceAnnotations: {{ .Values.loadBalancer.serviceAnnotations | toJson | quote }} 10 | service: {{ .Values.service | toJson | quote }} 11 | -------------------------------------------------------------------------------- /k8-util/helm/fluvio-sys/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /k8-util/helm/fluvio-sys/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: fluvio-sys 3 | description: A Helm chart for Fluvio 4 | type: application 5 | version: 0.9.20 6 | -------------------------------------------------------------------------------- /k8-util/helm/fluvio-sys/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for fluvio-sys -------------------------------------------------------------------------------- /k8-util/samples/crd/group1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: SpuGroup 3 | metadata: 4 | name: "group1" 5 | spec: 6 | replicas: 1 7 | minId: 10 8 | -------------------------------------------------------------------------------- /k8-util/samples/crd/group1_debug.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: SpuGroup 3 | metadata: 4 | name: "group1" 5 | spec: 6 | replicas: 1 7 | minId: 10 8 | template: 9 | spec: 10 | env: 11 | - name: "RUST_LOG" 12 | value: "spu_server=debug" 13 | -------------------------------------------------------------------------------- /k8-util/samples/crd/group2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: SpuGroup 3 | metadata: 4 | name: "group2" 5 | spec: 6 | replicas: 2 7 | minId: 11 8 | template: 9 | spec: 10 | storage: 11 | size: 2Gi 12 | logDir: "/tmp/mylog" 13 | -------------------------------------------------------------------------------- /k8-util/samples/crd/group3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: SpuGroup 3 | metadata: 4 | name: "group3" 5 | spec: 6 | replicas: 3 7 | -------------------------------------------------------------------------------- /k8-util/samples/crd/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # initialize CRD 3 | # Please add CRD 4 | DATA_DIR=$(dirname "$0") 5 | kubectl apply -f ${DATA_DIR}/crd_spu.yaml 6 | kubectl apply -f ${DATA_DIR}/crd_cluster.yaml 7 | kubectl apply -f ${DATA_DIR}/crd_partition.yaml 8 | kubectl apply -f ${DATA_DIR}/crd_topic.yaml 9 | kubectl apply -f ${DATA_DIR}/crd_mirror.yaml 10 | -------------------------------------------------------------------------------- /k8-util/samples/crd/partition.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Partition 3 | metadata: 4 | name: test 5 | spec: 6 | leader: 1 7 | replicas: [2,4] -------------------------------------------------------------------------------- /k8-util/samples/crd/spu_5001.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Spu 3 | metadata: 4 | name: "custom-spu-1" 5 | spec: 6 | spuId: 5001 7 | spuType: "Custom" 8 | publicEndpoint: 9 | port: 9005 10 | ingress: 11 | - hostname: localhost 12 | encryption: PLAINTEXT 13 | privateEndpoint: 14 | port: 9006 15 | host: localhost 16 | encryption: PLAINTEXT 17 | -------------------------------------------------------------------------------- /k8-util/samples/crd/spu_5002.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Spu 3 | metadata: 4 | name: "custom-spu-2" 5 | spec: 6 | spuId: 5002 7 | spuType: "Custom" 8 | publicEndpoint: 9 | port: 9007 10 | ingress: 11 | - hostname: localhost 12 | encryption: PLAINTEXT 13 | privateEndpoint: 14 | port: 9008 15 | host: localhost 16 | encryption: PLAINTEXT 17 | -------------------------------------------------------------------------------- /k8-util/samples/crd/spu_5003.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Spu 3 | metadata: 4 | name: "custom-spu-3" 5 | spec: 6 | spuId: 5003 7 | spuType: "Custom" 8 | publicEndpoint: 9 | port: 9009 10 | ingress: 11 | - hostname: localhost 12 | encryption: PLAINTEXT 13 | privateEndpoint: 14 | port: 9010 15 | host: localhost 16 | encryption: PLAINTEXT 17 | -------------------------------------------------------------------------------- /k8-util/samples/crd/start_5001.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ../../../dev-tools/log/debug-spu-min 5001 9005 9006 3 | -------------------------------------------------------------------------------- /k8-util/samples/crd/topic1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v2" 2 | kind: Topic 3 | metadata: 4 | name: topic1 5 | spec: 6 | replicas: 7 | computed: 8 | partitions: 1 9 | replicationFactor: 1 10 | -------------------------------------------------------------------------------- /k8-util/samples/crd/topic2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v2" 2 | kind: Topic 3 | metadata: 4 | name: topic2 5 | spec: 6 | replicas: 7 | computed: 8 | partitions: 1 9 | replicationFactor: 2 10 | -------------------------------------------------------------------------------- /k8-util/samples/crd/topic3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v2" 2 | kind: Topic 3 | metadata: 4 | name: topic3 5 | spec: 6 | replicas: 7 | computed: 8 | partitions: 1 9 | replicationFactor: 1 10 | cleanupPolicy: 11 | segment: 12 | timeInSeconds: 5000 13 | storage: 14 | segmentSize: 5000 15 | 16 | -------------------------------------------------------------------------------- /k8-util/samples/crd/topic4.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Topic 3 | metadata: 4 | name: topic4 5 | spec: 6 | partitions: 1 7 | replicationFactor: 2 8 | -------------------------------------------------------------------------------- /k8-util/samples/crd/topic5.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Topic 3 | metadata: 4 | name: topic5 5 | spec: 6 | partitions: 1 7 | replicationFactor: 2 8 | -------------------------------------------------------------------------------- /k8-util/samples/crd/topic5_custom.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Topic 3 | metadata: 4 | name: topic5 5 | spec: 6 | customReplicaAssignment: 7 | - partition: 8 | id: 0 9 | replicas: [5001, 5002] 10 | - partition: 11 | id: 1 12 | replicas: [5002, 5003] 13 | -------------------------------------------------------------------------------- /k8-util/samples/crd/topic6_custom.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "fluvio.infinyon.com/v1" 2 | kind: Topic 3 | metadata: 4 | name: topic6 5 | spec: 6 | customReplicaAssignment: 7 | - partition: 8 | replicas: [5001, 5002] 9 | id: 1 10 | -------------------------------------------------------------------------------- /k8-util/util/README.md: -------------------------------------------------------------------------------- 1 | To run dns lookup utility, first set up DNS pod by 2 | ```kubectl apply -f busybox.yaml``` 3 | 4 | To get dns entries, 5 | ```./dns.sh ``` 6 | 7 | For example, to get dns entries for spu 8 | ```bash 9 | ./dns.sh spu.default.svc.cluster.local 10 | Server: 10.96.0.10 11 | Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local 12 | 13 | Name: spu.default.svc.cluster.local 14 | Address 1: 172.17.0.2 spu-0.spu.default.svc.cluster.local 15 | Address 2: 172.17.0.3 spu-1.spu.default.svc.cluster.local 16 | Address 3: 172.17.0.6 spu-2.spu.default.svc.cluster.local 17 | ``` -------------------------------------------------------------------------------- /k8-util/util/busybox.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: busybox 5 | namespace: default 6 | spec: 7 | containers: 8 | - name: busybox 9 | image: busybox:1.28 10 | command: 11 | - sleep 12 | - "3600" 13 | imagePullPolicy: IfNotPresent 14 | restartPolicy: Always -------------------------------------------------------------------------------- /k8-util/util/deploy_dashboard.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml 3 | -------------------------------------------------------------------------------- /k8-util/util/dns.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Used for lookup up dns entry 3 | # kubectl run -i --tty --rm debug --image=busybox --restart=Never -- sh 4 | kubectl exec -ti busybox -- nslookup $1 5 | -------------------------------------------------------------------------------- /k8-util/util/run_dashboard.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | kubectl -n kube-system port-forward $(kubectl -n kube-system get pod -l k8s-app=kubernetes-dashboard -o jsonpath='{.items[0].metadata.name}') 8443:8443 & 4 | -------------------------------------------------------------------------------- /k8-util/util/snippet.md: -------------------------------------------------------------------------------- 1 | # Useful snippets 2 | 3 | **Tips**: For pretty display on a Mac, install **jsonpp** ```brew install jsonpp``` 4 | 5 | ## Display raw jason output 6 | 7 | Display all infinyon objects 8 | > kubectl get --raw /apis/fluvio.infinyon.com/v1 | jsonpp 9 | 10 | Display topic ```test``` 11 | > kubectl get --raw /apis/fluvio.infinyon.com/v1/namespaces/default/topics/test | jsonpp 12 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/.gitignore: -------------------------------------------------------------------------------- 1 | crates_io/ 2 | 3 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/publish-list.toml: -------------------------------------------------------------------------------- 1 | # This list is approximately sorted for dependency resolution 2 | 3 | publish_list = [ 4 | "fluvio-types", 5 | "fluvio-compression", 6 | "fluvio-protocol-derive", 7 | "fluvio-protocol", 8 | "fluvio-socket", 9 | "fluvio-stream-model", 10 | "fluvio-controlplane-metadata", 11 | "fluvio-spu-schema", 12 | "fluvio-smartengine", 13 | "fluvio-smartmodule-derive", 14 | "fluvio-smartmodule", 15 | "fluvio-sc-schema", 16 | "fluvio", 17 | "fluvio-stream-dispatcher", 18 | "fluvio-package-index", 19 | "fluvio-extension-common", 20 | ] 21 | 22 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "toml-diff" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2024" 6 | license = "Apache-2.0" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | toml = { workspace = true, features = ["display"] } 12 | 13 | [dev-dependencies] 14 | toml = { workspace = true, features = ["display", "parse"] } 15 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_add_a.toml: -------------------------------------------------------------------------------- 1 | array = [ 2 | "element_a", 3 | "element_b" 4 | ] 5 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_add_b.toml: -------------------------------------------------------------------------------- 1 | array = [ 2 | "element_a", 3 | ] 4 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_delete_a.toml: -------------------------------------------------------------------------------- 1 | array = [ 2 | "element_a", 3 | "element_c" 4 | ] 5 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_delete_b.toml: -------------------------------------------------------------------------------- 1 | array = [ 2 | "element_a", 3 | "element_b", 4 | "element_c" 5 | ] 6 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_delete_one_a.toml: -------------------------------------------------------------------------------- 1 | array = [] 2 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_delete_one_b.toml: -------------------------------------------------------------------------------- 1 | array = [ 2 | "element" 3 | ] 4 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_reorder_a.toml: -------------------------------------------------------------------------------- 1 | array = [ 2 | "element_a", 3 | "element_c", 4 | "element_b", 5 | "element_d", 6 | ] 7 | 8 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/array_reorder_b.toml: -------------------------------------------------------------------------------- 1 | array = [ 2 | "element_a", 3 | "element_b", 4 | "element_c", 5 | "element_d", 6 | ] 7 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/arrays_a.toml: -------------------------------------------------------------------------------- 1 | a = [1, 2, 3] 2 | b = [2, 3, 4] 3 | d = [4, 5, 6] 4 | 5 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/arrays_b.toml: -------------------------------------------------------------------------------- 1 | b = [2, 3, 4] 2 | c = [3, 4, 5] 3 | d = [4, 5, 6] 4 | e = [5, 6, 7] 5 | f = [6, 7, 8] 6 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/nested_tables_a.toml: -------------------------------------------------------------------------------- 1 | [outer.inner_a] 2 | a = 1 3 | 4 | [outer.inner_b] 5 | b = 2 6 | 7 | [outer.inner_d] 8 | d = 4 9 | 10 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/nested_tables_b.toml: -------------------------------------------------------------------------------- 1 | [outer.inner_a] 2 | a = 1 3 | 4 | [outer.inner_c] 5 | c = 3 6 | 7 | [outer.inner_d] 8 | d = 4 9 | 10 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/strings_a.toml: -------------------------------------------------------------------------------- 1 | a = "abc" 2 | b = "def" 3 | d = "jkl" 4 | e = "mno" 5 | f = "pqr" 6 | 7 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/strings_b.toml: -------------------------------------------------------------------------------- 1 | a = "abc" 2 | c = "ghi" 3 | d = "jkl" 4 | 5 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/tables_a.toml: -------------------------------------------------------------------------------- 1 | [a] 2 | a = "abc" 3 | b = "def" 4 | 5 | [b] 6 | c = "ghi" 7 | d = "jkl" 8 | 9 | [d] 10 | g = "stu" 11 | h = "vwx" 12 | 13 | -------------------------------------------------------------------------------- /release-tools/check-crate-version/toml-diff/test_data/tables_b.toml: -------------------------------------------------------------------------------- 1 | [a] 2 | a = "abc" 3 | b = "def" 4 | 5 | [c] 6 | e = "nmo" 7 | f = "pqr" 8 | 9 | [d] 10 | g = "stu" 11 | h = "vwx" 12 | 13 | -------------------------------------------------------------------------------- /release-tools/publish-list: -------------------------------------------------------------------------------- 1 | # This list is approximately sorted for dependency resolution 2 | # Used by publish crate and crate version check scripts 3 | 4 | PUBLISH_CRATES=( 5 | fluvio-types 6 | fluvio-compression 7 | fluvio-protocol-derive 8 | fluvio-protocol 9 | fluvio-socket 10 | fluvio-stream-model 11 | fluvio-controlplane-metadata 12 | fluvio-spu-schema 13 | fluvio-smartengine 14 | fluvio-smartmodule-derive 15 | fluvio-smartmodule 16 | fluvio-sc-schema 17 | fluvio 18 | fluvio-stream-dispatcher 19 | fluvio-package-index 20 | fluvio-extension-common 21 | ) -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.86.0" 3 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | reorder_modules = false 2 | reorder_imports = false -------------------------------------------------------------------------------- /smartmodule/cargo_template/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.hub 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /smartmodule/cargo_template/Cargo.toml.liquid: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "{{project-name}}" 3 | version = "0.1.0" 4 | authors = ["{{authors}}"] 5 | edition = "2024" 6 | 7 | [lib] 8 | crate-type = ['cdylib'] 9 | 10 | [dependencies] 11 | fluvio-smartmodule = {{fluvio-smartmodule-cargo-dependency}} 12 | serde = { version = "1", features = ["derive"] } 13 | serde_json = "1" 14 | {% if smartmodule-params %}{% if smartmodule-type == "filter" %}once_cell = "1.13.0"{% endif %}{% endif %} 15 | 16 | [profile.release-lto] 17 | inherits = "release" 18 | lto = true 19 | strip = "symbols" 20 | -------------------------------------------------------------------------------- /smartmodule/cargo_template/SmartModule.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "{{project-name}}" 3 | group = "{{project-group | kebab_case}}" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | description = "{{project-description}}" 7 | license = "Apache-2.0" 8 | visibility = {% if smartmodule-public %}"public"{% else %}"private"{% endif %} 9 | 10 | [[params]] 11 | name = "input" 12 | description = "input description" 13 | -------------------------------------------------------------------------------- /smartmodule/cargo_template/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "stable" 3 | targets = ["wasm32-unknown-unknown", "wasm32-wasip1"] 4 | -------------------------------------------------------------------------------- /smartmodule/examples/.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | target = "wasm32-wasip1" 3 | -------------------------------------------------------------------------------- /smartmodule/examples/.gitignore: -------------------------------------------------------------------------------- 1 | target -------------------------------------------------------------------------------- /smartmodule/examples/Makefile: -------------------------------------------------------------------------------- 1 | default: build 2 | 3 | SMDK_BIN ?= smdk 4 | SMDK_OPT ?= 5 | 6 | SUBDIRS := $(filter-out target/., $(wildcard */.)) 7 | 8 | .PHONY: build $(SUBDIRS) 9 | 10 | setup: 11 | rustup target add wasm32-wasip1 12 | @echo SMDK_BIN ${SMDK_BIN} 13 | 14 | dirs: 15 | echo ${SUBDIRS} 16 | 17 | build: setup $(SUBDIRS) 18 | 19 | $(SUBDIRS): 20 | @echo Building smartmodule example $@ 21 | cd $@ && ${SMDK_BIN} build ${SMDK_OPT} 22 | 23 | build-raw: setup 24 | cargo build --release 25 | 26 | clean: 27 | cargo clean 28 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate-average/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-aggregate-average" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | serde = "1" 13 | serde_json = "1" 14 | 15 | fluvio-smartmodule = { workspace = true } 16 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate-init/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sm-integer-sum-aggegrate" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | 14 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate-init/README.md: -------------------------------------------------------------------------------- 1 | In order to build and use this SmartModule in your cluster, follow these steps: 2 | 3 | 1. Install `smdk` tool: 4 | ```bash 5 | fluvio install smdk 6 | ``` 7 | 2. Build: 8 | ```bash 9 | smdk build 10 | ``` 11 | 12 | 3. Load built SmartModule into the cluster: 13 | ```bash 14 | smdk load 15 | ``` 16 | 17 | After that, you can consume from your topic and apply the aggregation as trasnformation: 18 | ```bash 19 | fluvio consume test-aggr --transforms transforms.yaml 20 | ``` 21 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate-init/SmartModule.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "integer-sum-aggegrate" 3 | group = "infinyon" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | description = "" 7 | license = "Apache-2.0" 8 | visibility = "private" 9 | 10 | [[params]] 11 | name = "initial_value" 12 | description = "initial value for aggregation" 13 | optional = false 14 | 15 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate-init/transforms.yaml: -------------------------------------------------------------------------------- 1 | transforms: 2 | - uses: infinyon/integer-sum-aggegrate@0.1.0 3 | with: 4 | initial_value: "10" 5 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate-json/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-aggregate-json" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | serde = { version = "1", features = ["derive"] } 13 | serde_json = "1" 14 | 15 | fluvio-smartmodule = { workspace = true } -------------------------------------------------------------------------------- /smartmodule/examples/aggregate-sum/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-aggregate-sum" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-aggregate" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate/src/lib.rs: -------------------------------------------------------------------------------- 1 | use fluvio_smartmodule::{smartmodule, Result, SmartModuleRecord, RecordData}; 2 | 3 | /// This aggregate concatenate accumulator and current value 4 | /// values: "a","b" 5 | // accumulator: "1", 6 | // "1a","1ab" 7 | #[smartmodule(aggregate)] 8 | pub fn aggregate(accumulator: RecordData, current: &SmartModuleRecord) -> Result { 9 | let mut acc = String::from_utf8(accumulator.as_ref().to_vec())?; 10 | let next = std::str::from_utf8(current.value.as_ref())?; 11 | acc.push_str(next); 12 | Ok(acc.into()) 13 | } 14 | -------------------------------------------------------------------------------- /smartmodule/examples/aggregate_with_timestamp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-aggregate-with-timestamp" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | chrono = { version = "0.4.26", default-features = false } 13 | 14 | # Fluvio 15 | fluvio-smartmodule = { workspace = true } 16 | -------------------------------------------------------------------------------- /smartmodule/examples/array_map_json_array/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-array-map-array" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | serde_json = "1" 13 | fluvio-smartmodule = { workspace = true } -------------------------------------------------------------------------------- /smartmodule/examples/array_map_json_array_with_timestamp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-array-map-array-with-timestamp" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | chrono = { version = "0.4.26", default-features = false } 13 | serde_json = "1" 14 | fluvio-smartmodule = { workspace = true } 15 | -------------------------------------------------------------------------------- /smartmodule/examples/array_map_json_object/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-array-map-object" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | serde_json = "1" 14 | -------------------------------------------------------------------------------- /smartmodule/examples/array_map_json_reddit/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-array-map-reddit" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | serde = { version = "1", features = ["derive"] } 13 | serde_json = "1" 14 | 15 | fluvio-smartmodule = { workspace = true } 16 | -------------------------------------------------------------------------------- /smartmodule/examples/filter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/filter/src/lib.rs: -------------------------------------------------------------------------------- 1 | use fluvio_smartmodule::{smartmodule, SmartModuleRecord, Result}; 2 | 3 | #[smartmodule(filter)] 4 | pub fn filter(record: &SmartModuleRecord) -> Result { 5 | let string = std::str::from_utf8(record.value.as_ref())?; 6 | Ok(string.contains('a')) 7 | } 8 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_hashset/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-hashset" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib', 'rlib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_hashset/README.md: -------------------------------------------------------------------------------- 1 | Filter-kind SmartModule that mimics bounded HashSet. 2 | 3 | In order to build and use this SmartModule in your cluster, follow these steps: 4 | 5 | 1. Install `smdk` tool: 6 | ```bash 7 | fluvio install smdk 8 | ``` 9 | 2. Build: 10 | ```bash 11 | smdk build 12 | ``` 13 | 14 | 3. Load built SmartModule into the cluster: 15 | ```bash 16 | smdk load 17 | ``` 18 | 19 | After that, you can consume from your topic and apply the aggregation as trasnformation: 20 | ```bash 21 | fluvio consume test-filter-hashset --transforms transforms.yaml 22 | ``` 23 | 24 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_hashset/SmartModule.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-hashset" 3 | group = "infinyon" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | description = "" 7 | license = "Apache-2.0" 8 | visibility = "private" 9 | 10 | [[params]] 11 | name = "param" 12 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_hashset/transforms.yaml: -------------------------------------------------------------------------------- 1 | transforms: 2 | - uses: infinyon/fluvio-smartmodule-filter-hashset@0.1.0 3 | lookback: 4 | last: 10 5 | 6 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_init/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-init" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | description = "SmartModule filter with mandatory param" 8 | 9 | [lib] 10 | crate-type = ['cdylib'] 11 | 12 | [dependencies] 13 | fluvio-smartmodule = { workspace = true } 14 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_json/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-filter-json" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | serde = { version = "1", features = ["derive"] } 13 | serde_json = "1" 14 | fluvio-smartmodule = { workspace = true } 15 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-lookback" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back/README.md: -------------------------------------------------------------------------------- 1 | In order to build and use this SmartModule in your cluster, follow these steps: 2 | 3 | 1. Install `smdk` tool: 4 | ```bash 5 | fluvio install smdk 6 | ``` 7 | 2. Build: 8 | ```bash 9 | smdk build 10 | ``` 11 | 12 | 3. Load built SmartModule into the cluster: 13 | ```bash 14 | smdk load 15 | ``` 16 | 17 | After that, you can consume from your topic and apply the aggregation as trasnformation: 18 | ```bash 19 | fluvio consume test-filter-lookback --transforms transforms.yaml 20 | ``` 21 | 22 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back/SmartModule.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-lookback" 3 | group = "infinyon" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | description = "" 7 | license = "Apache-2.0" 8 | visibility = "private" 9 | 10 | [[params]] 11 | name = "param" 12 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back/transforms.yaml: -------------------------------------------------------------------------------- 1 | transforms: 2 | - uses: infinyon/fluvio-smartmodule-filter-lookback@0.1.0 3 | lookback: 4 | last: 10 5 | 6 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back_with_timestamps/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-lookback-with-timestamps" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back_with_timestamps/README.md: -------------------------------------------------------------------------------- 1 | In order to build and use this SmartModule in your cluster, follow these steps: 2 | 3 | 1. Install `smdk` tool: 4 | ```bash 5 | fluvio install smdk 6 | ``` 7 | 2. Build: 8 | ```bash 9 | smdk build 10 | ``` 11 | 12 | 3. Load built SmartModule into the cluster: 13 | ```bash 14 | smdk load 15 | ``` 16 | 17 | After that, you can consume from your topic and apply the aggregation as trasnformation: 18 | ```bash 19 | fluvio consume test-filter-lookback --transforms transforms.yaml 20 | ``` 21 | 22 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back_with_timestamps/SmartModule.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-lookback-with-timestamps" 3 | group = "infinyon" 4 | version = "0.1.0" 5 | apiVersion = "0.1.0" 6 | description = "" 7 | license = "Apache-2.0" 8 | visibility = "private" 9 | 10 | [[params]] 11 | name = "param" 12 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_look_back_with_timestamps/transforms.yaml: -------------------------------------------------------------------------------- 1 | transforms: 2 | - uses: infinyon/fluvio-smartmodule-filter-lookback@0.1.0 3 | lookback: 4 | last: 10 5 | 6 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_map/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-map" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_map/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This SmartModule filters out all odd numbers, and divides all even numbers by 2. 2 | 3 | use fluvio_smartmodule::{smartmodule, SmartModuleRecord, RecordData, Result}; 4 | 5 | #[smartmodule(filter_map)] 6 | pub fn filter_map(record: &SmartModuleRecord) -> Result, RecordData)>> { 7 | let key = record.key.clone(); 8 | let string = String::from_utf8_lossy(record.value.as_ref()).to_string(); 9 | let int: i32 = string.parse()?; 10 | 11 | if int % 2 == 0 { 12 | let output = int / 2; 13 | Ok(Some((key.clone(), RecordData::from(output.to_string())))) 14 | } else { 15 | Ok(None) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_odd/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-filter-odd" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | thiserror = "1" 13 | fluvio-smartmodule = { workspace = true } 14 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_odd_key/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-filter-odd-key" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | thiserror = "1" 13 | fluvio-smartmodule = { workspace = true } 14 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_regex/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-filter-regex" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | regex = "1" 13 | fluvio-smartmodule = { workspace = true } 14 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_regex/src/lib.rs: -------------------------------------------------------------------------------- 1 | use fluvio_smartmodule::{smartmodule, SmartModuleRecord, Result}; 2 | use regex::Regex; 3 | 4 | #[smartmodule(filter)] 5 | pub fn filter(record: &SmartModuleRecord) -> Result { 6 | let string = std::str::from_utf8(record.value.as_ref())?; 7 | 8 | // Check whether the Record contains a Social Security number 9 | let social_security_regex = Regex::new(r"\d{3}-\d{2}-\d{4}").unwrap(); 10 | let has_ss = social_security_regex.is_match(string); 11 | 12 | // Only accept records that _do not_ have social security numbers in them 13 | Ok(!has_ss) 14 | } 15 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_with_param/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-filter-param" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | description = "Filter with default parameter" 7 | publish = false 8 | 9 | [lib] 10 | crate-type = ['cdylib'] 11 | 12 | [dependencies] 13 | fluvio-smartmodule = { workspace = true } 14 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_with_param_v1/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-filter-with-parameters-v1" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | description = "Old Fluvio WASM Filter with Parameters" 8 | 9 | [lib] 10 | crate-type = ['cdylib'] 11 | 12 | [dependencies] 13 | fluvio-smartmodule = "0.2.5" 14 | -------------------------------------------------------------------------------- /smartmodule/examples/filter_with_param_v1/src/lib.rs: -------------------------------------------------------------------------------- 1 | use fluvio_smartmodule::{smartmodule, SmartOpt, Record, Result}; 2 | 3 | #[derive(SmartOpt)] 4 | pub struct FilterOpt { 5 | key: String, 6 | } 7 | 8 | impl Default for FilterOpt { 9 | fn default() -> Self { 10 | Self { 11 | key: "a".to_string(), 12 | } 13 | } 14 | } 15 | 16 | #[smartmodule(filter, params)] 17 | pub fn filter(record: &Record, opt: &FilterOpt) -> Result { 18 | let string = std::str::from_utf8(record.value.as_ref())?; 19 | Ok(string.contains(&opt.key)) 20 | } 21 | -------------------------------------------------------------------------------- /smartmodule/examples/map/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-map" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/map/src/lib.rs: -------------------------------------------------------------------------------- 1 | use fluvio_smartmodule::{smartmodule, SmartModuleRecord, RecordData, Result}; 2 | 3 | #[smartmodule(map)] 4 | pub fn map(record: &SmartModuleRecord) -> Result<(Option, RecordData)> { 5 | let key = record.key.clone(); 6 | let mut value = Vec::from(record.value.as_ref()); 7 | 8 | value.make_ascii_uppercase(); 9 | Ok((key, value.into())) 10 | } 11 | -------------------------------------------------------------------------------- /smartmodule/examples/map_double/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-map-double" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | fluvio-smartmodule = { workspace = true } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/map_double/src/lib.rs: -------------------------------------------------------------------------------- 1 | use fluvio_smartmodule::{smartmodule, SmartModuleRecord, RecordData, Result}; 2 | 3 | #[smartmodule(map)] 4 | pub fn map(record: &SmartModuleRecord) -> Result<(Option, RecordData)> { 5 | let key = record.key.clone(); 6 | 7 | let string = std::str::from_utf8(record.value.as_ref())?; 8 | let int = string.parse::()?; 9 | let value = (int * 2).to_string(); 10 | 11 | Ok((key, value.into())) 12 | } 13 | -------------------------------------------------------------------------------- /smartmodule/examples/map_json/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-map-json" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | serde_json = "1" 13 | serde_yaml = "0.9.0" 14 | fluvio-smartmodule = { workspace = true } 15 | -------------------------------------------------------------------------------- /smartmodule/examples/map_json/src/lib.rs: -------------------------------------------------------------------------------- 1 | use fluvio_smartmodule::{smartmodule, SmartModuleRecord, RecordData, Result}; 2 | 3 | #[smartmodule(map)] 4 | pub fn map(record: &SmartModuleRecord) -> Result<(Option, RecordData)> { 5 | let json = serde_json::from_slice::(record.value.as_ref())?; 6 | let yaml_bytes = serde_yaml::to_string(&json)?.into_bytes(); 7 | 8 | Ok((record.key().cloned(), yaml_bytes.into())) 9 | } 10 | -------------------------------------------------------------------------------- /smartmodule/examples/map_regex/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-wasm-map-regex" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | regex = "1" 13 | once_cell = "1" 14 | fluvio-smartmodule = { workspace = true } -------------------------------------------------------------------------------- /smartmodule/examples/map_regex/src/lib.rs: -------------------------------------------------------------------------------- 1 | use regex::Regex; 2 | use once_cell::sync::Lazy; 3 | use fluvio_smartmodule::{smartmodule, Result, SmartModuleRecord, RecordData}; 4 | 5 | static SSN_RE: Lazy = Lazy::new(|| Regex::new(r"\d{3}-\d{2}-\d{4}").unwrap()); 6 | 7 | #[smartmodule(map)] 8 | pub fn map(record: &SmartModuleRecord) -> Result<(Option, RecordData)> { 9 | let key = record.key.clone(); 10 | 11 | let string = std::str::from_utf8(record.value.as_ref())?; 12 | let output = SSN_RE.replace_all(string, "***-**-****").to_string(); 13 | 14 | Ok((key, output.into())) 15 | } 16 | -------------------------------------------------------------------------------- /smartmodule/examples/map_with_timestamp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fluvio-smartmodule-map-with-timestamp" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | chrono = { version = "0.4.26", default-features = false } 13 | fluvio-smartmodule = { workspace = true } 14 | -------------------------------------------------------------------------------- /smartmodule/regex-filter/.gitignore: -------------------------------------------------------------------------------- 1 | hub -------------------------------------------------------------------------------- /smartmodule/regex-filter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "regex-filter" 3 | version = "0.0.0" 4 | authors = ["Fluvio Contributors "] 5 | edition = "2024" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ['cdylib'] 10 | 11 | [dependencies] 12 | regex = "1.5" 13 | fluvio-smartmodule = { path = "../../crates/fluvio-smartmodule" } 14 | 15 | 16 | [profile.release-lto] 17 | inherits = "release" 18 | lto = true -------------------------------------------------------------------------------- /smartmodule/regex-filter/Makefile: -------------------------------------------------------------------------------- 1 | 2 | SMDK_BIN ?= smdk 3 | SMDK_OPT ?= 4 | 5 | default: build 6 | 7 | setup: 8 | rustup target add wasm32-wasip1 9 | 10 | build: setup 11 | ${SMDK_BIN} build ${SMDK_OPT} 12 | 13 | build-raw: 14 | cargo build --target wasm32-wasip1 15 | 16 | -------------------------------------------------------------------------------- /smartmodule/regex-filter/README.md: -------------------------------------------------------------------------------- 1 | Example of SmartModule with regex. 2 | 3 | To run this, have fluvio stable installed. 4 | 5 | compile this package: 6 | ``` 7 | $ smdk build 8 | ``` 9 | 10 | run regex at top level: 11 | 12 | positive: 13 | ``` 14 | $ smdk test --text AA -e regex="[A-Z]" 15 | 1 records 16 | AA 17 | ``` 18 | 19 | negative: 20 | ``` 21 | $ smdk test --text aa -e regex="[A-Z]" 22 | 0 records 23 | ``` 24 | -------------------------------------------------------------------------------- /smartmodule/regex-filter/SmartModule.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "regex-filter" 3 | group = "infinyon" 4 | version = "0.2.0" 5 | apiVersion = "0.1.0" 6 | description = "Regex SmartModule" 7 | license = "Apache-2.0" 8 | visibility = "public" 9 | repository = "https://github.com/infinyon/fluvio" 10 | 11 | [[params]] 12 | name = "regex" 13 | description = "Regex filter" 14 | optional = false 15 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | *.tmp 2 | *.checksum -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | flvt=../target/debug/fluvio-test 2 | DRIVER=--local-driver 3 | COUNT=3000 4 | 5 | build: 6 | cargo build 7 | 8 | test: run-local run-k8 9 | 10 | smoke-test: build 11 | $(flvt) $(DRIVER) 12 | 13 | run-stress: build 14 | $(flvt) $(DRIVER) --produce-count $(COUNT) 15 | -------------------------------------------------------------------------------- /tests/cli/test_helper/.gitignore: -------------------------------------------------------------------------------- 1 | bats-assert 2 | bats-support -------------------------------------------------------------------------------- /tests/cli/test_helper/test-table-format-config.yml: -------------------------------------------------------------------------------- 1 | name: testtable 2 | inputFormat: "JSON" 3 | columns: 4 | - keyPath: "key2" 5 | display: true 6 | - keyPath: "key1" 7 | -------------------------------------------------------------------------------- /tests/fluvio-values.yaml: -------------------------------------------------------------------------------- 1 | loadBalancer: 2 | serviceAnnotations: 3 | cloud.fluvio.io/hostname: xyz 4 | service: 5 | type: ClusterIP 6 | -------------------------------------------------------------------------------- /tests/produce-large.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | # generate large data size 3 | set -e 4 | fluvio topic create t1 5 | fluvio produce t1 -r $(ls /var/log/journal/**/*.journal) -------------------------------------------------------------------------------- /tests/test-table-format-config.yaml: -------------------------------------------------------------------------------- 1 | name: smoke-test 2 | inputFormat: "JSON" 3 | columns: 4 | - keyPath: "key1" 5 | - keyPath: "key2" -------------------------------------------------------------------------------- /tls/cert.conf: -------------------------------------------------------------------------------- 1 | [req] 2 | default_bits = 4096 3 | prompt = no 4 | default_md = sha256 5 | req_extensions = req_ext 6 | distinguished_name = dn 7 | 8 | [ dn ] 9 | C=US 10 | ST=CA 11 | L=Sunnyvale 12 | O=End Point 13 | OU=Testing Domain 14 | emailAddress=test@fluvio.io 15 | CN = test.fluvio.io 16 | 17 | [ req_ext ] 18 | subjectAltName = @alt_names 19 | 20 | [ alt_names ] 21 | DNS.1 = localhost -------------------------------------------------------------------------------- /tls/certs/ca.srl: -------------------------------------------------------------------------------- 1 | 6F541A761C6110E1A5D439D7897200E81996F807 2 | --------------------------------------------------------------------------------