├── .github ├── codecov.yaml ├── renovate.json5 ├── repository-settings.md └── workflows │ ├── fossa.yml │ ├── go-ci.yml │ ├── ossf-scorecard.yml │ ├── pipeline-perf-test.yml │ ├── repo-lint.yaml │ ├── rust-audit.yml │ └── rust-ci.yml ├── .gitignore ├── .gitmodules ├── .golangci.yaml ├── .idea ├── .gitignore ├── modules.xml ├── otel-arrow-adapter.iml └── vcs.xml ├── .markdownlint.yaml ├── CHANGELOG.md ├── CODEOWNERS ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── RELEASING.md ├── SECURITY.md ├── collector ├── BUILDING.md ├── README.md ├── cmd │ └── otelarrowcol │ │ ├── components.go │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ ├── main_others.go │ │ └── main_windows.go ├── examples │ ├── README.md │ ├── bridge │ │ ├── README.md │ │ ├── edge-collector.yaml │ │ └── saas-collector.yaml │ ├── metadata-bridge │ │ ├── README.md │ │ ├── edge-collector.yaml │ │ └── saas-collector.yaml │ ├── recorder │ │ ├── README.md │ │ ├── replay.yaml │ │ └── target.yaml │ └── shutdown │ │ ├── README.md │ │ ├── edge-collector.yaml │ │ ├── middle-collector.yaml │ │ └── saas-collector.yaml ├── otelarrowcol-build.yaml └── processor │ └── concurrentbatchprocessor │ └── README.md ├── data ├── .gitignore ├── multivariate-metrics.json └── multivariate-metrics.pb ├── data_analysis ├── gen_heatmap_charts.py ├── hipster_results │ ├── average_improvement_heatmap.png │ └── side_by_side_heatmap.png └── prod_results │ ├── average_improvement_heatmap.png │ └── side_by_side_heatmap.png ├── docs ├── benchmarks.md ├── data_model.md ├── img │ ├── OTEL - Chaos engineering.png │ ├── OTEL - TMA scope.png │ ├── OTEL - validation process.png │ ├── average_improvement_heatmap.png │ ├── collector_internal_overview.png │ ├── compression_ratio_summary_multivariate_metrics.png │ ├── compression_ratio_summary_std_metrics.png │ ├── logs_compression_rate_benchmark.png │ ├── logs_end_to_end_benchmark.png │ ├── metrics_compression_rate_benchmark.png │ ├── metrics_end_to_end_benchmark.png │ ├── side_by_side_heatmap.png │ ├── traces_compressed_msg_size.png │ ├── traces_compression_rate_benchmark.png │ ├── traces_end_to_end_benchmark.png │ ├── traces_uncompressed_msg_size.png │ └── traffic_reduction_use_case.png ├── multivariate-design.md ├── phase2-design.md ├── project-phases.md ├── threat_model_assessment.md └── validation_process.md ├── go ├── api │ └── experimental │ │ └── arrow │ │ └── v1 │ │ ├── arrow_service.pb.go │ │ ├── arrow_service_grpc.pb.go │ │ └── mock │ │ └── arrow_service_mock.go ├── go.mod ├── go.sum ├── pkg │ ├── arrow │ │ ├── constants.go │ │ ├── doc.go │ │ ├── errors.go │ │ ├── from_array.go │ │ ├── from_record.go │ │ ├── from_schema.go │ │ ├── from_sparseunion.go │ │ ├── from_struct.go │ │ ├── list_structs.go │ │ ├── record.go │ │ └── schema.go │ ├── benchmark │ │ ├── compression.go │ │ ├── compression_test.go │ │ ├── config.go │ │ ├── dataset │ │ │ ├── doc.go │ │ │ ├── real_logs_dataset.go │ │ │ ├── real_metrics_dataset.go │ │ │ ├── real_trace_dataset.go │ │ │ └── synthetic_dataset.go │ │ ├── doc.go │ │ ├── profileable.go │ │ ├── profileable │ │ │ ├── arrow │ │ │ │ ├── doc.go │ │ │ │ ├── logs.go │ │ │ │ ├── metrics.go │ │ │ │ ├── otlp_arrow_profiler_test.go │ │ │ │ └── traces.go │ │ │ ├── doc.go │ │ │ └── otlp │ │ │ │ ├── doc.go │ │ │ │ ├── logs.go │ │ │ │ ├── metrics.go │ │ │ │ ├── otlp_profiler_test.go │ │ │ │ └── traces.go │ │ ├── profiler.go │ │ └── stats │ │ │ ├── logs.go │ │ │ ├── metrics.go │ │ │ ├── stats.go │ │ │ ├── stats_test.go │ │ │ └── system.go │ ├── config │ │ └── config.go │ ├── datagen │ │ ├── data_generator.go │ │ ├── default.go │ │ ├── doc.go │ │ ├── logs.go │ │ ├── metrics.go │ │ └── trace.go │ ├── internal │ │ └── debug │ │ │ ├── assert_off.go │ │ │ └── assert_on.go │ ├── otel │ │ ├── arrow_record │ │ │ ├── Makefile │ │ │ ├── consumer.go │ │ │ ├── doc.go │ │ │ ├── logs_dict_test.go │ │ │ ├── metrics_dict_test.go │ │ │ ├── mock │ │ │ │ └── mock.go │ │ │ ├── producer.go │ │ │ ├── producer_consumer_test.go │ │ │ └── traces_dict_test.go │ │ ├── assert │ │ │ ├── README.md │ │ │ ├── doc.go │ │ │ ├── equiv.go │ │ │ └── equiv_test.go │ │ ├── common │ │ │ ├── arrow │ │ │ │ ├── all_test.go │ │ │ │ ├── allocator.go │ │ │ │ ├── allocator_test.go │ │ │ │ ├── analyzer.go │ │ │ │ ├── any_value.go │ │ │ │ ├── attributes.go │ │ │ │ ├── attributes_16.go │ │ │ │ ├── attributes_32.go │ │ │ │ ├── builder.go │ │ │ │ ├── config.go │ │ │ │ ├── doc.go │ │ │ │ ├── dyn_attrs.go │ │ │ │ ├── dyn_attrs_test.go │ │ │ │ ├── errors.go │ │ │ │ ├── optimizer_options.go │ │ │ │ ├── related_data.go │ │ │ │ ├── resource.go │ │ │ │ ├── scope.go │ │ │ │ └── tmo │ │ │ │ │ └── dyn_attrs_sorted.go │ │ │ ├── arrow_test │ │ │ │ └── schema_test.go │ │ │ ├── attributes.go │ │ │ ├── cbor.go │ │ │ ├── cbor_test.go │ │ │ ├── doc.go │ │ │ ├── errors.go │ │ │ ├── otlp │ │ │ │ ├── any_value.go │ │ │ │ ├── any_value_test.go │ │ │ │ ├── attributes.go │ │ │ │ ├── doc.go │ │ │ │ ├── dyn_attrs.go │ │ │ │ ├── dyn_attrs_test.go │ │ │ │ ├── errors.go │ │ │ │ ├── ids.go │ │ │ │ ├── resource.go │ │ │ │ └── scope.go │ │ │ ├── schema │ │ │ │ ├── builder │ │ │ │ │ ├── binary.go │ │ │ │ │ ├── bool.go │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── duration.go │ │ │ │ │ ├── float.go │ │ │ │ │ ├── int.go │ │ │ │ │ ├── list.go │ │ │ │ │ ├── map.go │ │ │ │ │ ├── record.go │ │ │ │ │ ├── sparse_union.go │ │ │ │ │ ├── string.go │ │ │ │ │ ├── struct.go │ │ │ │ │ ├── timestamp.go │ │ │ │ │ └── uint.go │ │ │ │ ├── config │ │ │ │ │ └── dictionary.go │ │ │ │ ├── events │ │ │ │ │ └── counters.go │ │ │ │ ├── schema.go │ │ │ │ ├── transform │ │ │ │ │ ├── dictionary.go │ │ │ │ │ ├── dictionary_test.go │ │ │ │ │ ├── identity.go │ │ │ │ │ └── no_field.go │ │ │ │ ├── transform_node.go │ │ │ │ └── update │ │ │ │ │ └── schema_update_request.go │ │ │ └── test_utils.go │ │ ├── constants │ │ │ ├── constants.go │ │ │ └── doc.go │ │ ├── doc.go │ │ ├── errors.go │ │ ├── internal │ │ │ └── test_utilities.go │ │ ├── logs │ │ │ ├── arrow │ │ │ │ ├── all_test.go │ │ │ │ ├── analyzer.go │ │ │ │ ├── config.go │ │ │ │ ├── doc.go │ │ │ │ ├── logs.go │ │ │ │ ├── optimizer.go │ │ │ │ └── related_data.go │ │ │ ├── doc.go │ │ │ ├── otlp │ │ │ │ ├── doc.go │ │ │ │ ├── errors.go │ │ │ │ ├── logs.go │ │ │ │ └── related_data.go │ │ │ └── validation_test.go │ │ ├── metrics │ │ │ ├── arrow │ │ │ │ ├── analyzer.go │ │ │ │ ├── config.go │ │ │ │ ├── doc.go │ │ │ │ ├── ehistogram_dp.go │ │ │ │ ├── ehistogram_dpb.go │ │ │ │ ├── errors.go │ │ │ │ ├── exemplar.go │ │ │ │ ├── histogram_dp.go │ │ │ │ ├── metrics.go │ │ │ │ ├── number_data_point.go │ │ │ │ ├── optimizer.go │ │ │ │ ├── quantile_value.go │ │ │ │ ├── related_data.go │ │ │ │ └── summary_dp.go │ │ │ ├── data_point.go │ │ │ ├── data_point_test.go │ │ │ ├── doc.go │ │ │ ├── otlp │ │ │ │ ├── doc.go │ │ │ │ ├── ehistogram.go │ │ │ │ ├── ehistogram_dpb.go │ │ │ │ ├── errors.go │ │ │ │ ├── exemplar.go │ │ │ │ ├── histogram.go │ │ │ │ ├── metrics.go │ │ │ │ ├── number_data_point.go │ │ │ │ ├── quantile_value.go │ │ │ │ ├── related_data.go │ │ │ │ └── summary.go │ │ │ └── validation_test.go │ │ ├── observer │ │ │ └── producer_observer.go │ │ ├── pdata │ │ │ └── value.go │ │ ├── stats │ │ │ └── stats.go │ │ └── traces │ │ │ ├── arrow │ │ │ ├── all_test.go │ │ │ ├── analyzer.go │ │ │ ├── config.go │ │ │ ├── doc.go │ │ │ ├── empty_trace_test.go │ │ │ ├── event.go │ │ │ ├── link.go │ │ │ ├── optimizer.go │ │ │ ├── related_data.go │ │ │ ├── status.go │ │ │ └── traces.go │ │ │ ├── doc.go │ │ │ ├── otlp │ │ │ ├── doc.go │ │ │ ├── event.go │ │ │ ├── link.go │ │ │ ├── related_data.go │ │ │ └── traces.go │ │ │ └── validation_test.go │ ├── record_message │ │ └── arrow_record.go │ └── werror │ │ ├── error.go │ │ └── error_test.go └── tools │ ├── data_model_gen │ ├── data_model.tmpl │ └── main.go │ ├── doc.go │ ├── logs_benchmark │ ├── csv_test.go │ ├── doc.go │ ├── main.go │ └── test-logs.csv │ ├── logs_gen │ ├── doc.go │ └── main.go │ ├── mem_benchmark │ └── main.go │ ├── metrics_benchmark │ ├── doc.go │ └── main.go │ ├── metrics_gen │ ├── doc.go │ ├── main.go │ └── multivariate_converter_test.go │ ├── trace_analyzer │ ├── README.md │ ├── imgs │ │ ├── compression_ratio.png │ │ ├── producer_stats.png │ │ ├── record_stats.png │ │ ├── schema_stats_1.png │ │ ├── schema_stats_2.png │ │ ├── schema_updates.png │ │ ├── span_event_record.png │ │ └── test_sorting.png │ └── main.go │ ├── trace_benchmark │ ├── doc.go │ └── main.go │ ├── trace_gen │ ├── doc.go │ └── main.go │ ├── trace_head │ ├── doc.go │ └── main.go │ ├── trace_producer_simu │ └── main.go │ └── trace_verify │ └── main.go ├── nextest.toml ├── proto ├── generate.sh └── opentelemetry │ └── proto │ └── experimental │ └── arrow │ └── v1 │ └── arrow_service.proto ├── rust ├── README.md ├── beaubourg │ ├── .config │ │ └── config.toml │ ├── .githooks │ │ └── pre-commit │ ├── .gitignore │ ├── .idea │ │ ├── .gitignore │ │ ├── modules.xml │ │ └── vcs.xml │ ├── CHANGELOG.md │ ├── CONTRIBUTING.md │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── bacon.toml │ ├── crates │ │ ├── config │ │ │ ├── Cargo.toml │ │ │ ├── data │ │ │ │ └── config.yaml │ │ │ └── src │ │ │ │ └── lib.rs │ │ ├── context │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ └── lib.rs │ │ ├── engine │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ ├── controllers.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── multi_threaded.rs │ │ │ │ ├── processor_chain.rs │ │ │ │ ├── singleton.rs │ │ │ │ └── thread_per_core.rs │ │ ├── exporter │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ ├── effect.rs │ │ │ │ └── lib.rs │ │ ├── processor │ │ │ ├── Cargo.toml │ │ │ ├── data │ │ │ │ └── test_1.json │ │ │ └── src │ │ │ │ ├── effect.rs │ │ │ │ ├── lib.rs │ │ │ │ └── noop.rs │ │ ├── receiver │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ ├── effect.rs │ │ │ │ ├── lib.rs │ │ │ │ └── signal.rs │ │ ├── signal │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ └── lib.rs │ │ └── task │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ ├── labels.rs │ │ │ └── lib.rs │ ├── deny.toml │ ├── examples │ │ ├── common │ │ │ └── mod.rs │ │ ├── exporter │ │ │ └── mod.rs │ │ ├── multithread_config.yaml │ │ ├── multithread_engine_example.rs │ │ ├── processor │ │ │ └── mod.rs │ │ ├── receiver │ │ │ └── mod.rs │ │ ├── thread_per_core_config.yaml │ │ └── thread_per_core_engine_example.rs │ ├── images │ │ ├── Beaubourg.png │ │ ├── BeaubourgExporter.png │ │ ├── BeaubourgProcessor.png │ │ ├── BeaubourgReceiver.png │ │ └── beaubourg_building.png │ ├── rustfmt.toml │ ├── src │ │ └── lib.rs │ └── tests │ │ ├── common │ │ └── mod.rs │ │ ├── config.yaml │ │ ├── exporter │ │ └── mod.rs │ │ ├── multithread_integration_test.rs │ │ ├── processor │ │ └── mod.rs │ │ ├── receiver │ │ └── mod.rs │ │ └── thread_per_core_integration_test.rs ├── experimental │ ├── README.md │ ├── query_abstraction │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── cli │ │ │ ├── Cargo.toml │ │ │ ├── LICENSE │ │ │ └── src │ │ │ │ └── main.rs │ │ ├── deny.toml │ │ ├── intermediate_language │ │ │ ├── Cargo.toml │ │ │ ├── LICENSE │ │ │ └── src │ │ │ │ ├── grammar_objects.rs │ │ │ │ ├── lib.rs │ │ │ │ └── query_processor.rs │ │ ├── kql_plugin │ │ │ ├── Cargo.toml │ │ │ ├── LICENSE │ │ │ └── src │ │ │ │ ├── kql.pest │ │ │ │ ├── kql_parser.rs │ │ │ │ ├── kql_plugin.rs │ │ │ │ └── lib.rs │ │ └── ottl_plugin │ │ │ ├── Cargo.toml │ │ │ ├── LICENSE │ │ │ └── src │ │ │ ├── lib.rs │ │ │ ├── ottl.pest │ │ │ ├── ottl_parser.rs │ │ │ └── ottl_plugin.rs │ └── syslog-cef-receivers │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── cef-receiver │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ └── src │ │ │ └── lib.rs │ │ ├── deny.toml │ │ └── syslog-receiver │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ └── src │ │ └── lib.rs ├── otap-dataflow │ ├── .cargo │ │ └── config.toml │ ├── .clippy.toml │ ├── .config │ │ └── nextest.toml │ ├── .cursor │ │ └── rules │ │ │ └── single-threaded-async-runtime.mdc │ ├── .gitignore │ ├── .typos.toml │ ├── CHANGELOG.md │ ├── CONTRIBUTING.md │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── ROADMAP.md │ ├── benchmarks │ │ ├── Cargo.toml │ │ ├── README.md │ │ └── benches │ │ │ ├── channel │ │ │ └── main.rs │ │ │ └── config │ │ │ └── main.rs │ ├── crates │ │ ├── README.md │ │ ├── channel │ │ │ ├── Cargo.toml │ │ │ ├── README.md │ │ │ └── src │ │ │ │ ├── error.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── mpmc.rs │ │ │ │ └── mpsc.rs │ │ ├── config │ │ │ ├── Cargo.toml │ │ │ ├── README.md │ │ │ └── src │ │ │ │ ├── error.rs │ │ │ │ ├── lib.rs │ │ │ │ └── node.rs │ │ ├── engine │ │ │ ├── Cargo.toml │ │ │ ├── README.md │ │ │ └── src │ │ │ │ ├── config.rs │ │ │ │ ├── effect_handler.rs │ │ │ │ ├── error.rs │ │ │ │ ├── exporter.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── local │ │ │ │ ├── exporter.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── processor.rs │ │ │ │ └── receiver.rs │ │ │ │ ├── message.rs │ │ │ │ ├── pipeline.rs │ │ │ │ ├── processor.rs │ │ │ │ ├── receiver.rs │ │ │ │ ├── shared │ │ │ │ ├── exporter.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── processor.rs │ │ │ │ └── receiver.rs │ │ │ │ └── testing │ │ │ │ ├── exporter.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── processor.rs │ │ │ │ └── receiver.rs │ │ ├── otap │ │ │ ├── Cargo.toml │ │ │ ├── README.md │ │ │ └── src │ │ │ │ ├── grpc.rs │ │ │ │ ├── lib.rs │ │ │ │ ├── mock.rs │ │ │ │ ├── otap_exporter.rs │ │ │ │ ├── otap_receiver.rs │ │ │ │ └── proto │ │ │ │ ├── mod.rs │ │ │ │ └── opentelemetry.proto.experimental.arrow.v1.rs │ │ └── otlp │ │ │ ├── Cargo.toml │ │ │ ├── README.md │ │ │ └── src │ │ │ ├── compression.rs │ │ │ ├── grpc.rs │ │ │ ├── lib.rs │ │ │ ├── mock.rs │ │ │ ├── otlp_exporter.rs │ │ │ ├── otlp_receiver.rs │ │ │ └── proto │ │ │ ├── mod.rs │ │ │ ├── opentelemetry.proto.collector.logs.v1.rs │ │ │ ├── opentelemetry.proto.collector.metrics.v1.rs │ │ │ ├── opentelemetry.proto.collector.profiles.v1development.rs │ │ │ ├── opentelemetry.proto.collector.trace.v1.rs │ │ │ ├── opentelemetry.proto.common.v1.rs │ │ │ ├── opentelemetry.proto.logs.v1.rs │ │ │ ├── opentelemetry.proto.metrics.v1.rs │ │ │ ├── opentelemetry.proto.profiles.v1development.rs │ │ │ ├── opentelemetry.proto.resource.v1.rs │ │ │ └── opentelemetry.proto.trace.v1.rs │ ├── deny.toml │ ├── docs │ │ ├── architecture.md │ │ ├── design-principles.md │ │ ├── glossary.md │ │ └── images │ │ │ └── dependencies.svg │ ├── rust-toolchain.toml │ ├── rustfmt.toml │ ├── src │ │ ├── error.rs │ │ └── lib.rs │ └── xtask │ │ ├── Cargo.toml │ │ ├── README.md │ │ └── src │ │ ├── main.rs │ │ └── structure_check.rs └── otel-arrow-rust │ ├── .clippy.toml │ ├── .github │ └── workflows │ │ └── rust.yml │ ├── .gitignore │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── benches │ ├── materialize_parent_id.rs │ └── otlp_bytes.rs │ ├── build.rs │ ├── deny.toml │ ├── licenserc.toml │ ├── rust-toolchain.toml │ ├── rustfmt.toml │ ├── src │ ├── arrays.rs │ ├── decode.rs │ ├── decode │ │ ├── decoder.rs │ │ └── record_message.rs │ ├── error.rs │ ├── lib.rs │ ├── otap.rs │ ├── otap │ │ └── transform.rs │ ├── otlp.rs │ ├── otlp │ │ ├── attributes.rs │ │ ├── attributes │ │ │ ├── cbor.rs │ │ │ ├── decoder.rs │ │ │ ├── parent_id.rs │ │ │ └── store.rs │ │ ├── common.rs │ │ ├── logs.rs │ │ ├── logs │ │ │ └── related_data.rs │ │ ├── metrics.rs │ │ └── metrics │ │ │ ├── data_points.rs │ │ │ ├── data_points │ │ │ ├── data_point_store.rs │ │ │ ├── exp_histogram.rs │ │ │ ├── histogram.rs │ │ │ ├── number.rs │ │ │ └── summary.rs │ │ │ ├── exemplar.rs │ │ │ └── related_data.rs │ ├── pdata │ │ ├── README.md │ │ ├── mod.rs │ │ └── otlp │ │ │ ├── README.md │ │ │ ├── derive │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ └── lib.rs │ │ │ ├── mod.rs │ │ │ ├── model │ │ │ ├── Cargo.toml │ │ │ └── src │ │ │ │ └── lib.rs │ │ │ └── tests.rs │ ├── proto │ │ ├── mod.rs │ │ ├── opentelemetry.proto.collector.logs.v1.rs │ │ ├── opentelemetry.proto.collector.metrics.v1.rs │ │ ├── opentelemetry.proto.collector.trace.v1.rs │ │ ├── opentelemetry.proto.common.v1.rs │ │ ├── opentelemetry.proto.experimental.arrow.v1.rs │ │ ├── opentelemetry.proto.logs.v1.rs │ │ ├── opentelemetry.proto.metrics.v1.rs │ │ ├── opentelemetry.proto.resource.v1.rs │ │ └── opentelemetry.proto.trace.v1.rs │ ├── schema.rs │ ├── schema │ │ └── consts.rs │ ├── test_util │ │ ├── create_array.rs │ │ └── mod.rs │ └── validation │ │ ├── collector.rs │ │ ├── error.rs │ │ ├── mod.rs │ │ ├── otap.rs │ │ ├── otlp.rs │ │ ├── scenarios.rs │ │ ├── service_type.rs │ │ ├── tcp_stream.rs │ │ └── testdata.rs │ └── taplo.toml ├── tools ├── pipeline_perf_test │ ├── backend │ │ ├── Dockerfile │ │ ├── backend-manifest.yaml │ │ ├── backend.py │ │ ├── readme.md │ │ └── requirements.txt │ ├── load_generator │ │ ├── Dockerfile │ │ ├── loadgen-manifest.yaml │ │ ├── loadgen.py │ │ ├── readme.md │ │ └── requirements.txt │ ├── orchestrator │ │ ├── README.md │ │ ├── lib │ │ │ ├── __init__.py │ │ │ ├── core │ │ │ │ ├── __init__.py │ │ │ │ ├── component │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── component.py │ │ │ │ │ └── component_data.py │ │ │ │ ├── context │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── component_hook_context.py │ │ │ │ │ ├── test_contexts.py │ │ │ │ │ └── test_element_hook_context.py │ │ │ │ ├── runtime │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── runtime.py │ │ │ │ ├── strategies │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── configuration_strategy.py │ │ │ │ │ ├── deployment_strategy.py │ │ │ │ │ ├── execution_strategy.py │ │ │ │ │ ├── hook_strategy.py │ │ │ │ │ ├── monitoring_strategy.py │ │ │ │ │ └── reporting_strategy.py │ │ │ │ └── test_framework │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── test_data.py │ │ │ │ │ ├── test_definition.py │ │ │ │ │ ├── test_element.py │ │ │ │ │ ├── test_step.py │ │ │ │ │ └── test_suite.py │ │ │ ├── impl │ │ │ │ ├── __init__.py │ │ │ │ ├── component │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── managed_component.py │ │ │ │ └── strategies │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── deployment │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── docker.py │ │ │ ├── process │ │ │ │ ├── __init__.py │ │ │ │ ├── deployed_process │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── docker.py │ │ │ │ │ └── kubernetes.py │ │ │ │ ├── stats.py │ │ │ │ └── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── docker.py │ │ │ │ │ └── kubernetes.py │ │ │ ├── report │ │ │ │ ├── __init__.py │ │ │ │ └── report.py │ │ │ └── runner │ │ │ │ ├── __init__.py │ │ │ │ ├── factory.py │ │ │ │ ├── registry.py │ │ │ │ ├── schema │ │ │ │ ├── __init__.py │ │ │ │ ├── hook_config.py │ │ │ │ ├── loader.py │ │ │ │ └── test_config.py │ │ │ │ └── wrappers.py │ │ ├── orchestrator.py │ │ └── requirements.txt │ ├── readme.md │ └── system_under_test │ │ ├── otel-collector │ │ ├── collector-config-with-batch-processor.yaml │ │ ├── collector-config.yaml │ │ └── collector-manifest.yaml │ │ └── readme.md └── sanitycheck.py └── versions.yaml /.github/codecov.yaml: -------------------------------------------------------------------------------- 1 | comment: 2 | layout: "header, diff, flags, components" 3 | 4 | component_management: 5 | individual_components: 6 | - component_id: "otap_dataflow" 7 | name: "otap-dataflow" 8 | paths: 9 | - "rust/otap-dataflow/" 10 | - component_id: "beaubourg" 11 | name: "beaubourg" 12 | paths: 13 | - "rust/beaubourg/" 14 | - component_id: "otel_arrow_rust" 15 | name: "otel-arrow-rust" 16 | paths: 17 | - "rust/otel-arrow-rust/" 18 | - component_id: "query_abstraction" 19 | name: "query_abstraction" 20 | paths: 21 | - "rust/experimental/query_abstraction/" 22 | - component_id: "syslog_cef_receivers" 23 | name: "syslog_cef_receivers" 24 | paths: 25 | - "rust/experimental/syslog-cef-receivers/" 26 | - component_id: "otel_arrow_go" 27 | name: "otel-arrow-go" 28 | paths: 29 | - "go/" 30 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:recommended", 5 | "docker:pinDigests", 6 | "helpers:pinGitHubActionDigestsToSemver" 7 | ], 8 | "packageRules": [ 9 | { 10 | // reduces the number of Renovate PRs 11 | // (patch updates are typically non-breaking) 12 | "groupName": "all patch versions", 13 | "matchUpdateTypes": [ 14 | "patch" 15 | ], 16 | "schedule": [ 17 | "before 8am every weekday" 18 | ] 19 | }, 20 | { 21 | // avoids these Renovate PRs from trickling in throughout the week 22 | // (consolidating the review process) 23 | "matchUpdateTypes": [ 24 | "minor", 25 | "major" 26 | ], 27 | "schedule": [ 28 | "before 8am on Monday" 29 | ] 30 | } 31 | ], 32 | "postUpdateOptions": [ 33 | // see https://docs.renovatebot.com/golang/#module-tidying 34 | // go modules may require running go mod tidy to pass CI 35 | "gomodTidy" 36 | ], 37 | "labels": [ 38 | "dependencies" 39 | ] 40 | } -------------------------------------------------------------------------------- /.github/repository-settings.md: -------------------------------------------------------------------------------- 1 | # Log of Repository Settings Changes 2 | 3 | Maintainers are expected to maintain this log. This is required as per 4 | [OpenTelemetry Community 5 | guidelines](https://github.com/open-telemetry/community/blob/main/docs/how-to-configure-new-repository.md#collaborators-and-teams). 6 | 7 | ## 2025-05-12 8 | 9 | - Configure branch protection for `main` branch to 'Require' certain checks: 10 | - Go-CI 11 | - `pkg\otel` component 12 | - test_and_coverage (renamed from build_test) 13 | - gen_otelarrowcol 14 | 15 | ## 2025-05-07 16 | 17 | - Configure branch protection for `main` branch to 'Require' certain checks: 18 | - Repo Lint 19 | - Go-CI 20 | - build_test 21 | - Rust-CI 22 | - `otap-dataflow` component 23 | - test_and_coverage 24 | - fmt 25 | - clippy 26 | - deny 27 | - bench 28 | - docs 29 | - structure_check 30 | -------------------------------------------------------------------------------- /.github/workflows/fossa.yml: -------------------------------------------------------------------------------- 1 | name: FOSSA scanning 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | fossa: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 16 | 17 | - uses: fossas/fossa-action@3ebcea1862c6ffbd5cf1b4d0bd6b3fe7bd6f2cac # v1.7.0 18 | with: 19 | api-key: ${{secrets.FOSSA_API_KEY}} 20 | team: OpenTelemetry 21 | -------------------------------------------------------------------------------- /.github/workflows/ossf-scorecard.yml: -------------------------------------------------------------------------------- 1 | name: OSSF Scorecard 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | schedule: 8 | - cron: "19 1 * * 6" # once a week 9 | workflow_dispatch: 10 | 11 | permissions: read-all 12 | 13 | jobs: 14 | analysis: 15 | runs-on: ubuntu-latest 16 | permissions: 17 | # Needed for Code scanning upload 18 | security-events: write 19 | # Needed for GitHub OIDC token if publish_results is true 20 | id-token: write 21 | steps: 22 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 23 | with: 24 | persist-credentials: false 25 | 26 | - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 27 | with: 28 | results_file: results.sarif 29 | results_format: sarif 30 | publish_results: true 31 | 32 | # Upload the results as artifacts (optional). Commenting out will disable 33 | # uploads of run results in SARIF format to the repository Actions tab. 34 | # https://docs.github.com/en/actions/advanced-guides/storing-workflow-data-as-artifacts 35 | - name: "Upload artifact" 36 | uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 37 | with: 38 | name: SARIF file 39 | path: results.sarif 40 | retention-days: 5 41 | 42 | # Upload the results to GitHub's code scanning dashboard (optional). 43 | # Commenting out will disable upload of results to your repo's Code Scanning dashboard 44 | - name: "Upload to code-scanning" 45 | uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 46 | with: 47 | sarif_file: results.sarif 48 | -------------------------------------------------------------------------------- /.github/workflows/repo-lint.yaml: -------------------------------------------------------------------------------- 1 | # This workflow is for various linting tasks that act in bulk over the repository. 2 | # Scoped linting (i.e. code formatting) should be done in the respective language-specific workflows. 3 | name: Repo Lint 4 | 5 | permissions: read-all 6 | 7 | on: 8 | pull_request: 9 | branches: [ main ] 10 | 11 | jobs: 12 | markdownlint: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Harden the runner (Audit all outbound calls) 16 | uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 17 | with: 18 | egress-policy: audit 19 | 20 | - name: check out code 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 22 | 23 | - name: run markdownlint 24 | uses: DavidAnson/markdownlint-cli2-action@05f32210e84442804257b2a6f20b273450ec8265 # v19.1.0 25 | with: 26 | globs: | 27 | **/*.md 28 | 29 | sanity: 30 | runs-on: ubuntu-latest 31 | 32 | steps: 33 | - name: check out code 34 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 35 | 36 | - name: run sanitycheck.py 37 | run: python3 ./tools/sanitycheck.py 38 | -------------------------------------------------------------------------------- /.github/workflows/rust-audit.yml: -------------------------------------------------------------------------------- 1 | name: Rust-Audit 2 | permissions: 3 | contents: read 4 | 5 | on: 6 | push: 7 | branches: [main] 8 | paths: 9 | - 'rust/otap-dataflow/Cargo.toml' 10 | schedule: 11 | - cron: '0 2 * * 1' # run at 2 AM UTC every Monday 12 | 13 | jobs: 14 | audit: 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | folder: [otap-dataflow] 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 22 | - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b 23 | with: 24 | toolchain: stable 25 | - name: cargo install cargo-audit 26 | run: cargo install cargo-audit 27 | - name: cargo audit ${{ matrix.folder }} 28 | run: cargo audit 29 | working-directory: ./rust/${{ matrix.folder }} 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | dist/ 3 | 4 | # GoLand IDEA 5 | /.idea/ 6 | *.iml 7 | 8 | # VS Code 9 | .vscode/ 10 | .devcontainer/ 11 | 12 | # Emacs 13 | *~ 14 | \#*\# 15 | 16 | # Miscellaneous files 17 | *.sw[op] 18 | *.DS_Store 19 | 20 | # Go work sum can break things if checked-in. It cannot be checked-in with its own sum. 21 | go.work.sum 22 | 23 | # Coverage 24 | coverage.out 25 | coverage.txt 26 | coverage.html 27 | 28 | # Wix 29 | *.wixobj 30 | *.wixpdb 31 | 32 | # Binaries for programs and plugins 33 | *.exe 34 | *.exe~ 35 | *.dll 36 | *.so 37 | *.dylib 38 | 39 | # Test binary, built with `go test -c` 40 | *.test 41 | 42 | # Output of the go coverage tool, specifically when used with LiteIDE 43 | *.out 44 | 45 | # Dependency directories (remove the comment below to include it) 46 | # vendor/ 47 | 48 | # Python 49 | __pycache__/ 50 | 51 | # Benchmarks output 52 | /output/ 53 | 54 | # PerfTest pipeline output 55 | /tools/pipeline_perf_test/results/ 56 | 57 | # Collector test binary 58 | collector/cmd/otelarrowcol/otelarrowcol 59 | tools/pipeline_perf_test/load_generator/loadgen-manifest.yaml.tmp 60 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "proto/opentelemetry-proto"] 2 | path = proto/opentelemetry-proto 3 | url = git@github.com:open-telemetry/opentelemetry-proto.git 4 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | errcheck: 3 | check-type-assertions: true 4 | goconst: 5 | min-len: 2 6 | min-occurrences: 3 7 | gocritic: 8 | enabled-tags: 9 | - diagnostic 10 | - experimental 11 | - opinionated 12 | - performance 13 | - style 14 | govet: 15 | disable: 16 | - fieldalignment # too strict 17 | - shadow 18 | nolintlint: 19 | require-explanation: true 20 | require-specific: true 21 | 22 | linters: 23 | disable-all: true 24 | enable: 25 | - bodyclose 26 | - deadcode 27 | - depguard 28 | - dogsled 29 | - errcheck 30 | - exportloopref 31 | - exhaustive 32 | - goconst 33 | - gofmt 34 | - goimports 35 | - gocyclo 36 | - gosec 37 | - gosimple 38 | - govet 39 | - ineffassign 40 | - misspell 41 | - nolintlint 42 | - nakedret 43 | - prealloc 44 | - predeclared 45 | - staticcheck 46 | - structcheck 47 | - thelper 48 | - tparallel 49 | - typecheck 50 | - unconvert 51 | - unparam 52 | - varcheck 53 | - whitespace 54 | 55 | run: 56 | issues-exit-code: 1 -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/otel-arrow-adapter.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | # Default state for all rules 2 | default: true 3 | 4 | # MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md 5 | MD013: 6 | code_blocks: false 7 | tables: false 8 | headings: false 9 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | ##################################################### 2 | # 3 | # List of approvers for this repository 4 | # 5 | ##################################################### 6 | # 7 | # Learn about membership in OpenTelemetry community: 8 | # https://github.com/open-telemetry/community/blob/main/community-membership.md 9 | # 10 | # 11 | # Learn about CODEOWNERS file format: 12 | # https://help.github.com/en/articles/about-code-owners 13 | # 14 | 15 | * @open-telemetry/arrow-approvers 16 | 17 | CODEOWNERS @lquerel @jmacd 18 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright The OpenTelemetry Authors 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # This Dockerfile builds the OpenTelemetry Protocol with Apache Arrow 5 | # Collector from the code in this repository. It builds using the 6 | # "otelarrowcol" configuration. See collector/otelarrowcol-build.yaml 7 | # for the components that are included in the build, which are all of 8 | # those with sources in this repository plus a few commonly useful 9 | # accessories (e.g., the profiler extension). 10 | FROM golang:1.21@sha256:4746d26432a9117a5f58e95cb9f954ddf0de128e9d5816886514199316e4a2fb AS sandbox 11 | 12 | WORKDIR /otel-arrow 13 | COPY . . 14 | ENV CGO_ENABLED=0 15 | 16 | # Note the version MUST MATCH otelarrowcol-build.yaml 17 | RUN go install go.opentelemetry.io/collector/cmd/builder@v0.89.0 18 | 19 | # This command generates main.go, go.mod but does not update deps. 20 | RUN builder --skip-compilation --skip-get-modules --config=collector/otelarrowcol-build.yaml 21 | 22 | # This build will update the go.mod, using the checked-in go.work file 23 | # in the repository. 24 | RUN go build -o otelarrowcol ./collector/cmd/otelarrowcol 25 | 26 | # This build uses an Alpine Linux container. 27 | FROM alpine@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c AS release 28 | COPY --from=sandbox /otel-arrow/otelarrowcol / 29 | 30 | # Network ports 31 | # 4317 - OpenTelemetry gRPC services: 32 | # - OpenTelemetry Protocol with Apache Arrow 33 | # - OpenTelemetry Protocol (OTLP) 34 | # 1777 - Profiling support 35 | EXPOSE 4317/tcp 1777/tcp 36 | 37 | ENTRYPOINT ["/otelarrowcol"] 38 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | collector/grpcutil contains code derived from gRPC-Go: 2 | 3 | Copyright 2014 gRPC authors. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security 2 | 3 | OTel Arrow takes the security of our project seriously. 4 | 5 | ## Reporting a Vulnerability 6 | 7 | **Please do not report security vulnerabilities through public GitHub issues.** 8 | 9 | Please see the [OpenTelemetry org-wide security 10 | policy](https://github.com/open-telemetry/.github/blob/main/SECURITY.md) for 11 | instructions on reporting vulnerabilities, including how to contact the 12 | maintainers technical committee of the project secure using a secure channel. 13 | 14 | ## Threat Model 15 | 16 | See the following [threat model assessment](docs/threat_model_assessment.md) for 17 | more details. 18 | -------------------------------------------------------------------------------- /collector/cmd/otelarrowcol/main_others.go: -------------------------------------------------------------------------------- 1 | // Code generated by "go.opentelemetry.io/collector/cmd/builder". DO NOT EDIT. 2 | 3 | //go:build !windows 4 | 5 | package main 6 | 7 | import "go.opentelemetry.io/collector/otelcol" 8 | 9 | func run(params otelcol.CollectorSettings) error { 10 | return runInteractive(params) 11 | } 12 | -------------------------------------------------------------------------------- /collector/cmd/otelarrowcol/main_windows.go: -------------------------------------------------------------------------------- 1 | // Code generated by "go.opentelemetry.io/collector/cmd/builder". DO NOT EDIT. 2 | 3 | //go:build windows 4 | 5 | package main 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "golang.org/x/sys/windows" 11 | "golang.org/x/sys/windows/svc" 12 | "go.opentelemetry.io/collector/otelcol" 13 | ) 14 | 15 | func run(params otelcol.CollectorSettings) error { 16 | // No need to supply service name when startup is invoked through 17 | // the Service Control Manager directly. 18 | if err := svc.Run("", otelcol.NewSvcHandler(params)); err != nil { 19 | if errors.Is(err, windows.ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) { 20 | // Per https://learn.microsoft.com/en-us/windows/win32/api/winsvc/nf-winsvc-startservicectrldispatchera#return-value 21 | // this means that the process is not running as a service, so run interactively. 22 | return runInteractive(params) 23 | } 24 | 25 | return fmt.Errorf("failed to start collector server: %w", err) 26 | } 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /collector/examples/README.md: -------------------------------------------------------------------------------- 1 | # OpenTelemetry Protocol with Apache Arrow examples 2 | 3 | Examples demonstrating how to configure and test an OpenTelemetry Collector with 4 | OpenTelemetry Protocol with Apache Arrow components. 5 | 6 | To run any of the following examples, first build a collector using one of the 7 | methods document in [BUILDING](../BUILDING.md). 8 | 9 | - [`bridge`](./bridge/README.md): A compression bridge between "edge" (gateway) 10 | and "saas" (reverse gateway) collectors. 11 | - [`metadata-bridge`](./metadata-bridge/README.md): A compression bridge between 12 | "edge" (gateway) and "saas" (reverse gateway) collectors with metadata 13 | support, allowing request headers through. 14 | - [`recorder`](./recorder/README.md): A collector with support for recording 15 | data files for diagnostic and benchmark purposes. 16 | - [`shutdown`](./shutdown/README.md): Sets up two OTAP bridges with different 17 | stream lifetimes to exercise gRPC stream failure modes. 18 | 19 | For each example directory, change your the working directory to the example. 20 | Set a `COLLECTOR` in your shell according to the build method used. 21 | 22 | If you used docker, 23 | 24 | ```shell 25 | COLLECTOR=`docker run -v `pwd`:/config -w /config otelarrowcol` 26 | ``` 27 | 28 | if you used an installed Golang toolchain and local sources, 29 | 30 | ```shell 31 | COLLECTOR=../../../bin/otelarrowcol 32 | ``` 33 | 34 | and if you used the `go install` method, 35 | 36 | ```shell 37 | COLLECTOR=${GOPATH}/bin/otelarrowcol 38 | ``` 39 | -------------------------------------------------------------------------------- /collector/examples/bridge/README.md: -------------------------------------------------------------------------------- 1 | # Example: OpenTelemetry Protocol with Apache Arrow bridge 2 | 3 | This example demonstrates the most basic setup for sending and receiving 4 | OpenTelemetry Protocol with Apache Arrow data. 5 | 6 | To run the exporting side of the bridge, 7 | 8 | ```shell 9 | $COLLECTOR --config edge-collector.yaml 10 | ``` 11 | 12 | To run the receiving side of the bridge, 13 | 14 | ```shell 15 | $COLLECTOR --config saas-collector.yaml 16 | ``` 17 | 18 | You may use the 19 | [`telemetrygen`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/cmd/telemetrygen/README.md) 20 | generator to exercise this pipeline. For example, to send traces: 21 | 22 | ```shell 23 | telemetrygen traces --otlp-insecure --duration 1000s 24 | ``` 25 | 26 | Prometheus metrics describing the OpenTelemetry Protocol with Apache Arrow 27 | components are available at `127.0.0.1:8888` and `127.0.0.1:8889`. 28 | -------------------------------------------------------------------------------- /collector/examples/bridge/edge-collector.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | # otelarrow/standard is an OTelArrow receiver. 3 | # it uses port 4317, the standard port for OTLP/gRPC. 4 | # There are no required configuration fields. 5 | otelarrow/standard: 6 | 7 | exporters: 8 | # otelarrow/arrow is an OTel-Arrow exporter. 9 | otelarrow/arrow: 10 | # For the sample configuration, the other side of the bridge 11 | # runs on port 8100. 12 | endpoint: 127.0.0.1:8100 13 | 14 | # For demonstration purposes, use an insecure port. This would 15 | # also be normal for a collector behind a loadbalancer that 16 | # terminates TLS. 17 | tls: 18 | insecure: true 19 | 20 | # Static headers will be attached to every export. 21 | headers: 22 | - X-Scope-OrgID: example_tenant 23 | 24 | # wait_for_ready lets the producer block until the connection 25 | # is ready. 26 | wait_for_ready: true 27 | 28 | debug: 29 | 30 | service: 31 | pipelines: 32 | traces: 33 | receivers: [otelarrow/standard] 34 | exporters: [otelarrow/arrow, debug] 35 | metrics: 36 | receivers: [otelarrow/standard] 37 | exporters: [otelarrow/arrow, debug] 38 | 39 | telemetry: 40 | resource: 41 | "service.name": "example-bridge" 42 | metrics: 43 | address: 127.0.0.1:8888 44 | level: detailed 45 | logs: 46 | level: info 47 | -------------------------------------------------------------------------------- /collector/examples/bridge/saas-collector.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | # otelarrow is an OTel Arrow receiver that will operate as the SaaS-side 3 | # of the bridge. 4 | otelarrow: 5 | protocols: 6 | grpc: 7 | # Port 5000 is the endpoint used in edge-collector. 8 | endpoint: 127.0.0.1:8100 9 | 10 | # Include metadata so that the exporter can copy it 11 | # to the next hop. 12 | include_metadata: true 13 | 14 | keepalive: 15 | server_parameters: 16 | max_connection_age: 10s 17 | max_connection_age_grace: 10s 18 | 19 | exporters: 20 | debug: 21 | 22 | otlphttp: 23 | # You can use an HTTP listener on port 5001 to see the headers 24 | # and raw data. 25 | endpoint: http://127.0.0.1:8101 26 | compression: none 27 | 28 | # Associate the headers_setter extension with this exporter 29 | # so that it passes through headers set on the edge collector. 30 | auth: 31 | authenticator: headers_setter 32 | 33 | extensions: 34 | # Configure the headers_setter extension to propagate the 35 | # X-Scope-OrgID property in the outgoing context. 36 | headers_setter: 37 | headers: 38 | - key: X-Scope-OrgID 39 | from_context: X-Scope-OrgID 40 | 41 | service: 42 | extensions: [headers_setter] 43 | pipelines: 44 | traces: 45 | receivers: [otelarrow] 46 | exporters: [debug, otlphttp] 47 | 48 | metrics: 49 | receivers: [otelarrow] 50 | exporters: [debug, otlphttp] 51 | 52 | telemetry: 53 | metrics: 54 | address: 127.0.0.1:8889 55 | level: normal 56 | -------------------------------------------------------------------------------- /collector/examples/metadata-bridge/README.md: -------------------------------------------------------------------------------- 1 | # Example: OpenTelemetry Protocol with Apache Arrow metadata bridge 2 | 3 | This example demonstrates how to setup basic authentication and propagate header 4 | context through the bridge. 5 | 6 | To run the exporting side of the bridge, 7 | 8 | ```shell 9 | $COLLECTOR --config edge-collector.yaml 10 | ``` 11 | 12 | To run the receiving side of the bridge, 13 | 14 | ```shell 15 | $COLLECTOR --config saas-collector.yaml 16 | ``` 17 | 18 | You may use the 19 | [`telemetrygen`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/cmd/telemetrygen/README.md) 20 | generator to exercise this pipeline. For example, to send traces: 21 | 22 | ```shell 23 | telemetrygen traces --otlp-insecure --duration 1000s 24 | ``` 25 | -------------------------------------------------------------------------------- /collector/examples/metadata-bridge/saas-collector.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | # otelarrow is an OTLP-Arrow receiver that will operate as the SaaS-side 3 | # of the bridge. 4 | otelarrow: 5 | protocols: 6 | grpc: 7 | # Port 8100 is the endpoint used in edge-collector. 8 | endpoint: 127.0.0.1:8100 9 | 10 | # include_metadata is required for the receiver to pass 11 | # per-request metadata through to the pipeline. This 12 | # is supported for both arrow and standard modes. 13 | include_metadata: true 14 | # perform an auth on the SaaS 15 | auth: 16 | authenticator: basicauth 17 | 18 | exporters: 19 | debug: 20 | verbosity: normal 21 | 22 | otlphttp: 23 | # You can use an HTTP listener on port 8101 to see the headers 24 | # and raw data. 25 | endpoint: http://127.0.0.1:8101 26 | compression: none 27 | 28 | # Associate the headers_setter extension with this exporter 29 | # so that it passes through headers set on the edge collector. 30 | auth: 31 | authenticator: headers_setter 32 | 33 | extensions: 34 | # Configure the headers_setter extension to propagate the 35 | # X-Scope-OrgID property in the outgoing context. 36 | headers_setter: 37 | headers: 38 | - key: X-Scope-OrgID 39 | from_context: X-Scope-OrgID 40 | - key: X-Cluster-Name 41 | from_context: X-Cluster-Name 42 | - key: Authorization 43 | from_context: Authorization 44 | # Configure basic auth for incoming data. 45 | basicauth: 46 | htpasswd: 47 | inline: | 48 | testuser:testpw 49 | arrowstream:arrowpw 50 | 51 | service: 52 | extensions: [headers_setter, basicauth] 53 | pipelines: 54 | traces: 55 | receivers: [otelarrow] 56 | exporters: [debug, otlphttp] 57 | 58 | telemetry: 59 | metrics: 60 | address: 127.0.0.1:8889 61 | -------------------------------------------------------------------------------- /collector/examples/recorder/README.md: -------------------------------------------------------------------------------- 1 | # Example: record and replay OpenTelemetry Protocol data 2 | 3 | First, run `make otelarrowcol` in the top-level directory. 4 | 5 | To execute the data recorder, for example, where GOOS=darwin and GOARCH=arm64: 6 | 7 | ```shell 8 | $COLLECTOR --config target.yaml 9 | ``` 10 | 11 | During this phase, data received is copied through a "loopback" Arrow pipeline 12 | that encodes and decodes the data and should second, identical copies of the 13 | inputs. 14 | 15 | When enough data has been collected to `first.traces.json` and 16 | `first.metrics.json`, you may stop the target collector. 17 | 18 | Note this configuration exercises Arrow in the loopback pipeline, and if Arrow 19 | is failing for any reason the sender will receive errors. When this is 20 | happening, modify the forwarding exporter to disable Arrow in the recording 21 | step, e.g.,: 22 | 23 | ```yml 24 | exporters: 25 | otelarrow/forward: 26 | arrow: 27 | enabled: false 28 | ``` 29 | 30 | Then, to re-exercise the same data though an Arrow pipeline using data recorded 31 | in the first step, run: 32 | 33 | ```shell 34 | $COLLECTOR --config replay.yaml 35 | ``` 36 | 37 | Note that this example only supports traces and metrics. Logs are not supported 38 | in these configurations. 39 | -------------------------------------------------------------------------------- /collector/examples/recorder/replay.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otelarrow/first: 3 | protocols: 4 | grpc: 5 | endpoint: 127.0.0.1:8081 6 | otelarrow/second: 7 | protocols: 8 | grpc: 9 | endpoint: 127.0.0.1:8082 10 | file/first_metrics: 11 | path: "first.metrics.json" 12 | throttle: 0 13 | file/first_traces: 14 | path: "first.traces.json" 15 | throttle: 0 16 | 17 | exporters: 18 | otelarrow/forward: 19 | endpoint: 127.0.0.1:8082 20 | wait_for_ready: true 21 | arrow: 22 | num_streams: 1 23 | disable_downgrade: true 24 | tls: 25 | insecure: true 26 | retry_on_failure: 27 | enabled: false 28 | sending_queue: 29 | enabled: false 30 | timeout: 10s 31 | debug/first: 32 | debug/second: 33 | file/second_traces: 34 | path: "second.traces.json" 35 | file/second_metrics: 36 | path: "second.metrics.json" 37 | 38 | service: 39 | pipelines: 40 | traces/first: 41 | receivers: [file/first_traces] 42 | exporters: [debug/first, otelarrow/forward] 43 | metrics/first: 44 | receivers: [file/first_metrics] 45 | exporters: [debug/first, otelarrow/forward] 46 | 47 | traces/second: 48 | receivers: [otelarrow/second] 49 | exporters: [debug/second, file/second_traces] 50 | metrics/second: 51 | receivers: [otelarrow/second] 52 | exporters: [debug/second, file/second_metrics] 53 | 54 | telemetry: 55 | resource: 56 | "service.name": "data-replayer" 57 | metrics: 58 | address: 127.0.0.1:8888 59 | level: detailed 60 | logs: 61 | level: debug 62 | -------------------------------------------------------------------------------- /collector/examples/recorder/target.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | # Send test data to the first pipeline using standard OTel ports 3 | # 4317 and 4318. 4 | otelarrow/first: 5 | protocols: 6 | grpc: 7 | endpoint: 127.0.0.1:4317 8 | http: 9 | endpoint: 127.0.0.1:4318 10 | 11 | # Data will be repeated to the second pipeline via Arrow. 12 | otelarrow/second: 13 | protocols: 14 | grpc: 15 | endpoint: 127.0.0.1:8100 16 | 17 | exporters: 18 | otelarrow/forward: 19 | # Data is forwarded via Arrow to this receiver. 20 | endpoint: 127.0.0.1:8100 21 | wait_for_ready: true 22 | arrow: 23 | num_streams: 1 24 | disable_downgrade: true 25 | tls: 26 | insecure: true 27 | retry_on_failure: 28 | enabled: false 29 | sending_queue: 30 | enabled: false 31 | timeout: 1s 32 | file/first_traces: 33 | path: "first.traces.json" 34 | file/second_traces: 35 | path: "second.traces.json" 36 | file/first_metrics: 37 | path: "first.metrics.json" 38 | file/second_metrics: 39 | path: "second.metrics.json" 40 | debug/first: 41 | debug/second: 42 | 43 | service: 44 | pipelines: 45 | traces/first: 46 | receivers: [otelarrow/first] 47 | exporters: [debug/first, file/first_traces, otelarrow/forward] 48 | metrics/first: 49 | receivers: [otelarrow/first] 50 | exporters: [debug/first, file/first_metrics, otelarrow/forward] 51 | 52 | traces/second: 53 | receivers: [otelarrow/second] 54 | exporters: [debug/second, file/second_traces] 55 | metrics/second: 56 | receivers: [otelarrow/second] 57 | exporters: [debug/second, file/second_metrics] 58 | 59 | telemetry: 60 | resource: 61 | "service.name": "data-recorder" 62 | metrics: 63 | address: 127.0.0.1:8888 64 | level: detailed 65 | logs: 66 | level: debug 67 | -------------------------------------------------------------------------------- /collector/examples/shutdown/README.md: -------------------------------------------------------------------------------- 1 | # Shutdown test example 2 | 3 | This example configures three collectors with two Arrow bridges with different 4 | stream lifetimes. It can be used to experiment with mismatched lifetime and 5 | keepalive settings, too. 6 | 7 | To run this setup, start the three collectors as follows: 8 | 9 | ```shell 10 | $COLLECTOR --config saas-collector.yaml 11 | $COLLECTOR --config middle-collector.yaml 12 | $COLLECTOR --config edge-collector.yaml 13 | ``` 14 | 15 | You may use the 16 | [`telemetrygen`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/cmd/telemetrygen/README.md) 17 | generator to exercise this pipeline. For example, to send traces: 18 | 19 | ```shell 20 | telemetrygen traces --otlp-insecure --duration 1000s 21 | ``` 22 | 23 | To test debug-level logging, change the service configurations, e.g.,: 24 | 25 | ```yml 26 | logs: 27 | level: debug 28 | ``` 29 | 30 | To test a condition where max-stream-lifetime is too short, lower keepalive 31 | `max_connection_age_grace`, e.g., 32 | 33 | ```yml 34 | keepalive: 35 | server_parameters: 36 | max_connection_age: 5s 37 | max_connection_age_grace: 10s 38 | ``` 39 | -------------------------------------------------------------------------------- /collector/examples/shutdown/edge-collector.yaml: -------------------------------------------------------------------------------- 1 | # This collector listens on the standard OTLP/gRPC port (4317) 2 | # and forwards to an OTel-Arrow on port 8100. 3 | 4 | receivers: 5 | otelarrow: 6 | 7 | exporters: 8 | otelarrow: 9 | endpoint: 127.0.0.1:8100 10 | 11 | tls: 12 | insecure: true 13 | 14 | wait_for_ready: true 15 | 16 | arrow: 17 | max_stream_lifetime: 120s 18 | 19 | service: 20 | pipelines: 21 | traces: 22 | receivers: [otelarrow] 23 | exporters: [otelarrow] 24 | metrics: 25 | receivers: [otelarrow] 26 | exporters: [otelarrow] 27 | logs: 28 | receivers: [otelarrow] 29 | exporters: [otelarrow] 30 | 31 | telemetry: 32 | metrics: 33 | level: none 34 | logs: 35 | level: info 36 | -------------------------------------------------------------------------------- /collector/examples/shutdown/middle-collector.yaml: -------------------------------------------------------------------------------- 1 | # This collector listens for OTLP/gRPC on port 8100 and forwards to an 2 | # OTel-Arrow on port 8101. 3 | 4 | receivers: 5 | otelarrow: 6 | protocols: 7 | grpc: 8 | endpoint: 127.0.0.1:8100 9 | keepalive: 10 | server_parameters: 11 | max_connection_age: 5s 12 | max_connection_age_grace: 20s 13 | 14 | exporters: 15 | otelarrow: 16 | endpoint: 127.0.0.1:8101 17 | 18 | tls: 19 | insecure: true 20 | 21 | wait_for_ready: true 22 | 23 | arrow: 24 | max_stream_lifetime: 120s 25 | 26 | service: 27 | pipelines: 28 | traces: 29 | receivers: [otelarrow] 30 | exporters: [otelarrow] 31 | metrics: 32 | receivers: [otelarrow] 33 | exporters: [otelarrow] 34 | logs: 35 | receivers: [otelarrow] 36 | exporters: [otelarrow] 37 | 38 | telemetry: 39 | metrics: 40 | level: none 41 | logs: 42 | level: info 43 | -------------------------------------------------------------------------------- /collector/examples/shutdown/saas-collector.yaml: -------------------------------------------------------------------------------- 1 | # This collector listens for OTLP/gRPC on port 8101 and forwards to a 2 | # debug exporter. 3 | 4 | receivers: 5 | otelarrow: 6 | protocols: 7 | grpc: 8 | endpoint: 127.0.0.1:8101 9 | 10 | keepalive: 11 | server_parameters: 12 | max_connection_age: 5s 13 | max_connection_age_grace: 60s 14 | 15 | exporters: 16 | debug: 17 | 18 | service: 19 | pipelines: 20 | traces: 21 | receivers: [otelarrow] 22 | exporters: [debug] 23 | 24 | metrics: 25 | receivers: [otelarrow] 26 | exporters: [debug] 27 | 28 | logs: 29 | receivers: [otelarrow] 30 | exporters: [debug] 31 | 32 | telemetry: 33 | metrics: 34 | level: none 35 | logs: 36 | level: info 37 | -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | otlp_*.pg 2 | otlp_*.pb 3 | otlp_*.json.zst 4 | otap_*.pb 5 | otap_*.json.zst 6 | hipstershop_*.pb 7 | generated_otlp_*.pb 8 | 9 | -------------------------------------------------------------------------------- /data/multivariate-metrics.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/data/multivariate-metrics.pb -------------------------------------------------------------------------------- /data_analysis/hipster_results/average_improvement_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/data_analysis/hipster_results/average_improvement_heatmap.png -------------------------------------------------------------------------------- /data_analysis/hipster_results/side_by_side_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/data_analysis/hipster_results/side_by_side_heatmap.png -------------------------------------------------------------------------------- /data_analysis/prod_results/average_improvement_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/data_analysis/prod_results/average_improvement_heatmap.png -------------------------------------------------------------------------------- /data_analysis/prod_results/side_by_side_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/data_analysis/prod_results/side_by_side_heatmap.png -------------------------------------------------------------------------------- /docs/img/OTEL - Chaos engineering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/OTEL - Chaos engineering.png -------------------------------------------------------------------------------- /docs/img/OTEL - TMA scope.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/OTEL - TMA scope.png -------------------------------------------------------------------------------- /docs/img/OTEL - validation process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/OTEL - validation process.png -------------------------------------------------------------------------------- /docs/img/average_improvement_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/average_improvement_heatmap.png -------------------------------------------------------------------------------- /docs/img/collector_internal_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/collector_internal_overview.png -------------------------------------------------------------------------------- /docs/img/compression_ratio_summary_multivariate_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/compression_ratio_summary_multivariate_metrics.png -------------------------------------------------------------------------------- /docs/img/compression_ratio_summary_std_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/compression_ratio_summary_std_metrics.png -------------------------------------------------------------------------------- /docs/img/logs_compression_rate_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/logs_compression_rate_benchmark.png -------------------------------------------------------------------------------- /docs/img/logs_end_to_end_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/logs_end_to_end_benchmark.png -------------------------------------------------------------------------------- /docs/img/metrics_compression_rate_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/metrics_compression_rate_benchmark.png -------------------------------------------------------------------------------- /docs/img/metrics_end_to_end_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/metrics_end_to_end_benchmark.png -------------------------------------------------------------------------------- /docs/img/side_by_side_heatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/side_by_side_heatmap.png -------------------------------------------------------------------------------- /docs/img/traces_compressed_msg_size.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/traces_compressed_msg_size.png -------------------------------------------------------------------------------- /docs/img/traces_compression_rate_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/traces_compression_rate_benchmark.png -------------------------------------------------------------------------------- /docs/img/traces_end_to_end_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/traces_end_to_end_benchmark.png -------------------------------------------------------------------------------- /docs/img/traces_uncompressed_msg_size.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/traces_uncompressed_msg_size.png -------------------------------------------------------------------------------- /docs/img/traffic_reduction_use_case.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/docs/img/traffic_reduction_use_case.png -------------------------------------------------------------------------------- /go/pkg/arrow/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package arrow 19 | 20 | const AbsentFieldID = -1 21 | -------------------------------------------------------------------------------- /go/pkg/arrow/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package arrow provides a set of utility functions to access Arrow data structures. 19 | package arrow 20 | -------------------------------------------------------------------------------- /go/pkg/arrow/from_sparseunion.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package arrow 19 | 20 | import ( 21 | "github.com/apache/arrow-go/v18/arrow" 22 | ) 23 | 24 | func StructFromSparseUnion(dt *arrow.SparseUnionType, code int8) *arrow.StructType { 25 | codes := dt.TypeCodes() 26 | for i, c := range codes { 27 | if c == code { 28 | return dt.Fields()[i].Type.(*arrow.StructType) 29 | } 30 | } 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /go/pkg/benchmark/config.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package benchmark 16 | 17 | type Config struct { 18 | Compression bool 19 | Stats bool 20 | } 21 | -------------------------------------------------------------------------------- /go/pkg/benchmark/dataset/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package dataset defines the concept of dataset used in this benchmarking framework. 19 | package dataset 20 | -------------------------------------------------------------------------------- /go/pkg/benchmark/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package benchmark is a framework for benchmarking the performance of the OTLP protocol vs the OTLP Arrow protocol. 19 | package benchmark 20 | -------------------------------------------------------------------------------- /go/pkg/benchmark/profileable.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package benchmark 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | "strings" 21 | ) 22 | 23 | type ProfileableSystem interface { 24 | Name() string 25 | Tags() []string 26 | DatasetSize() int 27 | CompressionAlgorithm() CompressionAlgorithm 28 | 29 | StartProfiling(writer io.Writer) 30 | EndProfiling(writer io.Writer) 31 | 32 | InitBatchSize(writer io.Writer, batchSize int) 33 | PrepareBatch(writer io.Writer, startAt, size int) 34 | ConvertOtlpToOtlpArrow(writer io.Writer, startAt, size int) 35 | 36 | Process(writer io.Writer) string 37 | 38 | Serialize(writer io.Writer) ([][]byte, error) 39 | Deserialize(writer io.Writer, buffers [][]byte) 40 | ConvertOtlpArrowToOtlp(writer io.Writer) 41 | 42 | Clear() 43 | ShowStats() 44 | } 45 | 46 | func ProfileableSystemID(ps ProfileableSystem) string { 47 | return fmt.Sprintf("%s:%s", ps.Name(), strings.Join(ps.Tags()[:], "+")) 48 | } 49 | -------------------------------------------------------------------------------- /go/pkg/benchmark/profileable/arrow/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package arrow implements the Profile interface for the OTLP Arrow protocol. 19 | package arrow 20 | -------------------------------------------------------------------------------- /go/pkg/benchmark/profileable/arrow/otlp_arrow_profiler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arrow 16 | 17 | import ( 18 | "path/filepath" 19 | "testing" 20 | 21 | "github.com/open-telemetry/otel-arrow/pkg/benchmark" 22 | "github.com/open-telemetry/otel-arrow/pkg/benchmark/dataset" 23 | ) 24 | 25 | func TestOtlpArrowMetricsProfiler(t *testing.T) { 26 | t.Skip("Skipping this test because it's not fully implemented yet") 27 | t.Parallel() 28 | 29 | // Configuration 30 | cfg := &benchmark.Config{} 31 | 32 | maxIter := uint64(10) 33 | systemToProfile := NewMetricsProfileable([]string{"multivariate"}, dataset.NewFakeMetricsDataset(1000), cfg) 34 | profiler := benchmark.NewProfiler([]int{10, 100, 1000}, filepath.Join(t.TempDir(), "tmpfile"), 2) 35 | if err := profiler.Profile(systemToProfile, maxIter); err != nil { 36 | t.Errorf("expected no error, got %v", err) 37 | } 38 | profiler.CheckProcessingResults() 39 | profiler.PrintResults(maxIter) 40 | profiler.ExportMetricsTimesCSV("otlp_arrow_metrics") 41 | profiler.ExportMetricsBytesCSV("otlp_arrow_metrics") 42 | } 43 | 44 | func TestOtlpArrowLogsProfiler(t *testing.T) { 45 | t.Parallel() 46 | } 47 | 48 | func TestOtlpArrowTracesProfiler(t *testing.T) { 49 | t.Parallel() 50 | } 51 | -------------------------------------------------------------------------------- /go/pkg/benchmark/profileable/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package profileable defines the different protocols that can be profiled. 19 | package profileable 20 | -------------------------------------------------------------------------------- /go/pkg/benchmark/profileable/otlp/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package otlp implements the Profile interface for the OTLP protocol. 19 | package otlp 20 | -------------------------------------------------------------------------------- /go/pkg/datagen/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package datagen is a basic framework for generating fake telemetry data for benchmarking. 19 | package datagen 20 | -------------------------------------------------------------------------------- /go/pkg/internal/debug/assert_off.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | //go:build !assert 5 | 6 | package debug 7 | 8 | // Assert will not panic or print any message when assertions are disabled. 9 | // Use --tags=assert in the Go build. 10 | func Assert(cond bool, msg any) {} 11 | 12 | // AssertionsOn indicates when an Assert() will be real. 13 | func AssertionsOn() bool { return false } 14 | -------------------------------------------------------------------------------- /go/pkg/internal/debug/assert_on.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | //go:build assert 5 | 6 | package debug 7 | 8 | import "fmt" 9 | 10 | // Assert will panic and print any message when assertions are enabled. 11 | // Use --tags=assert in the Go build. 12 | func Assert(cond bool, msg any) { 13 | if !cond { 14 | panic(fmt.Sprint(msg)) 15 | } 16 | } 17 | 18 | // AssertionsOn indicates when an Assert() will be real. 19 | func AssertionsOn() bool { return true } 20 | -------------------------------------------------------------------------------- /go/pkg/otel/arrow_record/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: mockgen 2 | mockgen: 3 | go install go.uber.org/mock/mockgen@latest 4 | mkdir -p ./mock 5 | mockgen -package mock . ProducerAPI,ConsumerAPI > mock/mock.go 6 | -------------------------------------------------------------------------------- /go/pkg/otel/arrow_record/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package arrow_record contains the consumer and producer for OTLP Arrow protocol. 19 | package arrow_record 20 | -------------------------------------------------------------------------------- /go/pkg/otel/assert/README.md: -------------------------------------------------------------------------------- 1 | # Assert 2 | 3 | This package supports validation of data that has passed through Arrow encoding 4 | and decoding, recognizing that there may be structural differences for 5 | semantically equivalent data. 6 | 7 | Examples of transformations that are allowed by `assert.Equiv()`: 8 | 9 | - Appearance of duplicate Resource, Scope, and Metric entities 10 | - Order of Resource instances in a Resource list 11 | - Order of Scope items in a Resource 12 | - Order of Span, Metric, and LogRecord items in a Scope 13 | - Order of Links/Events in a Span 14 | - and so on. 15 | 16 | The `assert.Equiv()` method in this package should be used for unittesting and 17 | validation of data in an OTel Arrow pipeline. See the [code](equiv.go) for 18 | details. 19 | -------------------------------------------------------------------------------- /go/pkg/otel/assert/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package assert provides a set of helper functions to assert conditions in OTLP Arrow tests. 19 | package assert 20 | -------------------------------------------------------------------------------- /go/pkg/otel/common/arrow/builder.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arrow 16 | 17 | import ( 18 | "github.com/apache/arrow-go/v18/arrow" 19 | "go.opentelemetry.io/collector/pdata/plog" 20 | "go.opentelemetry.io/collector/pdata/pmetric" 21 | "go.opentelemetry.io/collector/pdata/ptrace" 22 | ) 23 | 24 | type EntityBuilder[T pmetric.Metrics | plog.Logs | ptrace.Traces] interface { 25 | Append(T) error 26 | Build() (arrow.Record, error) 27 | } 28 | -------------------------------------------------------------------------------- /go/pkg/otel/common/arrow/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package arrow 19 | 20 | type ( 21 | Attrs16Config struct { 22 | Sorter Attrs16Sorter 23 | } 24 | 25 | Attrs32Config struct { 26 | Sorter Attrs32Sorter 27 | } 28 | ) 29 | -------------------------------------------------------------------------------- /go/pkg/otel/common/arrow/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package arrow contains common types and functions used to convert OTLP entities into their Arrow representation. 19 | package arrow 20 | -------------------------------------------------------------------------------- /go/pkg/otel/common/arrow/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package arrow 19 | 20 | import "errors" 21 | 22 | var ( 23 | ErrBuilderAlreadyReleased = errors.New("builder already released") 24 | ErrInvalidResourceID = errors.New("invalid resource ID") 25 | ErrInvalidScopeID = errors.New("invalid scope ID") 26 | ) 27 | -------------------------------------------------------------------------------- /go/pkg/otel/common/arrow/optimizer_options.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package arrow 19 | 20 | type Options struct { 21 | Sort bool 22 | Stats bool 23 | } 24 | 25 | func WithSort() func(*Options) { 26 | return func(o *Options) { 27 | o.Sort = true 28 | } 29 | } 30 | 31 | func WithStats() func(*Options) { 32 | return func(o *Options) { 33 | o.Stats = true 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /go/pkg/otel/common/attributes.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "sort" 19 | 20 | "go.opentelemetry.io/collector/pdata/pcommon" 21 | ) 22 | 23 | type ( 24 | AttributeEntry struct { 25 | Key string 26 | Value pcommon.Value 27 | } 28 | 29 | AttributeEntries []AttributeEntry 30 | ) 31 | 32 | var _ sort.Interface = AttributeEntries{} 33 | 34 | func (ae AttributeEntries) Len() int { 35 | return len(ae) 36 | } 37 | 38 | func (ae AttributeEntries) Swap(i, j int) { 39 | ae[i], ae[j] = ae[j], ae[i] 40 | } 41 | 42 | func (ae AttributeEntries) Less(i, j int) bool { 43 | return ae[i].Key < ae[j].Key 44 | } 45 | -------------------------------------------------------------------------------- /go/pkg/otel/common/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package common defines the common types and functions used by the packages logs, metrics and traces. 19 | package common 20 | -------------------------------------------------------------------------------- /go/pkg/otel/common/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package common 19 | 20 | import ( 21 | "errors" 22 | ) 23 | 24 | var ( 25 | ErrInvalidKeyMap = errors.New("invalid key map") 26 | ErrUnsupportedCborType = errors.New("unsupported cbor type") 27 | ErrInvalidTypeConversion = errors.New("invalid type conversion") 28 | 29 | ErrInvalidSpanIDLength = errors.New("invalid span id length") 30 | ErrInvalidTraceIDLength = errors.New("invalid trace id length") 31 | 32 | ErrNotArraySparseUnion = errors.New("not an arrow array.SparseUnion") 33 | ErrNotArrayMap = errors.New("not an arrow array.Map") 34 | ) 35 | -------------------------------------------------------------------------------- /go/pkg/otel/common/otlp/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package otlp contains common types and functions used to convert OTLP Arrow entities into their OTLP representation. 19 | package otlp 20 | -------------------------------------------------------------------------------- /go/pkg/otel/common/otlp/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package otlp 19 | 20 | import "errors" 21 | 22 | var ( 23 | ErrMissingRelatedData = errors.New("missing related data") 24 | ErrInvalidTypeCode = errors.New("invalid type code") 25 | ErrInvalidFieldId = errors.New("invalid field id") 26 | ErrParentIDMissing = errors.New("parent id missing") 27 | ErrInvalidAttrName = errors.New("invalid attribute name") 28 | ErrMissingTypeMetadata = errors.New("missing type metadata") 29 | ) 30 | -------------------------------------------------------------------------------- /go/pkg/otel/common/schema/builder/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package builder decorates the Apache Arrow record/array builders with 19 | // additional functionality to support the concept of adaptive schema and 20 | // transformation nodes. 21 | // 22 | // Everytime that a new data added affects the schema (e.g. a field marked as 23 | // optional becomes required or a dictionary field overflows its index type), 24 | // the schema is updated and the system must be able to re-inject the data into 25 | // the `RecordBuilderExt` in order to avoid data loss. 26 | 27 | package builder 28 | -------------------------------------------------------------------------------- /go/pkg/otel/common/schema/events/counters.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package events 19 | 20 | // Events is a collection of events that occurred while using RecordBuilderExt 21 | type Events struct { 22 | // Dictionary fields that have overflowed and that have been downgraded to 23 | // their value type. 24 | DictionariesWithOverflow map[string]bool 25 | 26 | // Dictionary fields that have their dictionary index type changed. 27 | DictionariesIndexTypeChanged map[string]string 28 | } 29 | -------------------------------------------------------------------------------- /go/pkg/otel/common/schema/transform/identity.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package transform 19 | 20 | import "github.com/apache/arrow-go/v18/arrow" 21 | 22 | // IdentityField is a FieldTransform that returns a copy of the field. 23 | type IdentityField struct { 24 | path string 25 | } 26 | 27 | func NewIdentityField(path string) *IdentityField { 28 | return &IdentityField{path: path} 29 | } 30 | 31 | func (t *IdentityField) Transform(field *arrow.Field) *arrow.Field { 32 | return &arrow.Field{Name: field.Name, Type: field.Type, Nullable: field.Nullable, Metadata: field.Metadata} 33 | } 34 | 35 | func (t *IdentityField) RevertCounters() {} 36 | 37 | func (t *IdentityField) Path() string { 38 | return t.path 39 | } 40 | -------------------------------------------------------------------------------- /go/pkg/otel/common/schema/transform/no_field.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package transform 19 | 20 | import "github.com/apache/arrow-go/v18/arrow" 21 | 22 | // NoField is a FieldTransform that returns nil, so in practice it removes the 23 | // field. 24 | type NoField struct{} 25 | 26 | func (t *NoField) Transform(_ *arrow.Field) *arrow.Field { 27 | return nil 28 | } 29 | 30 | func (t *NoField) RevertCounters() {} 31 | 32 | func (t *NoField) Path() string { 33 | return "undefined" 34 | } 35 | -------------------------------------------------------------------------------- /go/pkg/otel/common/test_utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package common 19 | 20 | import ( 21 | "math/rand" 22 | 23 | "github.com/apache/arrow-go/v18/arrow" 24 | 25 | "github.com/open-telemetry/otel-arrow/pkg/record_message" 26 | ) 27 | 28 | func MixUpArrowRecords(rng *rand.Rand, record arrow.Record, relatedRecords []*record_message.RecordMessage) (bool, arrow.Record, []*record_message.RecordMessage) { 29 | mainRecordChanged := false 30 | 31 | if rng.Intn(100)%2 == 0 { 32 | // exchange one of the related records with the main record 33 | relatedRecordPos := rng.Intn(len(relatedRecords)) 34 | relatedRecord := relatedRecords[relatedRecordPos].Record() 35 | relatedRecords[relatedRecordPos].SetRecord(record) 36 | record = relatedRecord 37 | mainRecordChanged = true 38 | } 39 | 40 | // mix up the related records 41 | payloadTypes := make([]record_message.PayloadType, len(relatedRecords)) 42 | for i := 0; i < len(relatedRecords); i++ { 43 | payloadTypes[i] = relatedRecords[i].PayloadType() 44 | } 45 | rng.Shuffle(len(payloadTypes), func(i, j int) { payloadTypes[i], payloadTypes[j] = payloadTypes[j], payloadTypes[i] }) 46 | for i := 0; i < len(relatedRecords); i++ { 47 | relatedRecords[i].SetPayloadType(payloadTypes[i]) 48 | } 49 | 50 | return mainRecordChanged, record, relatedRecords 51 | } 52 | -------------------------------------------------------------------------------- /go/pkg/otel/constants/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package constants defines the constants used in the sibling packages. 19 | package constants 20 | -------------------------------------------------------------------------------- /go/pkg/otel/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package otel provides a set of functions to convert OTLP entities to OTLP Arrow entities and vice versa. 16 | package otel 17 | -------------------------------------------------------------------------------- /go/pkg/otel/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package otel 19 | 20 | import ( 21 | "errors" 22 | ) 23 | 24 | var ( 25 | ErrMultipleTracesRecords = errors.New("multiple traces records found") 26 | ErrMultipleSpanEventsRecords = errors.New("multiple span events records found") 27 | ErrDuplicatePayloadType = errors.New("duplicate payload type") 28 | UnknownPayloadType = errors.New("unknown payload type") 29 | ) 30 | -------------------------------------------------------------------------------- /go/pkg/otel/logs/arrow/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package arrow contains types and functions used to convert OTLP logs into their Arrow representation. 19 | package arrow 20 | -------------------------------------------------------------------------------- /go/pkg/otel/logs/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package logs provides functions to convert OTLP logs to OTLP Arrow logs and vice versa. 16 | package logs 17 | -------------------------------------------------------------------------------- /go/pkg/otel/logs/otlp/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package otlp contains types and functions used to convert OTLP Arrow logs into their OTLP representation. 19 | package otlp 20 | -------------------------------------------------------------------------------- /go/pkg/otel/logs/otlp/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package otlp 19 | 20 | import "errors" 21 | 22 | var ( 23 | ErrBodyNotSparseUnion = errors.New("body is not a sparse union") 24 | ) 25 | -------------------------------------------------------------------------------- /go/pkg/otel/metrics/arrow/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package arrow contains types and functions used to convert OTLP metrics into their Arrow representation. 19 | package arrow 20 | -------------------------------------------------------------------------------- /go/pkg/otel/metrics/arrow/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package arrow 19 | 20 | import "errors" 21 | 22 | var ( 23 | ErrUnknownMetricType = errors.New("unknown metric type") 24 | ) 25 | -------------------------------------------------------------------------------- /go/pkg/otel/metrics/data_point_test.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metrics 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/require" 21 | "go.opentelemetry.io/collector/pdata/pmetric" 22 | ) 23 | 24 | func TestDataPointSig(t *testing.T) { 25 | t.Parallel() 26 | 27 | ndp := pmetric.NewNumberDataPoint() 28 | ndp.SetStartTimestamp(1) 29 | ndp.SetTimestamp(2) 30 | attrs := ndp.Attributes() 31 | 32 | attrs.PutDouble("k4", 1.) 33 | attrs.PutInt("k1", 2) 34 | attrs.PutBool("k3", false) 35 | attrs.PutStr("k5", "bla") 36 | attrs.PutEmptyBytes("k2").FromRaw([]byte{1, 2, 3}) 37 | k8val := attrs.PutEmptyMap("k8") 38 | k8val.PutDouble("k4", 1) 39 | k8val.PutInt("k1", 2) 40 | k7val := attrs.PutEmptyMap("k7") 41 | k7val.PutDouble("k4", 1) 42 | k7val.PutInt("k1", 2) 43 | 44 | sig := DataPointSig[pmetric.NumberDataPoint](ndp, "k5") 45 | 46 | expected := []byte{1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 107, 49, 2, 0, 0, 0, 0, 0, 0, 0, 107, 50, 1, 2, 3, 107, 51, 0, 107, 52, 0, 0, 0, 0, 0, 0, 240, 63, 107, 55, 107, 49, 2, 0, 0, 0, 0, 0, 0, 0, 107, 52, 0, 0, 0, 0, 0, 0, 240, 63, 107, 56, 107, 49, 2, 0, 0, 0, 0, 0, 0, 0, 107, 52, 0, 0, 0, 0, 0, 0, 240, 63} 47 | 48 | require.Equal(t, expected, sig) 49 | } 50 | -------------------------------------------------------------------------------- /go/pkg/otel/metrics/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package metrics provides functions to convert OTLP metrics to OTLP Arrow metrics and vice versa. 16 | // This package also supports the conversion of uni-variate metrics into multi-variate metrics. 17 | package metrics 18 | -------------------------------------------------------------------------------- /go/pkg/otel/metrics/otlp/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package otlp contains types and functions used to convert OTLP Arrow metrics into their OTLP representation. 19 | package otlp 20 | -------------------------------------------------------------------------------- /go/pkg/otel/metrics/otlp/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package otlp 19 | 20 | import ( 21 | "errors" 22 | ) 23 | 24 | var ( 25 | ErrNotArrayInt32 = errors.New("not an arrow array.Int32") 26 | ErrNotArrayUint64 = errors.New("not an arrow array.Uint64") 27 | ErrNotArrayFloat64 = errors.New("not an arrow array.Float64") 28 | ErrNotArrayList = errors.New("not an arrow array.List") 29 | ) 30 | -------------------------------------------------------------------------------- /go/pkg/otel/observer/producer_observer.go: -------------------------------------------------------------------------------- 1 | package observer 2 | 3 | import ( 4 | "github.com/apache/arrow-go/v18/arrow" 5 | 6 | "github.com/open-telemetry/otel-arrow/pkg/record_message" 7 | ) 8 | 9 | // ProducerObserver is an interface for observing the OTel Arrow producer. 10 | type ProducerObserver interface { 11 | // OnNewField is called when a new field is added to the schema. 12 | OnNewField(recordName string, fieldPath string) 13 | 14 | // OnDictionaryUpgrade is called when a dictionary index is upgraded. 15 | OnDictionaryUpgrade(recordName string, fieldPath string, prevIndexType, newIndexType arrow.DataType, card, total uint64) 16 | 17 | // OnDictionaryOverflow is called when a dictionary index overflows, i.e. 18 | // the cardinality of the dictionary exceeds the maximum cardinality of the 19 | // index type. 20 | // The column type is no longer a dictionary and is downgraded to its value 21 | // type. 22 | OnDictionaryOverflow(recordName string, fieldPath string, card, total uint64) 23 | 24 | // OnSchemaUpdate is called when the schema is updated. 25 | OnSchemaUpdate(recordName string, old, new *arrow.Schema) 26 | 27 | // OnDictionaryReset is called when a dictionary is reset instead of being 28 | // overflowed. This happens when dictionary entries are reused in average 29 | // more than a specific threshold. 30 | OnDictionaryReset(recordName string, fieldPath string, indexType arrow.DataType, card, total uint64) 31 | 32 | // OnMetadataUpdate is called when schema metadata are updated. 33 | OnMetadataUpdate(recordName, metadataKey string) 34 | 35 | // OnRecord is called when a record is produced. 36 | OnRecord(arrow.Record, record_message.PayloadType) 37 | } 38 | -------------------------------------------------------------------------------- /go/pkg/otel/traces/arrow/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package arrow contains types and functions used to convert OTLP traces into their Arrow representation. 19 | package arrow 20 | -------------------------------------------------------------------------------- /go/pkg/otel/traces/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package traces provides functions to convert OTLP traces to OTLP Arrow traces and vice versa. 16 | package traces 17 | -------------------------------------------------------------------------------- /go/pkg/otel/traces/otlp/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package otlp contains types and functions used to convert OTLP Arrow traces into their OTLP representation. 19 | package otlp 20 | -------------------------------------------------------------------------------- /go/pkg/werror/error_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package werror 19 | 20 | import ( 21 | "errors" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestWError(t *testing.T) { 28 | t.Parallel() 29 | 30 | err := Level1a() 31 | require.Equal(t, "github.com/open-telemetry/otel-arrow/pkg/werror.Level1a:40->github.com/open-telemetry/otel-arrow/pkg/werror.Level2:48{id=1}->test error", err.Error()) 32 | 33 | err = Level1b() 34 | require.Equal(t, "github.com/open-telemetry/otel-arrow/pkg/werror.Level1b:44->github.com/open-telemetry/otel-arrow/pkg/werror.Level2:48{id=2}->test error", err.Error()) 35 | } 36 | 37 | var ErrTest = errors.New("test error") 38 | 39 | func Level1a() error { 40 | return Wrap(Level2(1)) 41 | } 42 | 43 | func Level1b() error { 44 | return Wrap(Level2(2)) 45 | } 46 | 47 | func Level2(id int) error { 48 | return WrapWithContext(ErrTest, map[string]interface{}{"id": id}) 49 | } 50 | -------------------------------------------------------------------------------- /go/tools/data_model_gen/data_model.tmpl: -------------------------------------------------------------------------------- 1 | # Arrow Data Model 2 | 3 | This document describes the Arrow Schema used for each OTLP entity as Entity Relation diagrams. 4 | 5 | The Arrow Data Model has been carefully designed to optimize: 6 | - The compression ratio for metrics, logs, and traces, 7 | - Its compatibility within the extensive Arrow ecosystem, 8 | - Its compatibility with file formats, such as Parquet. 9 | 10 | This document has been generated directly from the source code. To regenerate this document, run the following command: 11 | 12 | ```bash 13 | make doc 14 | ``` 15 | 16 | ## Metrics Arrow Records 17 | 18 | The following ER diagram describes the Arrow Schema used for metrics. 19 | 20 | {{.Metrics}} 21 | 22 | ## Logs Arrow Records 23 | 24 | The following ER diagram describes the Arrow Schema used for logs. 25 | 26 | {{.Logs}} 27 | 28 | ## Traces Arrow Records 29 | 30 | The following ER diagram describes the Arrow Schema used for traces. 31 | 32 | {{.Traces}} 33 | -------------------------------------------------------------------------------- /go/tools/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package tools contains tools used to benchmark OTLP, OTLP Arrow protocols, and to generate fake data. 19 | package tools 20 | -------------------------------------------------------------------------------- /go/tools/logs_benchmark/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package main contains a CLI tool for benchmarking the logs between OTLP and OTLP Arrow protocols. 19 | package main 20 | -------------------------------------------------------------------------------- /go/tools/logs_benchmark/test-logs.csv: -------------------------------------------------------------------------------- 1 | ts,level,source-host,log-method,log-url,log-status,body,log-duration,log-content-size,source-type,log-valid 2 | 1672942667866,info,156.122.44.5,GET,http://www.example.com/,200,OK,10.5,2000,http,true 3 | 1672942667866,info,156.122.44.5,GET,http://www.example.com/abc,404,Not Found,34.5,3000,http,true 4 | 1672942667866,info,156.122.44.5,POST,http://www.example.com/abc,503,Service Unavailable,7.6,4000,http,false -------------------------------------------------------------------------------- /go/tools/logs_gen/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package main contains a CLI tool for generating fake OTLP logs. 19 | package main 20 | -------------------------------------------------------------------------------- /go/tools/metrics_benchmark/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package main contains a CLI tool for benchmarking the metrics between OTLP and OTLP Arrow protocols. 19 | package main 20 | -------------------------------------------------------------------------------- /go/tools/metrics_gen/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package main contains a CLI tool for generating fake OTLP metrics. 19 | package main 20 | -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/compression_ratio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/compression_ratio.png -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/producer_stats.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/producer_stats.png -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/record_stats.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/record_stats.png -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/schema_stats_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/schema_stats_1.png -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/schema_stats_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/schema_stats_2.png -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/schema_updates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/schema_updates.png -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/span_event_record.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/span_event_record.png -------------------------------------------------------------------------------- /go/tools/trace_analyzer/imgs/test_sorting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/go/tools/trace_analyzer/imgs/test_sorting.png -------------------------------------------------------------------------------- /go/tools/trace_benchmark/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package main contains a CLI tool for benchmarking the traces between OTLP and OTLP Arrow protocols. 19 | package main 20 | -------------------------------------------------------------------------------- /go/tools/trace_gen/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package main contains a CLI tool for generating fake OTLP traces. 19 | package main 20 | -------------------------------------------------------------------------------- /go/tools/trace_head/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The OpenTelemetry Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | // Package main contains a CLI tool used to extract the first n spans from a trace file (protobuf file). 19 | package main 20 | -------------------------------------------------------------------------------- /nextest.toml: -------------------------------------------------------------------------------- 1 | [profile.default] 2 | test-binary-threads = 4 3 | 4 | [profile.default.reporters] 5 | pretty = true 6 | junit = true 7 | 8 | [profile.default.output] 9 | format = "pretty" 10 | 11 | [profile.default.test-runner] 12 | no-fail-fast = true 13 | 14 | [profile.default.test-discovery] 15 | threads = 4 16 | -------------------------------------------------------------------------------- /proto/generate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Run this in the top-level directory. 4 | rm -rf api 5 | mkdir api 6 | 7 | # Get current directory. 8 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 9 | 10 | for dir in $(find ${DIR}/opentelemetry -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq); do 11 | files=$(find "${dir}" -name '*.proto') 12 | 13 | # Generate all files with protoc-gen-go. 14 | echo ${files} 15 | protoc -I ${DIR} --go_out=api --go-grpc_out=api ${files} 16 | done 17 | 18 | mv api/github.com/open-telemetry/otel-arrow/api/experimental api 19 | rm -rf api/github.com 20 | 21 | # Generate the mock files 22 | go install go.uber.org/mock/mockgen@latest 23 | 24 | mkdir -p api/experimental/arrow/v1/mock 25 | mockgen -package mock github.com/open-telemetry/otel-arrow/api/experimental/arrow/v1 ArrowTracesServiceClient,ArrowTracesService_ArrowTracesClient,ArrowTracesServiceServer,ArrowTracesService_ArrowTracesServer,ArrowLogsServiceClient,ArrowLogsService_ArrowLogsClient,ArrowLogsServiceServer,ArrowLogsService_ArrowLogsServer,ArrowMetricsServiceClient,ArrowMetricsService_ArrowMetricsClient,ArrowMetricsServiceServer,ArrowMetricsService_ArrowMetricsServer > api/experimental/arrow/v1/mock/arrow_service_mock.go 26 | go mod tidy 27 | -------------------------------------------------------------------------------- /rust/README.md: -------------------------------------------------------------------------------- 1 | # Rust components 2 | 3 | This folder contains various Rust projects of varying stages of maturity. 4 | 5 | The `otap-dataflow/` folder contains the main deliverable of Phase 2 of the 6 | otel-arrow project, [as mentioned in its README](./otap-dataflow/README.md). 7 | 8 | All other folders are either experimental or initial donations of components 9 | that have yet to be incorporated into the main library. 10 | 11 | | Folder | Type | 12 | |-----------------|------------------------------| 13 | | beaubourg | :handshake: Contributed Code | 14 | | experimental | :mag: Prototype | 15 | | otap-dataflow | :hammer: Core Component | 16 | | otel-arrow-rust | :handshake: Contributed Code | 17 | -------------------------------------------------------------------------------- /rust/beaubourg/.config/config.toml: -------------------------------------------------------------------------------- 1 | [profile.ci] 2 | failure-output = "immediate-final" 3 | # Do not cancel the test run on the first failure. 4 | fail-fast = false 5 | 6 | [profile.ci.junit] 7 | path = "junit.xml" 8 | 9 | [build] 10 | rustflags = ["-W", "missing_docs"] 11 | 12 | [target.'cfg(all())'] 13 | rustflags = [ 14 | "-Dunsafe_code", 15 | "-Wmissing_docs", 16 | "-Wclippy::all", 17 | "-Wclippy::await_holding_lock", 18 | "-Wclippy::char_lit_as_u8", 19 | ] 20 | 21 | [target.'cfg(feature = "cargo-clippy")'] 22 | rustflags = [ 23 | "-Wmissing_docs", 24 | ] -------------------------------------------------------------------------------- /rust/beaubourg/.githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | if ! cargo +nightly fmt --all -- --check 6 | then 7 | echo "There are some code style issues." 8 | exit 1 9 | fi 10 | 11 | if ! cargo clippy --all-targets -- -D warnings 12 | then 13 | echo "There are some clippy issues." 14 | exit 1 15 | fi 16 | 17 | if ! cargo test --all 18 | then 19 | echo "There are some test issues." 20 | exit 1 21 | fi 22 | 23 | if ! cargo deny check 24 | then 25 | echo "There are some cargo deny issues." 26 | exit 1 27 | fi 28 | 29 | exit 0 -------------------------------------------------------------------------------- /rust/beaubourg/.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | crates/*/target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | -------------------------------------------------------------------------------- /rust/beaubourg/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /rust/beaubourg/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /rust/beaubourg/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /rust/beaubourg/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## V0.3.0 - Feb 21, 2025 4 | 5 | - Prepare Beaubourg for contribution to OpenTelemetry 6 | 7 | ## V0.2.0 - Feb 2022 8 | 9 | - Reorganize the codebase in a mutli-crate structure 10 | - Single threaded and multi-threaded engines 11 | - Tests 12 | 13 | ## V0.1.0 - Jan 2022 14 | 15 | - Pipeline and Engine 16 | - AsyncReceiver, AsyncProcessor, AsyncExporter 17 | -------------------------------------------------------------------------------- /rust/beaubourg/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /rust/beaubourg/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "crates/*", 4 | ] 5 | 6 | 7 | [package] 8 | name = "beaubourg" 9 | version = "0.3.0" 10 | description = "Lightweight Rust library for building generic pipelines consisting of a collection of receivers, processors, and exporters." 11 | authors = ["Laurent Querel "] 12 | edition = "2021" 13 | license = "Apache-2.0" 14 | exclude = [ 15 | "/.github", 16 | "/.githooks", 17 | "/.config", 18 | ] 19 | 20 | 21 | [dependencies] 22 | receiver = { path = "crates/receiver" } 23 | exporter = { path = "crates/exporter" } 24 | processor = { path = "crates/processor" } 25 | task = { path = "crates/task" } 26 | config = { path = "crates/config" } 27 | engine = { path = "crates/engine" } 28 | signal = { path = "crates/signal" } 29 | context = { path = "crates/context" } 30 | tracing = "0.1.41" 31 | 32 | 33 | [dev-dependencies] 34 | tokio = { version = "1.44.0", features = ["rt-multi-thread", "macros", "io-util"] } 35 | futures = "0.3.31" 36 | async-trait = "0.1.87" 37 | serde = { version = "1.0.218", features = ["derive"] } 38 | serde_yaml = "0.8.26" 39 | color-eyre = "0.6.2" 40 | tracing = "0.1.41" 41 | tracing-subscriber = "0.3.18" 42 | mimalloc-rust = "0.1.5" 43 | socket2 = { version="0.5.8", features = ["all"]} 44 | tokio-stream = { version = "0.1.17", features = ["net"] } 45 | once_cell = "1.19.0" 46 | num_cpus = "1.16.0" 47 | 48 | [[example]] 49 | name = "multithread_engine_example" 50 | path = "examples/multithread_engine_example.rs" 51 | 52 | [[example]] 53 | name = "thread_per_core_engine_example" 54 | path = "examples/thread_per_core_engine_example.rs" 55 | 56 | [profile.release] 57 | debug = 1 58 | incremental = true # disable incremental when sscache is involved via CARGO_INCREMENTAL=0 59 | lto = "thin" 60 | 61 | #lto = "fat" 62 | #codegen-units = 1 63 | #opt-level = 3 64 | #debug = false 65 | # debug = true 66 | 67 | # To compile with the production profile `cargo --profile production ...` 68 | #[profile.production] 69 | #inherits = "release" 70 | #lto = true 71 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/config/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "config" 3 | version = "0.3.0" 4 | authors = ["Laurent Querel "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | 8 | [dependencies] 9 | receiver = { path = "../receiver" } 10 | exporter = { path = "../exporter" } 11 | processor = { path = "../processor" } 12 | signal = { path = "../signal" } 13 | 14 | serde = { version = "1.0.218", features = ["derive"] } 15 | serde_yaml = "0.8.26" 16 | thiserror = "2.0.12" 17 | tracing = "0.1.41" 18 | validator = { version = "0.20.0", features = ["derive"] } 19 | 20 | [dev-dependencies] 21 | async-trait = "0.1.87" 22 | tokio = { version = "1.44.0", features = ["rt-multi-thread"] } 23 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/config/data/config.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | test: 3 | test/1: 4 | test/2: 5 | processors: 6 | noop: 7 | exporters: 8 | test: 9 | test/1: 10 | test/2: 11 | extensions: {} 12 | service: 13 | extensions: [] 14 | pipelines: 15 | test: 16 | receivers: 17 | - test 18 | - test/1 19 | - test/2 20 | processors: 21 | - noop 22 | exporters: 23 | - test 24 | - test/1 25 | - test/2 26 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/context/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "context" 3 | version = "0.3.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/engine/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "engine" 3 | version = "0.3.0" 4 | authors = ["Laurent Querel "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | 8 | [dependencies] 9 | config = { path = "../config" } 10 | receiver = { path = "../receiver" } 11 | processor = { path = "../processor" } 12 | exporter = { path = "../exporter" } 13 | task = { path = "../task" } 14 | signal = { path = "../signal" } 15 | context = { path = "../context" } 16 | 17 | thiserror = "2.0.12" 18 | tracing = "0.1.41" 19 | num_cpus = "1.16.0" 20 | tokio = { version = "1.44.0", features = ["rt-multi-thread"] } 21 | async-trait = "0.1.87" 22 | futures = "0.3.31" 23 | flume = "0.11.1" 24 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/exporter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "exporter" 3 | version = "0.3.0" 4 | authors = ["Laurent Querel "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | 8 | [dependencies] 9 | signal = { path = "../signal" } 10 | task = { path = "../task" } 11 | context = { path = "../context" } 12 | 13 | serde = { version = "1.0.218", features = ["derive"] } 14 | serde_yaml = "0.8.26" 15 | thiserror = "2.0.12" 16 | async-trait = "0.1.87" 17 | tokio = { version = "1.44.0", features = ["rt-multi-thread", "macros"] } 18 | futures = "0.3.31" 19 | tracing = "0.1.41" 20 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/processor/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "processor" 3 | version = "0.3.0" 4 | authors = ["Laurent Querel "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | 8 | [dependencies] 9 | task = { path = "../task" } 10 | signal = { path = "../signal" } 11 | context = { path = "../context" } 12 | 13 | serde = { version = "1.0.218", features = ["derive"] } 14 | serde_yaml = "0.8.26" 15 | thiserror = "2.0.12" 16 | tracing = "0.1.41" 17 | async-trait = "0.1.87" 18 | tokio = { version = "1.44.0", features = ["rt-multi-thread"] } 19 | flume = "0.11.1" 20 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/processor/data/test_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "timestamps":[0,1,2], 3 | "attributes": { 4 | "attribute_2": { 5 | "string": { 6 | "required": false, 7 | "values": [ 8 | "val1", 9 | "val2", 10 | null 11 | ] 12 | } 13 | }, 14 | "attribute_1": { 15 | "string": { 16 | "required": true, 17 | "values": [ 18 | "val1", 19 | "val2", 20 | "val3" 21 | ] 22 | } 23 | } 24 | }, 25 | "metrics": { 26 | "metric_1": { 27 | "int64": { 28 | "required": true, 29 | "values": [ 30 | 1, 31 | 2, 32 | 3 33 | ] 34 | } 35 | }, 36 | "metric_2": { 37 | "double": { 38 | "required": false, 39 | "values": [ 40 | 1.0, 41 | 2.0, 42 | null 43 | ] 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/processor/src/noop.rs: -------------------------------------------------------------------------------- 1 | //! Definition of the Noop processor. 2 | 3 | use async_trait::async_trait; 4 | use signal::Signal; 5 | 6 | use crate::{AsyncProcessor, EffectHandler, Error}; 7 | 8 | /// A processor that does nothing. 9 | pub struct NoOp { 10 | name: String, 11 | } 12 | 13 | impl NoOp { 14 | /// Creates a new NoOp processor. 15 | pub fn new(name: String) -> Self { 16 | Self { name } 17 | } 18 | } 19 | 20 | /// The NoOp processor implementation. 21 | #[async_trait] 22 | impl AsyncProcessor for NoOp 23 | where 24 | Msg: 'static + Clone + Send, 25 | { 26 | /// All received messages are sent to the next processor or to all the 27 | /// exporters. 28 | async fn process(&mut self, signal: Signal, effects_handler: &mut EffectHandler) -> Result<(), Error> { 29 | match signal { 30 | Signal::TimerTick { .. } => Ok(()), 31 | Signal::Messages { messages } => { 32 | tracing::trace!("noop '{}' is forwarding {} messages", self.name, messages.len()); 33 | effects_handler.emit_messages(messages); 34 | Ok(()) 35 | } 36 | Signal::Stop => Ok(()), 37 | _ => Err(Error::UnsupportedEvent { 38 | processor: self.name.clone(), 39 | signal: signal.to_string(), 40 | }), 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/receiver/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "receiver" 3 | version = "0.3.0" 4 | authors = ["Laurent Querel "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | 8 | [dependencies] 9 | task = { path = "../task" } 10 | context = { path = "../context" } 11 | 12 | serde = { version = "1.0.218", features = ["derive"] } 13 | serde_yaml = "0.8.26" 14 | thiserror = "2.0.12" 15 | tokio = { version = "1.44.0", features = ["rt-multi-thread", "macros", "net"] } 16 | async-trait = "0.1.87" 17 | tracing = "0.1.41" 18 | futures = "0.3.31" 19 | tokio-stream = "0.1.17" 20 | socket2 = { version="0.5.8", features = ["all"]} 21 | maplit = "1.0.2" 22 | flume = "0.11.1" 23 | 24 | [build-dependencies] 25 | tonic-build = "0.12.3" 26 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/signal/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "signal" 3 | version = "0.3.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | tokio = { version = "1.44.0", features = ["rt-multi-thread", "macros"] } 11 | futures = "0.3.31" 12 | flume = "0.11.1" 13 | tracing = "0.1.41" 14 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/task/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "task" 3 | version = "0.3.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | [dependencies] 8 | thiserror = "2.0.12" 9 | tracing = "0.1.41" 10 | futures = "0.3.31" 11 | tokio = { version = "1.44.0", features = ["rt-multi-thread"] } 12 | tokio-stream = "0.1.17" 13 | async-trait = "0.1.87" 14 | -------------------------------------------------------------------------------- /rust/beaubourg/crates/task/src/labels.rs: -------------------------------------------------------------------------------- 1 | //! List of labels for a process or a task. 2 | use std::collections::HashMap; 3 | 4 | /// List of labels for a process. 5 | #[derive(Debug, Clone, PartialEq, Eq)] 6 | pub struct ProcessLabels { 7 | /// The process ID. 8 | pub process_id: String, 9 | } 10 | 11 | /// List of labels for a task. 12 | #[derive(Debug, Clone, PartialEq, Eq)] 13 | pub struct TaskLabels { 14 | /// The task category. 15 | pub task_cat: String, 16 | /// The task id. 17 | pub task_id: String, 18 | /// The task source id. 19 | pub task_source: String, 20 | } 21 | 22 | impl Default for ProcessLabels { 23 | fn default() -> Self { 24 | Self { 25 | process_id: "undefined".to_string(), 26 | } 27 | } 28 | } 29 | 30 | impl ProcessLabels { 31 | /// Create a new `ProcessLabels` instance. 32 | pub fn new(process_id: &str) -> Self { 33 | Self { 34 | process_id: process_id.into(), 35 | } 36 | } 37 | } 38 | 39 | impl TaskLabels { 40 | /// Create a new `TaskLabels` instance. 41 | pub fn new(task_cat: &str, task_id: &str, task_source: &str) -> Self { 42 | Self { 43 | task_cat: task_cat.into(), 44 | task_id: task_id.into(), 45 | task_source: task_source.into(), 46 | } 47 | } 48 | 49 | /// Create a unique task id. 50 | pub fn unique_id(&self) -> String { 51 | format!("{}:{}:{}", self.task_cat, self.task_id, self.task_source) 52 | } 53 | 54 | /// Convert this TaskLabels to a HashMap. 55 | pub fn as_map(&self) -> HashMap { 56 | let mut map = HashMap::new(); 57 | let _ = map.insert("task_cat".into(), self.task_cat.clone()); 58 | let _ = map.insert("task_id".into(), self.task_id.clone()); 59 | let _ = map.insert("task_source".into(), self.task_source.clone()); 60 | 61 | map 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /rust/beaubourg/examples/common/mod.rs: -------------------------------------------------------------------------------- 1 | /// Just a basic message. 2 | /// 3 | /// Note: a message must be at the minimum 'static + Clone + Send. 4 | #[derive(Clone, Debug)] 5 | pub struct Message { 6 | pub origin: String, 7 | #[allow(dead_code)] 8 | pub payload: usize, 9 | } 10 | -------------------------------------------------------------------------------- /rust/beaubourg/examples/multithread_config.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | test: 3 | test/1: 4 | test/2: 5 | processors: 6 | noop: 7 | exporters: 8 | test: 9 | concurrency_model: singleton 10 | test/1: 11 | concurrency_model: 12 | task_per_core: 2 13 | test/2: 14 | concurrency_model: 15 | task_per_core: 2 16 | extensions: {} 17 | service: 18 | extensions: [] 19 | pipelines: 20 | test: 21 | receivers: 22 | - test 23 | - test/1 24 | - test/2 25 | processors: 26 | - noop 27 | exporters: 28 | - test 29 | - test/1 30 | - test/2 31 | -------------------------------------------------------------------------------- /rust/beaubourg/examples/multithread_engine_example.rs: -------------------------------------------------------------------------------- 1 | use beaubourg::{ 2 | engine::{multi_threaded, Engine}, 3 | task::labels::ProcessLabels, 4 | }; 5 | use color_eyre::eyre::Result; 6 | use mimalloc_rust::GlobalMiMalloc; 7 | use tracing::Level; 8 | use tracing_subscriber::FmtSubscriber; 9 | 10 | use crate::{exporter::TestExporterFactory, processor::TestProcessorFactory, receiver::TestReceiverFactory}; 11 | 12 | mod common; 13 | mod exporter; 14 | mod processor; 15 | mod receiver; 16 | 17 | // Recommended global allocator to get the best performance of the pipeline 18 | // engine. 19 | #[global_allocator] 20 | static GLOBAL_MIMALLOC: GlobalMiMalloc = GlobalMiMalloc; 21 | 22 | fn main() -> Result<()> { 23 | init()?; 24 | 25 | let mut engine = multi_threaded::Engine::new( 26 | TestReceiverFactory::default(), 27 | TestProcessorFactory::default(), 28 | TestExporterFactory::default(), 29 | ); 30 | engine.run(ProcessLabels::new("test"), "examples/multithread_config.yaml")?; 31 | 32 | Ok(()) 33 | } 34 | 35 | /// Initializes the collector 36 | fn init() -> Result<()> { 37 | color_eyre::install()?; 38 | 39 | let subscriber = FmtSubscriber::builder().with_max_level(Level::INFO).finish(); 40 | 41 | tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /rust/beaubourg/examples/receiver/mod.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use beaubourg::receiver::{AsyncReceiver, ReceiverFactory}; 3 | use receiver::signal::SignalReceiver; 4 | use serde_yaml::Value; 5 | 6 | use crate::common::Message; 7 | 8 | pub struct TestReceiver { 9 | name: String, 10 | } 11 | 12 | #[async_trait] 13 | impl AsyncReceiver for TestReceiver { 14 | async fn receive( 15 | &mut self, 16 | _signal_receiver: SignalReceiver, 17 | effect_handler: beaubourg::receiver::effect::EffectHandler, 18 | ) -> Result<(), beaubourg::receiver::Error> { 19 | for i in 0..10 { 20 | effect_handler 21 | .send_messages(vec![Message { 22 | origin: self.name.clone(), 23 | payload: i, 24 | }]) 25 | .await 26 | .map_err(|e| beaubourg::receiver::Error::Receiver { 27 | receiver: self.name.clone(), 28 | error: e.to_string(), 29 | context: Default::default(), 30 | })?; 31 | } 32 | 33 | Ok(()) 34 | } 35 | } 36 | 37 | #[derive(Default)] 38 | pub struct TestReceiverFactory {} 39 | 40 | impl ReceiverFactory for TestReceiverFactory { 41 | fn create( 42 | &self, 43 | receiver_name: &str, 44 | receiver_type: &str, 45 | _config: Value, 46 | ) -> Result + Send + Sync>, beaubourg::receiver::Error> { 47 | match receiver_type { 48 | "test" => { 49 | let receiver = Box::new(TestReceiver { 50 | name: receiver_name.into(), 51 | }); 52 | 53 | Ok(receiver as Box + Send + Sync>) 54 | } 55 | _ => Err(beaubourg::receiver::Error::UnknownReceiver { 56 | receiver: receiver_name.into(), 57 | receiver_type: receiver_type.into(), 58 | }), 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rust/beaubourg/examples/thread_per_core_config.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | test: 3 | processors: 4 | noop: 5 | exporters: 6 | test: 7 | extensions: {} 8 | service: 9 | extensions: [] 10 | pipelines: 11 | test: 12 | receivers: 13 | - test 14 | processors: 15 | - noop 16 | exporters: 17 | - test 18 | -------------------------------------------------------------------------------- /rust/beaubourg/examples/thread_per_core_engine_example.rs: -------------------------------------------------------------------------------- 1 | use beaubourg::{engine::Engine, task::labels::ProcessLabels}; 2 | use color_eyre::eyre::Result; 3 | use engine::thread_per_core; 4 | use mimalloc_rust::GlobalMiMalloc; 5 | use tracing::Level; 6 | use tracing_subscriber::FmtSubscriber; 7 | 8 | use crate::{exporter::TestExporterFactory, processor::TestProcessorFactory, receiver::TestReceiverFactory}; 9 | 10 | mod common; 11 | mod exporter; 12 | mod processor; 13 | mod receiver; 14 | 15 | // Recommended global allocator to get the best performance of the pipeline 16 | // engine. 17 | #[global_allocator] 18 | static GLOBAL_MIMALLOC: GlobalMiMalloc = GlobalMiMalloc; 19 | 20 | fn main() -> Result<()> { 21 | init()?; 22 | 23 | let mut engine = thread_per_core::Engine::new( 24 | TestReceiverFactory::default(), 25 | TestProcessorFactory::default(), 26 | TestExporterFactory::default(), 27 | ); 28 | engine.run(ProcessLabels::new("test"), "examples/thread_per_core_config.yaml")?; 29 | 30 | Ok(()) 31 | } 32 | 33 | /// Initializes the collector 34 | fn init() -> Result<()> { 35 | color_eyre::install()?; 36 | 37 | let subscriber = FmtSubscriber::builder().with_max_level(Level::INFO).finish(); 38 | 39 | tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); 40 | 41 | Ok(()) 42 | } 43 | -------------------------------------------------------------------------------- /rust/beaubourg/images/Beaubourg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/rust/beaubourg/images/Beaubourg.png -------------------------------------------------------------------------------- /rust/beaubourg/images/BeaubourgExporter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/rust/beaubourg/images/BeaubourgExporter.png -------------------------------------------------------------------------------- /rust/beaubourg/images/BeaubourgProcessor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/rust/beaubourg/images/BeaubourgProcessor.png -------------------------------------------------------------------------------- /rust/beaubourg/images/BeaubourgReceiver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/rust/beaubourg/images/BeaubourgReceiver.png -------------------------------------------------------------------------------- /rust/beaubourg/images/beaubourg_building.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-telemetry/otel-arrow/227584536036072e9402ef770f8bb8c75b69d1bb/rust/beaubourg/images/beaubourg_building.png -------------------------------------------------------------------------------- /rust/beaubourg/rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | max_width = 120 3 | imports_granularity = "Crate" 4 | reorder_impl_items = true 5 | group_imports = "StdExternalCrate" 6 | wrap_comments = true 7 | report_todo = "Always" 8 | report_fixme = "Always" -------------------------------------------------------------------------------- /rust/beaubourg/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![doc = include_str!("../README.md")] 2 | #![deny( 3 | trivial_numeric_casts, 4 | missing_docs, 5 | unsafe_code, 6 | unstable_features, 7 | unused_import_braces, 8 | unused_qualifications, 9 | unused_extern_crates, 10 | unused_results, 11 | variant_size_differences 12 | )] 13 | #![warn(rust_2021_compatibility, unreachable_pub)] 14 | 15 | //! Beaubourg is a library to build **pipelines** combining 3 types of 16 | //! components: **receiver**, **processor**, and **exporter**. The inputs of the 17 | //! pipeline are represented by one or more receivers connected to a chain of 18 | //! processors which are themselves connected to one or more exporters 19 | //! representing the outputs of the pipeline. 20 | //! 21 | //! A pipeline must have at least one receiver and one exporter, processors are 22 | //! optional. The receivers are executed in parallel and are not connected to 23 | //! each other. The processors form a chain, where the first processor consumes 24 | //! the data of the receivers, transform the data and emit the data for the next 25 | //! processor or for the exporters when the processor is the last of the 26 | //! processor chain. The exporters are executed in parallel and are not 27 | //! connected to each other. Optionally, a processor can send its result to a 28 | //! specific receiver. 29 | //! 30 | //! An **engine** is an execution environment for one or more pipelines running 31 | //! in parallel. 32 | //! 33 | //! A configuration file is used to define the receivers, processors, exporters, 34 | //! and their connections. A more programmatic way is possible. 35 | 36 | pub use config; 37 | pub use context; 38 | pub use engine; 39 | pub use exporter; 40 | pub use processor; 41 | pub use receiver; 42 | pub use signal; 43 | pub use task; 44 | -------------------------------------------------------------------------------- /rust/beaubourg/tests/config.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | test: 3 | concurrency: 2 4 | processors: 5 | noop: 6 | concurrency: 2 7 | exporters: 8 | test/0: 9 | concurrency_model: singleton 10 | test/1: 11 | concurrency_model: 12 | task_per_core: 2 13 | extensions: {} 14 | service: 15 | extensions: [] 16 | pipelines: 17 | test: 18 | receivers: 19 | - test 20 | processors: 21 | - noop 22 | exporters: 23 | - test/0 24 | - test/1 25 | -------------------------------------------------------------------------------- /rust/experimental/README.md: -------------------------------------------------------------------------------- 1 | # Experimental 2 | 3 | The `experimental` directory contains isolated Rust components that are not 4 | fully supported but are related to project goals mentioned in [OTel-Arrow 5 | Project Phases](../../docs/project-phases.md). 6 | -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "intermediate_language", 4 | "kql_plugin", 5 | "ottl_plugin", 6 | "cli" 7 | ] 8 | resolver = "2" -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cli" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | kql_plugin = { version = "0.1.0", path = "../kql_plugin" } 8 | ottl_plugin = { version = "0.1.0", path = "../ottl_plugin" } 9 | intermediate_language = { version = "0.1.0", path = "../intermediate_language" } -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/cli/src/main.rs: -------------------------------------------------------------------------------- 1 | use intermediate_language::query_processor::QueryProcessor; 2 | use kql_plugin::kql_plugin::KqlPlugin; 3 | use ottl_plugin::ottl_plugin::OttlPlugin; 4 | use std::env; 5 | 6 | fn main() { 7 | let args: Vec = env::args().collect(); 8 | if args.len() < 3 { 9 | eprintln!("Usage: {} ", args[0]); 10 | eprintln!("Parser options: kql, ottl"); 11 | std::process::exit(1); 12 | } 13 | 14 | let plugin = &args[1]; 15 | let input = &args[2]; 16 | 17 | match plugin.as_str() { 18 | "kql" => { 19 | let result = KqlPlugin::process_query(input); 20 | match result { 21 | Ok(query) => { 22 | println!("Parsed KQL query:\n{:?}", query); 23 | } 24 | Err(e) => { 25 | eprintln!("Error parsing KQL query:\n{}", e); 26 | std::process::exit(1); 27 | } 28 | } 29 | } 30 | "ottl" => { 31 | let result = OttlPlugin::process_query(input); 32 | match result { 33 | Ok(query) => { 34 | println!("Parsed OTTL query: {:?}", query); 35 | } 36 | Err(e) => { 37 | eprintln!("Error parsing OTTL query: {}", e); 38 | std::process::exit(1); 39 | } 40 | } 41 | } 42 | _ => { 43 | eprintln!("Invalid plugin option. Use 'kql' or 'ottl'."); 44 | std::process::exit(1); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/intermediate_language/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intermediate_language" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/intermediate_language/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod grammar_objects; 2 | pub mod query_processor; 3 | -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/intermediate_language/src/query_processor.rs: -------------------------------------------------------------------------------- 1 | use crate::grammar_objects::*; 2 | use std::error::Error; 3 | use std::fmt; 4 | 5 | #[derive(Debug)] 6 | pub enum QueryError { 7 | ParseError(String), 8 | ProcessingError(String), 9 | } 10 | 11 | impl fmt::Display for QueryError { 12 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 13 | match self { 14 | QueryError::ParseError(msg) => write!(f, "Parse Error: {}", msg), 15 | QueryError::ProcessingError(msg) => write!(f, "Processing Error: {}", msg), 16 | } 17 | } 18 | } 19 | 20 | impl Error for QueryError {} 21 | 22 | pub type QueryResult = Result; 23 | 24 | pub trait QueryProcessor { 25 | fn process_query(input: &str) -> Result; 26 | } 27 | -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/kql_plugin/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "kql_plugin" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | intermediate_language = { verison = "0.1.0", path = "../intermediate_language" } 8 | pest = "2.8" 9 | pest_derive = "2.8" -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/kql_plugin/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod kql_parser; 2 | pub mod kql_plugin; 3 | -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/ottl_plugin/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ottl_plugin" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | intermediate_language = { verison = "0.1.0", path = "../intermediate_language" } 8 | pest = "2.8" 9 | pest_derive = "2.8" -------------------------------------------------------------------------------- /rust/experimental/query_abstraction/ottl_plugin/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod ottl_parser; 2 | pub mod ottl_plugin; 3 | -------------------------------------------------------------------------------- /rust/experimental/syslog-cef-receivers/.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | -------------------------------------------------------------------------------- /rust/experimental/syslog-cef-receivers/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["syslog-receiver", "cef-receiver"] 3 | -------------------------------------------------------------------------------- /rust/experimental/syslog-cef-receivers/cef-receiver/.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | -------------------------------------------------------------------------------- /rust/experimental/syslog-cef-receivers/cef-receiver/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cef-receiver" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | -------------------------------------------------------------------------------- /rust/experimental/syslog-cef-receivers/cef-receiver/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | #[test] 4 | fn cef_receiver_dummy_test() { 5 | assert!(true); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /rust/experimental/syslog-cef-receivers/syslog-receiver/.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | -------------------------------------------------------------------------------- /rust/experimental/syslog-cef-receivers/syslog-receiver/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syslog-receiver" 3 | version = "0.1.0" 4 | edition = "2024" 5 | 6 | [dependencies] 7 | chrono = "0.4.40" 8 | syslog_loose = "0.21.0" 9 | -------------------------------------------------------------------------------- /rust/otap-dataflow/.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [alias] 2 | xtask = "run --package xtask --bin xtask --" 3 | 4 | [target.aarch64-unknown-linux-musl] 5 | linker = "aarch64-linux-gnu-gcc" 6 | rustflags = ["-C", "target-cpu=native"] 7 | 8 | [target.x86_64-unknown-linux-musl] 9 | linker = "x86_64-linux-gnu-gcc" 10 | rustflags = ["-C", "target-cpu=native"] -------------------------------------------------------------------------------- /rust/otap-dataflow/.clippy.toml: -------------------------------------------------------------------------------- 1 | msrv = "1.86.0" 2 | warn-on-all-wildcard-imports = true 3 | allow-expect-in-tests = true 4 | allow-unwrap-in-tests = true 5 | allow-dbg-in-tests = true 6 | allow-print-in-tests = true 7 | 8 | # Disallow specific methods 9 | disallowed-methods = [] 10 | 11 | # Disallow specific types 12 | disallowed-types = [ 13 | { path = "once_cell::sync::Lazy", reason = "Please use `std::sync::LazyLock` instead." }, 14 | ] 15 | 16 | # Disallow specific macros 17 | disallowed-macros = [ 18 | { path = "lazy_static::lazy_static", reason = "Please use `std::sync::LazyLock` instead." }, 19 | ] 20 | 21 | doc-valid-idents = [ 22 | "OTEL", 23 | "..", # add the default list 24 | ] 25 | -------------------------------------------------------------------------------- /rust/otap-dataflow/.config/nextest.toml: -------------------------------------------------------------------------------- 1 | [profile.default] 2 | slow-timeout = { period = "60s", terminate-after = 2 } -------------------------------------------------------------------------------- /rust/otap-dataflow/.cursor/rules/single-threaded-async-runtime.mdc: -------------------------------------------------------------------------------- 1 | --- 2 | description: 3 | globs: 4 | alwaysApply: true 5 | --- 6 | # Design Principles 7 | 8 | - Target a single-threaded async runtime 9 | - Declare async traits as `?Send`, providing `!Send` implementations and futures whenever practical 10 | - Avoid synchronization primitives as much as possible 11 | - Optimize for performance 12 | - Avoid unbounded channels and data structures 13 | - Minimize dependencies -------------------------------------------------------------------------------- /rust/otap-dataflow/.gitignore: -------------------------------------------------------------------------------- 1 | # Rust specific 2 | /target 3 | **/*.rs.bk 4 | Cargo.lock 5 | bench-log 6 | 7 | # Just 8 | just.zsh 9 | 10 | # IntelliJ IDEs 11 | /.idea 12 | *.iml 13 | 14 | # VS Code 15 | .vscode/ 16 | .devcontainer/ 17 | 18 | # Visual Studio 19 | .vs/ 20 | 21 | # Emacs 22 | *~ 23 | \#*\# 24 | 25 | # Miscellaneous files 26 | *.sw[op] 27 | *.DS_Store 28 | 29 | # test generated code 30 | **/observed_output/* 31 | 32 | # Ignore output files generated by weaver 33 | **/output/* 34 | 35 | # Coverage results 36 | lcov.info 37 | 38 | # Ignore the output of the `weaver resolve` command 39 | output.json 40 | resolved-schema.yaml 41 | 42 | # Ignore the temporary repo dir created by `cargo xtask history` command 43 | history-temp-repo/ -------------------------------------------------------------------------------- /rust/otap-dataflow/.typos.toml: -------------------------------------------------------------------------------- 1 | [files] 2 | extend-exclude = ["*.cast", "**/*.yaml", "**/*.svg", "**/allowed-external-types.toml"] 3 | 4 | [default.extend-words] -------------------------------------------------------------------------------- /rust/otap-dataflow/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | -------------------------------------------------------------------------------- /rust/otap-dataflow/benchmarks/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "benchmarks" 3 | version.workspace = true 4 | authors.workspace = true 5 | edition.workspace = true 6 | repository.workspace = true 7 | license.workspace = true 8 | publish.workspace = true 9 | rust-version.workspace = true 10 | 11 | [dependencies] 12 | tokio.workspace = true 13 | serde_json.workspace = true 14 | 15 | [dev-dependencies] 16 | criterion = { workspace = true, features = ["html_reports", "async_tokio"] } 17 | mimalloc = "0.1.46" 18 | otap-df-config = { path = "../crates/config" } 19 | otap-df-channel = { path = "../crates/channel" } 20 | futures-channel = "0.3" 21 | futures = "0.3.31" 22 | flume = "0.11.1" 23 | core_affinity = "0.8.3" 24 | local-sync = "0.1.1" 25 | async-unsync = "0.3.0" 26 | unsync = "0.1.2" 27 | 28 | [lints] 29 | workspace = true 30 | 31 | [[bench]] 32 | name = "config" 33 | harness = false 34 | 35 | [[bench]] 36 | name = "channel" 37 | harness = false 38 | 39 | [profile.bench] 40 | opt-level = 3 41 | debug = false 42 | incremental = false 43 | lto = "fat" 44 | codegen-units = 1 45 | panic = "abort" 46 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/README.md: -------------------------------------------------------------------------------- 1 | # Crates 2 | 3 | ![Dependencies](/docs/images/dependencies.svg) 4 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/channel/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "otap-df-channel" 3 | description = "Async channel implementations optimised for single-threaded async runtime" 4 | version.workspace = true 5 | authors.workspace = true 6 | repository.workspace = true 7 | license.workspace = true 8 | publish.workspace = true 9 | edition.workspace = true 10 | rust-version.workspace = true 11 | 12 | [lints] 13 | workspace = true 14 | 15 | [dependencies] 16 | thiserror = { workspace = true } 17 | tokio = { workspace = true } 18 | arrayvec = "0.7.6" -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/channel/README.md: -------------------------------------------------------------------------------- 1 | # Channel implementations optimized for single-threaded async runtime 2 | 3 | This crate contains various implementations of optimized asynchronous channels 4 | for: 5 | 6 | - A single-threaded async runtime 7 | - Detailed instrumentation (not yet implemented) 8 | - Maximum control within the context of this project 9 | 10 | Current implementations include: 11 | 12 | - MPMC: A multi-producer multi-consumer channel 13 | - MPSC: A multi-producer single-consumer channel 14 | 15 | Implementations not yet available: SPSC and broadcast. 16 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/channel/src/error.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Errors for the channels. 4 | //! 5 | //! Important note: It is important not to use `!Send` data types in errors (e.g. avoid using Rc) to 6 | //! ensure these errors can be emitted in both `Send` and `!Send` contexts. 7 | 8 | /// Errors that can occur sending messages to a channel. 9 | #[derive(thiserror::Error, Debug)] 10 | pub enum SendError { 11 | /// The channel is full and the message could not be sent. 12 | #[error("Channel is full and the message could not be sent")] 13 | Full(T), 14 | 15 | /// The channel is closed and the message could not be sent. 16 | #[error("Channel is closed and the message could not be sent")] 17 | Closed(T), 18 | } 19 | 20 | /// Errors that can occur when consuming messages from a channel. 21 | #[derive(thiserror::Error, Debug)] 22 | pub enum RecvError { 23 | /// The channel is closed. 24 | #[error("The channel is closed")] 25 | Closed, 26 | 27 | /// The channel is empty. 28 | #[error("The channel is empty")] 29 | Empty, 30 | } 31 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/channel/src/lib.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Channel implementations optimized for single-threaded async runtime 4 | 5 | pub mod error; 6 | pub mod mpmc; 7 | pub mod mpsc; 8 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/config/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "otap-df-config" 3 | description = "Configuration model for OTAP pipeline" 4 | version.workspace = true 5 | authors.workspace = true 6 | repository.workspace = true 7 | license.workspace = true 8 | publish.workspace = true 9 | edition.workspace = true 10 | rust-version.workspace = true 11 | 12 | [lints] 13 | workspace = true 14 | 15 | [dependencies] 16 | thiserror = { workspace = true } 17 | serde = { workspace = true } 18 | serde_json = { workspace = true } -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/config/README.md: -------------------------------------------------------------------------------- 1 | # OTAP Pipeline Configuration Model 2 | 3 | Status: **WIP** 4 | 5 | This crate contains the definition of a configuration model for defining 6 | pipelines. The configuration consists of nodes 7 | and edges. A validation system is provided to ensure that the configuration 8 | forms a Directed Acyclic Graph (DAG). A 9 | basic optimization mechanism is also included to optimize the DAG. 10 | 11 | This crate is currently under development. 12 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/config/src/error.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Errors for the config crate. 4 | 5 | use crate::SignalType; 6 | 7 | #[derive(thiserror::Error, Debug)] 8 | pub enum Error { 9 | #[error("Cycle detected involving nodes: {0:?}")] 10 | CycleDetected(Vec), 11 | 12 | #[error("Type mismatch on edge from `{from_id}` ({from_out:?}) to `{to_id}` ({to_in:?})")] 13 | TypeMismatch { 14 | from_id: String, 15 | to_id: String, 16 | from_out: SignalType, 17 | to_in: SignalType, 18 | }, 19 | 20 | #[error("Duplicated node id `{0}`")] 21 | DuplicatedNodeId(String), 22 | 23 | #[error("Edge references unknown node `{0}`")] 24 | UnknownNode(String), 25 | } 26 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/engine/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "otap-df-engine" 3 | description = "Async pipeline engine" 4 | version.workspace = true 5 | authors.workspace = true 6 | repository.workspace = true 7 | license.workspace = true 8 | publish.workspace = true 9 | edition.workspace = true 10 | rust-version.workspace = true 11 | 12 | [lints] 13 | workspace = true 14 | 15 | [dependencies] 16 | otap-df-channel = { path = "../channel" } 17 | otap-df-config = { path = "../config" } 18 | 19 | thiserror = { workspace = true } 20 | serde_json = { workspace = true } 21 | tokio = { workspace = true } 22 | async-trait = { workspace = true } 23 | 24 | socket2 = "0.5.9" -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/engine/src/lib.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Async Pipeline Engine 4 | 5 | pub mod error; 6 | pub mod exporter; 7 | pub mod message; 8 | pub mod processor; 9 | pub mod receiver; 10 | 11 | pub mod config; 12 | mod effect_handler; 13 | pub mod local; 14 | pub mod pipeline; 15 | pub mod shared; 16 | 17 | pub mod testing; 18 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/engine/src/local/mod.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Traits and structs defining the local (!Send) version of receivers, processors, and exporters. 4 | 5 | pub mod exporter; 6 | pub mod processor; 7 | pub mod receiver; 8 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/engine/src/shared/mod.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Traits and structs defining the shared (Send) version of receivers, processors, and exporters. 4 | 5 | pub mod exporter; 6 | pub mod processor; 7 | pub mod receiver; 8 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otap/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "otap-df-otap" 3 | description = "OTAP nodes" 4 | version.workspace = true 5 | authors.workspace = true 6 | repository.workspace = true 7 | license.workspace = true 8 | publish.workspace = true 9 | edition.workspace = true 10 | rust-version.workspace = true 11 | 12 | [lints] 13 | workspace = true 14 | 15 | [dependencies] 16 | otap-df-engine = { path = "../engine" } 17 | otap-df-channel = { path = "../channel" } 18 | otap-df-otlp = { path = "../otlp" } 19 | tonic = { version = "0.13.1", default-features = false, features = [ 20 | "server", 21 | "channel", 22 | "router", 23 | "transport", 24 | "codegen", 25 | "prost", 26 | "zstd", 27 | "gzip", 28 | "deflate", 29 | ] } 30 | prost = "0.13.5" 31 | tokio-stream = "0.1.17" 32 | async-stream = "0.3.6" 33 | thiserror = { workspace = true } 34 | serde_json = { workspace = true } 35 | tokio = { workspace = true } 36 | async-trait = { workspace = true } 37 | 38 | [dev-dependencies] 39 | portpicker = "0.1.1" -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otap/README.md: -------------------------------------------------------------------------------- 1 | # OTAP Nodes 2 | 3 | Status: **Skeleton** 4 | 5 | This crate will contain the implementation of the OTAP receiver, processor, 6 | exporter. 7 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otap/src/lib.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Implementation of the OTAP nodes (receiver, exporter, processor). 4 | //! 5 | /// gRPC service implementation 6 | pub mod grpc; 7 | /// Implementation of OTAP Exporter that implements the exporter trait 8 | pub mod otap_exporter; 9 | /// Implementation of OTAP Receiver that implements the receiver trait 10 | pub mod otap_receiver; 11 | /// Generated protobuf files 12 | pub mod proto; 13 | 14 | /// testing utilities 15 | #[cfg(test)] 16 | mod mock; 17 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otap/src/proto/mod.rs: -------------------------------------------------------------------------------- 1 | // Disallow some rustc and clippy lints for the generated code 2 | // (applied to all modules in this file). 3 | 4 | #![allow(unused_results)] 5 | #![allow(missing_docs)] 6 | #![allow(unused_qualifications)] 7 | #![allow(clippy::must_use_candidate)] 8 | #![allow(clippy::enum_variant_names)] 9 | #![allow(rustdoc::invalid_html_tags)] 10 | 11 | #[path = ""] 12 | pub mod opentelemetry { 13 | #[path = ""] 14 | pub mod experimental { 15 | #[path = ""] 16 | pub mod arrow { 17 | #[path = "opentelemetry.proto.experimental.arrow.v1.rs"] 18 | pub mod v1; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otlp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "otap-df-otlp" 3 | description = "OTLP nodes" 4 | version.workspace = true 5 | authors.workspace = true 6 | repository.workspace = true 7 | license.workspace = true 8 | publish.workspace = true 9 | edition.workspace = true 10 | rust-version.workspace = true 11 | 12 | [lints] 13 | workspace = true 14 | 15 | [dependencies] 16 | otap-df-engine = { path = "../engine" } 17 | otap-df-channel = { path = "../channel" } 18 | tonic = { version = "0.13.1", default-features = false, features = [ 19 | "server", 20 | "channel", 21 | "router", 22 | "transport", 23 | "codegen", 24 | "prost", 25 | "zstd", 26 | "gzip", 27 | "deflate", 28 | ] } 29 | prost = "0.13.5" 30 | thiserror = { workspace = true } 31 | serde_json = { workspace = true } 32 | tokio = { workspace = true } 33 | async-trait = { workspace = true } 34 | 35 | 36 | [dev-dependencies] 37 | portpicker = "0.1.1" -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otlp/README.md: -------------------------------------------------------------------------------- 1 | # OTLP Nodes 2 | 3 | Status: **Skeleton** 4 | 5 | This crate will contain the implementation of the OTLP receiver, OTLP exporter, 6 | and potentially other OTLP-related processors. 7 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otlp/src/compression.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! 4 | //! Defines a compression enum to abstract from tonic and allows the exporter and receiver to get the respective tonic equivalent 5 | //! 6 | 7 | use tonic::codec::CompressionEncoding; 8 | 9 | /// Enum to represent various compression methods 10 | #[derive(Debug)] 11 | pub enum CompressionMethod { 12 | /// Fastest compression 13 | Zstd, 14 | /// Most compatible compression method 15 | Gzip, 16 | /// Used for legacy systems 17 | Deflate, 18 | } 19 | 20 | impl CompressionMethod { 21 | /// map the compression method to the proper tonic compression encoding equivalent 22 | /// use the CompressionMethod enum to abstract from tonic 23 | #[must_use] 24 | pub fn map_to_compression_encoding(&self) -> CompressionEncoding { 25 | match *self { 26 | CompressionMethod::Gzip => CompressionEncoding::Gzip, 27 | CompressionMethod::Zstd => CompressionEncoding::Zstd, 28 | CompressionMethod::Deflate => CompressionEncoding::Deflate, 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otlp/src/lib.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! Implementation of the OTLP nodes (receiver, exporter, processor). 4 | 5 | /// compression formats 6 | pub mod compression; 7 | /// gRPC service implementation 8 | pub mod grpc; 9 | /// otlp exporter implementation 10 | pub mod otlp_exporter; 11 | /// Implementation of OTLP Receiver that implements the receiver trait 12 | pub mod otlp_receiver; 13 | /// Generated protobuf files 14 | pub mod proto; 15 | 16 | /// grpc mock server for testing 17 | #[cfg(test)] 18 | mod mock; 19 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otlp/src/proto/mod.rs: -------------------------------------------------------------------------------- 1 | // Disallow some rustc and clippy lints for the generated code 2 | // (applied to all modules in this file). 3 | 4 | #![allow(unused_results)] 5 | #![allow(missing_docs)] 6 | #![allow(unused_qualifications)] 7 | #![allow(clippy::must_use_candidate)] 8 | #![allow(clippy::enum_variant_names)] 9 | #![allow(rustdoc::invalid_html_tags)] 10 | 11 | #[path = ""] 12 | pub mod opentelemetry { 13 | 14 | #[path = ""] 15 | pub mod collector { 16 | #[path = ""] 17 | pub mod logs { 18 | #[path = "opentelemetry.proto.collector.logs.v1.rs"] 19 | pub mod v1; 20 | } 21 | #[path = ""] 22 | pub mod metrics { 23 | #[path = "opentelemetry.proto.collector.metrics.v1.rs"] 24 | pub mod v1; 25 | } 26 | #[path = ""] 27 | pub mod trace { 28 | #[path = "opentelemetry.proto.collector.trace.v1.rs"] 29 | pub mod v1; 30 | } 31 | #[path = ""] 32 | pub mod profiles { 33 | #[path = "opentelemetry.proto.collector.profiles.v1development.rs"] 34 | pub mod v1development; 35 | } 36 | } 37 | 38 | #[path = ""] 39 | pub mod logs { 40 | #[path = "opentelemetry.proto.logs.v1.rs"] 41 | pub mod v1; 42 | } 43 | 44 | #[path = ""] 45 | pub mod metrics { 46 | #[path = "opentelemetry.proto.metrics.v1.rs"] 47 | pub mod v1; 48 | } 49 | 50 | #[path = ""] 51 | pub mod trace { 52 | #[path = "opentelemetry.proto.trace.v1.rs"] 53 | pub mod v1; 54 | } 55 | 56 | #[path = ""] 57 | pub mod profiles { 58 | #[path = "opentelemetry.proto.profiles.v1development.rs"] 59 | pub mod v1development; 60 | } 61 | 62 | #[path = ""] 63 | pub mod common { 64 | #[path = "opentelemetry.proto.common.v1.rs"] 65 | pub mod v1; 66 | } 67 | 68 | #[path = ""] 69 | pub mod resource { 70 | #[path = "opentelemetry.proto.resource.v1.rs"] 71 | pub mod v1; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /rust/otap-dataflow/crates/otlp/src/proto/opentelemetry.proto.resource.v1.rs: -------------------------------------------------------------------------------- 1 | // This file is @generated by prost-build. 2 | /// Resource information. 3 | #[derive(Clone, PartialEq, ::prost::Message)] 4 | pub struct Resource { 5 | /// Set of attributes that describe the resource. 6 | /// Attribute keys MUST be unique (it is not allowed to have more than one 7 | /// attribute with the same key). 8 | #[prost(message, repeated, tag = "1")] 9 | pub attributes: ::prost::alloc::vec::Vec, 10 | /// dropped_attributes_count is the number of dropped attributes. If the value is 0, then 11 | /// no attributes were dropped. 12 | #[prost(uint32, tag = "2")] 13 | pub dropped_attributes_count: u32, 14 | /// Set of entities that participate in this Resource. 15 | /// 16 | /// Note: keys in the references MUST exist in attributes of this message. 17 | /// 18 | /// Status: \[Development\] 19 | #[prost(message, repeated, tag = "3")] 20 | pub entity_refs: ::prost::alloc::vec::Vec, 21 | } 22 | -------------------------------------------------------------------------------- /rust/otap-dataflow/docs/architecture.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /rust/otap-dataflow/docs/glossary.md: -------------------------------------------------------------------------------- 1 | # Glossary 2 | 3 | **Control Message** 4 | TBD 5 | 6 | **Pipeline** 7 | TBD 8 | 9 | **Pipeline Engine** 10 | TBD 11 | 12 | **Exporter** 13 | TBD 14 | 15 | **OTAP** 16 | TBD 17 | 18 | **OTLP** 19 | TBD 20 | 21 | **Processor** 22 | TBD 23 | 24 | **Receiver** 25 | TBD 26 | 27 | **Telemetry Data** 28 | TBD 29 | -------------------------------------------------------------------------------- /rust/otap-dataflow/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "stable" 3 | components = [ "rustfmt", "clippy" ] -------------------------------------------------------------------------------- /rust/otap-dataflow/rustfmt.toml: -------------------------------------------------------------------------------- 1 | style_edition = "2024" 2 | max_width = 100 3 | reorder_imports = true -------------------------------------------------------------------------------- /rust/otap-dataflow/src/error.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! All top-level errors that can occur in the OTAP Pipeline project. 4 | 5 | /// All top-level errors that can occur in the OTAP Pipeline project. 6 | #[derive(thiserror::Error, Debug)] 7 | pub enum Error { 8 | /// A wrapper for the config errors. 9 | #[error("A config error occurred: {0}")] 10 | ConfigError(#[from] crates::config::Error), 11 | 12 | /// A wrapper for the pipeline engine errors. 13 | #[error("A pipeline engine error occurred: {0}")] 14 | EngineError(#[from] crates::engine::Error), 15 | } -------------------------------------------------------------------------------- /rust/otap-dataflow/src/lib.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | 3 | //! OTAP Pipeline Engine Library 4 | 5 | pub mod error; -------------------------------------------------------------------------------- /rust/otap-dataflow/xtask/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "xtask" 3 | publish = false # xtask is not meant to be published 4 | version.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | repository.workspace = true 8 | license.workspace = true 9 | rust-version.workspace = true 10 | 11 | [dependencies] 12 | anyhow = "1.0.98" 13 | toml = "0.8.22" 14 | tonic-build = { version = "0.13.1", features = ["cleanup-markdown"] } 15 | 16 | [lints.rust] 17 | unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin_include)'] } 18 | -------------------------------------------------------------------------------- /rust/otap-dataflow/xtask/README.md: -------------------------------------------------------------------------------- 1 | # Readme 2 | 3 | TBD 4 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/.clippy.toml: -------------------------------------------------------------------------------- 1 | msrv = "1.86.0" 2 | warn-on-all-wildcard-imports = true 3 | allow-expect-in-tests = true 4 | allow-unwrap-in-tests = true 5 | allow-dbg-in-tests = true 6 | allow-print-in-tests = true 7 | 8 | # Disallow specific methods 9 | disallowed-methods = [] 10 | 11 | # Disallow specific types 12 | disallowed-types = [ 13 | { path = "once_cell::sync::Lazy", reason = "Please use `std::sync::LazyLock` instead." }, 14 | ] 15 | 16 | # Disallow specific macros 17 | disallowed-macros = [ 18 | { path = "lazy_static::lazy_static", reason = "Please use `std::sync::LazyLock` instead." }, 19 | ] 20 | 21 | doc-valid-idents = [ 22 | "OTEL", 23 | "..", # add the default list 24 | ] 25 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 18 | - name: Pull & update submodules recursively 19 | run: | 20 | git submodule update --init --recursive 21 | git submodule update --recursive --remote 22 | - name: Install Protoc 23 | uses: arduino/setup-protoc@v3 24 | - name: Clippy 25 | run: cargo clippy --workspace --all-targets -- -D warnings 26 | - name: Build 27 | run: cargo build --verbose 28 | - name: Run tests 29 | run: cargo test --verbose 30 | license-header-check: 31 | runs-on: ubuntu-20.04 32 | name: Check License Header 33 | steps: 34 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 35 | - uses: korandoru/hawkeye@fac79d40d62222ef10fa8ecf9c097af6ae13ea95 # v5 36 | 37 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | crates/*/target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/README.md: -------------------------------------------------------------------------------- 1 | # OTEL Arrow Protocol Implementation in Rust 2 | 3 | The Rust implementation for [OTEL Arrow 4 | protocol](https://github.com/open-telemetry/otel-arrow). 5 | 6 | - Decoding Arrow IPC record batches to Opentelemetry data structures. 7 | - :construction: Metrics 8 | - :white_check_mark: Univariate metrics 9 | - [ ] Multivariate metrics 10 | - :white_check_mark: Logs 11 | - [ ] Traces 12 | - Encoding Opentelemetry data structures to Arrow IPC record batches. 13 | - [ ] Metrics 14 | - :construction: Logs 15 | - [ ] Traces 16 | 17 | ## Build 18 | 19 | ```bash 20 | git clone https://github.com/open-telemetry/otel-arrow.git 21 | cd rust/otel-arrow-rust && git submodule update --init --recursive 22 | cargo build --release 23 | ``` 24 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/licenserc.toml: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | inlineHeader = """Licensed under the Apache License, Version 2.0 (the "License"); 14 | you may not use this file except in compliance with the License. 15 | You may obtain a copy of the License at 16 | 17 | http://www.apache.org/licenses/LICENSE-2.0 18 | 19 | Unless required by applicable law or agreed to in writing, software 20 | distributed under the License is distributed on an "AS IS" BASIS, 21 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 22 | See the License for the specific language governing permissions and 23 | limitations under the License.""" 24 | 25 | includes = [ 26 | "*.rs", 27 | ] 28 | 29 | excludes = [ 30 | "src/proto/opentelemetry.proto.experimental.arrow.v1.rs", 31 | ] 32 | 33 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2024-12-25" 3 | components = [ "rustfmt", "clippy" ] -------------------------------------------------------------------------------- /rust/otel-arrow-rust/rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2024" 2 | style_edition = "2024" 3 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/decode.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | pub mod decoder; 14 | pub mod record_message; 15 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/decode/record_message.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | use crate::proto::opentelemetry::arrow::v1::ArrowPayloadType; 14 | use arrow::array::RecordBatch; 15 | 16 | /// Wrapper for [RecordBatch]. 17 | pub struct RecordMessage { 18 | #[allow(unused)] 19 | pub(crate) batch_id: i64, 20 | #[allow(unused)] 21 | pub(crate) schema_id: String, 22 | pub(crate) payload_type: ArrowPayloadType, 23 | pub(crate) record: RecordBatch, 24 | } 25 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | //! This crate contains the rust implementation of the OTEL Arrow Protocol. 14 | //! It contains code for decoding OTAP Arrow record batches into OTLP protos, 15 | //! and encoding OTLP protos into OTAP Messages. It also contains 16 | //! the rust implementation of pdata. 17 | 18 | #[allow(dead_code)] 19 | pub(crate) mod arrays; 20 | mod decode; 21 | mod error; 22 | pub mod otap; 23 | pub mod otlp; 24 | #[allow(dead_code)] 25 | pub mod schema; 26 | #[cfg(test)] 27 | mod test_util; 28 | #[cfg(test)] 29 | mod validation; 30 | 31 | pub mod pdata; 32 | pub mod proto; 33 | 34 | pub use decode::decoder::Consumer; 35 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/otlp.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | // TODO write documentation for this crate 14 | #![allow(missing_docs)] 15 | 16 | pub mod attributes; 17 | pub mod logs; 18 | pub mod metrics; 19 | 20 | mod common; 21 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/otlp/attributes.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | pub mod cbor; 14 | pub mod decoder; 15 | mod parent_id; 16 | pub mod store; 17 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/otlp/logs/related_data.rs: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | use crate::error; 5 | use crate::otap::OtapBatch; 6 | use crate::otlp::attributes::store::Attribute16Store; 7 | use crate::proto::opentelemetry::arrow::v1::ArrowPayloadType; 8 | 9 | pub struct RelatedData { 10 | pub(crate) log_record_id: u16, 11 | 12 | pub(crate) res_attr_map_store: Option, 13 | pub(crate) scope_attr_map_store: Option, 14 | pub(crate) log_record_attr_map_store: Option, 15 | } 16 | 17 | impl<'a> TryFrom<&'a OtapBatch> for RelatedData { 18 | type Error = error::Error; 19 | 20 | fn try_from(otap_batch: &'a OtapBatch) -> error::Result { 21 | Ok(Self { 22 | log_record_id: 0, 23 | res_attr_map_store: otap_batch 24 | .get(ArrowPayloadType::ResourceAttrs) 25 | .map(Attribute16Store::try_from) 26 | .transpose()?, 27 | scope_attr_map_store: otap_batch 28 | .get(ArrowPayloadType::ScopeAttrs) 29 | .map(Attribute16Store::try_from) 30 | .transpose()?, 31 | log_record_attr_map_store: otap_batch 32 | .get(ArrowPayloadType::LogAttrs) 33 | .map(Attribute16Store::try_from) 34 | .transpose()?, 35 | }) 36 | } 37 | } 38 | 39 | impl RelatedData { 40 | pub fn log_record_id_from_delta(&mut self, delta: u16) -> u16 { 41 | self.log_record_id += delta; 42 | self.log_record_id 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/otlp/metrics/data_points.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | pub mod data_point_store; 14 | mod exp_histogram; 15 | pub mod histogram; 16 | pub mod number; 17 | pub mod summary; 18 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/otlp/metrics/data_points/data_point_store.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | use crate::proto::opentelemetry::metrics::v1::{ 14 | ExponentialHistogramDataPoint, HistogramDataPoint, NumberDataPoint, SummaryDataPoint, 15 | }; 16 | use std::collections::HashMap; 17 | 18 | #[derive(Default)] 19 | pub struct DataPointStore { 20 | //todo: looks like this field is also unused in otel-arrow: https://github.com/open-telemetry/otel-arrow/blob/985aa1500a012859cec44855e187eacf46eda7c8/pkg/otel/metrics/otlp/number_data_point.go#L40 21 | #[allow(dead_code)] 22 | next_id: u16, 23 | data_point_by_id: HashMap>, 24 | } 25 | 26 | impl DataPointStore 27 | where 28 | T: Default, 29 | { 30 | pub fn get_or_default(&mut self, key: u16) -> &mut Vec { 31 | self.data_point_by_id.entry(key).or_default() 32 | } 33 | } 34 | 35 | pub type NumberDataPointsStore = DataPointStore; 36 | pub type SummaryDataPointsStore = DataPointStore; 37 | pub type HistogramDataPointsStore = DataPointStore; 38 | pub type EHistogramDataPointsStore = DataPointStore; 39 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/pdata/README.md: -------------------------------------------------------------------------------- 1 | # Pipeline data 2 | 3 | The pipeline data object is an enriched library of code based on 4 | underlying objects used in transport. 5 | 6 | For OTLP, the underlying object is a protobuf message defined in the 7 | [opentelemetry-proto](https://github.com/open-telemetry/opentelemetry-proto) 8 | repository. 9 | 10 | - **[OTLP pipeline data interface](./otlp/README.md)** 11 | 12 | For OTAP data frames, the underlying object is a set of Arrow arrays 13 | corresponding with OpenTelemetry concepts such as Resource, Scope, 14 | LogRecord, Span, etc. 15 | 16 | For OTAP streams, the associated protobuf messages are defined in this 17 | repository, see ../../../../proto/README.md. 18 | 19 | - **OTAP pipeline data interface: TODO** 20 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/pdata/otlp/README.md: -------------------------------------------------------------------------------- 1 | # OTLP pipeline data 2 | 3 | Status: **Development** 4 | 5 | This package defines an OTLP pipeline data interface based on 6 | prost::Message objects. The package layout: 7 | 8 | - `mod.rs`: Core module that re-exports the procedural macros and 9 | defines the main OTLP message traits. 10 | - `tests.rs`: Contains test cases for the OTLP pipeline data 11 | functionality. 12 | - `derive/`: A procedural macro crate that provides derive macros for 13 | OTLP message types: 14 | - `src/lib.rs`: Implements the `Message` procedural macro for 15 | deriving OTLP interfaces. 16 | - `model/`: A crate that provides model definitions for OTLP types: 17 | - `src/lib.rs`: Contains type definitions and metadata structures 18 | for OTLP messages. 19 | 20 | ## Design 21 | 22 | This package aims to resemble the OpenTelemetry Collector "pdata" 23 | interface, but for Rust. Like the `pdatagen` command from that 24 | repository (Golang), this package includes a model definition that 25 | simplifies a few details, especially related to "oneof" features in 26 | the protocol. 27 | 28 | The file `model/src/lib.rs` defines how each message type is treated, 29 | with a few fields in each type being declared as parameters (in order 30 | of importance), with remaining fields set through a builder 31 | pattern. There are cases where the builder pattern is not used because 32 | all fields are specified as parameters. 33 | 34 | Oneof cases are expanded into one constructor per field. Since OTLP 35 | has no more than one oneof field per message, this leads to a simple 36 | pattern. The most important example is AnyValue, and in this case 37 | there are constructors named new_string, new_int, new_double, etc. 38 | 39 | See the [tests](./tests.rs) for examples of the resulting syntax. 40 | 41 | ## Under development 42 | 43 | This package is subject to change, still under development. 44 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/pdata/otlp/derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "otlp-derive" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "Apache-2.0" 6 | 7 | [lib] 8 | proc-macro = true 9 | 10 | [dependencies] 11 | syn = { version = "2.0", features = ["full", "extra-traits"] } 12 | quote = "1.0" 13 | proc-macro2 = "1.0" 14 | convert_case = "0.6.0" 15 | otlp-model = { path = "../model" } -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/pdata/otlp/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | //! This module contains traits and utilities for OTLP (OpenTelemetry Protocol) message types. 14 | 15 | // Re-export derive macros (required for generated code) 16 | pub use otlp_derive::Message; 17 | pub use otlp_derive::qualified; 18 | 19 | // Include tests 20 | #[cfg(test)] 21 | mod tests; 22 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/pdata/otlp/model/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "otlp-model" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "Apache-2.0" 6 | 7 | [lib] 8 | 9 | [dependencies] 10 | tonic-build = "0.13" 11 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/proto/opentelemetry.proto.resource.v1.rs: -------------------------------------------------------------------------------- 1 | // This file is @generated by prost-build. 2 | #[crate::pdata::otlp::qualified("opentelemetry.proto.resource.v1.Resource")] 3 | #[derive(crate::pdata::otlp::Message, Clone, PartialEq, ::prost::Message)] 4 | pub struct Resource { 5 | #[prost(message, repeated, tag = "1")] 6 | pub attributes: ::prost::alloc::vec::Vec, 7 | #[prost(uint32, tag = "2")] 8 | pub dropped_attributes_count: u32, 9 | #[prost(message, repeated, tag = "3")] 10 | pub entity_refs: ::prost::alloc::vec::Vec, 11 | } 12 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/test_util/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed under the Apache License, Version 2.0 (the "License"); 2 | // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // 7 | // Unless required by applicable law or agreed to in writing, software 8 | // distributed under the License is distributed on an "AS IS" BASIS, 9 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | // See the License for the specific language governing permissions and 11 | // limitations under the License. 12 | 13 | mod create_array; 14 | 15 | pub(crate) use create_array::{create_record_batch, create_test_schema}; 16 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/src/validation/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // This module contains a validation test suite for OTLP and OTAP data. 5 | 6 | mod collector; 7 | mod error; 8 | mod otap; 9 | mod otlp; 10 | mod scenarios; 11 | mod service_type; 12 | mod tcp_stream; 13 | 14 | #[cfg(test)] 15 | mod testdata; 16 | -------------------------------------------------------------------------------- /rust/otel-arrow-rust/taplo.toml: -------------------------------------------------------------------------------- 1 | ## https://taplo.tamasfe.dev/configuration/file.html 2 | 3 | include = ["**/Cargo.toml"] 4 | 5 | [formatting] 6 | # Align consecutive entries vertically. 7 | align_entries = false 8 | # Append trailing commas for multi-line arrays. 9 | array_trailing_comma = true 10 | # Expand arrays to multiple lines that exceed the maximum column width. 11 | array_auto_expand = true 12 | # Collapse arrays that don't exceed the maximum column width and don't contain comments. 13 | array_auto_collapse = false 14 | # Omit white space padding from single-line arrays 15 | compact_arrays = true 16 | # Omit white space padding from the start and end of inline tables. 17 | compact_inline_tables = false 18 | # Maximum column width in characters, affects array expansion and collapse, this doesn't take whitespace into account. 19 | # Note that this is not set in stone, and works on a best-effort basis. 20 | column_width = 120 21 | # Indent based on tables and arrays of tables and their subtables, subtables out of order are not indented. 22 | indent_tables = false 23 | # The substring that is used for indentation, should be tabs or spaces (but technically can be anything). 24 | indent_string = ' ' 25 | # Add trailing newline at the end of the file if not present. 26 | trailing_newline = true 27 | # Alphabetically reorder keys that are not separated by empty lines. 28 | reorder_keys = false 29 | # Maximum amount of allowed consecutive blank lines. This does not affect the whitespace at the end of the document, as it is always stripped. 30 | allowed_blank_lines = 1 31 | # Use CRLF for line endings. 32 | crlf = false 33 | 34 | [[rule]] 35 | keys = ["build-dependencies", "dependencies", "dev-dependencies", "workspace.dependencies"] 36 | formatting = { reorder_keys = true } 37 | 38 | [[rule]] 39 | keys = ["package", "workspace.package"] 40 | formatting = { reorder_keys = false } 41 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/backend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13-slim@sha256:56a11364ffe0fee3bd60af6d6d5209eba8a99c2c16dc4c7c5861dc06261503cc 2 | 3 | WORKDIR /app 4 | 5 | COPY backend.py . 6 | COPY requirements.txt . 7 | 8 | # Install dependencies 9 | RUN pip install -r requirements.txt 10 | 11 | EXPOSE 5317 5000 12 | 13 | CMD ["python", "backend.py"] -------------------------------------------------------------------------------- /tools/pipeline_perf_test/backend/backend-manifest.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: backend 5 | labels: 6 | app: backend 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: backend 12 | template: 13 | metadata: 14 | labels: 15 | app: backend 16 | spec: 17 | containers: 18 | - name: backend 19 | image: backend-service:latest 20 | imagePullPolicy: IfNotPresent # Use the local image 21 | env: 22 | - name: LISTEN_PORT 23 | value: "5317" 24 | resources: 25 | requests: 26 | cpu: 100m 27 | memory: 200Mi 28 | limits: 29 | cpu: 1 30 | memory: 1Gi 31 | ports: 32 | - containerPort: 5317 33 | name: otlp 34 | - containerPort: 5000 35 | name: metrics 36 | 37 | --- 38 | apiVersion: v1 39 | kind: Service 40 | metadata: 41 | name: backend-service 42 | spec: 43 | selector: 44 | app: backend 45 | ports: 46 | - name: otlp 47 | port: 5317 48 | targetPort: 5317 49 | - name: metrics 50 | port: 5000 51 | targetPort: 5000 52 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/backend/backend.py: -------------------------------------------------------------------------------- 1 | import time 2 | from concurrent import futures 3 | import grpc 4 | from flask import Flask, jsonify 5 | from threading import Thread 6 | 7 | from opentelemetry.proto.collector.logs.v1 import logs_service_pb2_grpc 8 | from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceResponse 9 | 10 | # Constants for ports 11 | FLASK_PORT = 5000 12 | GRPC_PORT = 5317 13 | 14 | app = Flask(__name__) 15 | received_logs = 0 16 | 17 | class FakeLogsExporter(logs_service_pb2_grpc.LogsServiceServicer): 18 | def Export(self, request, context): 19 | global received_logs 20 | count = sum(len(ss.log_records) for rs in request.resource_logs for ss in rs.scope_logs) 21 | received_logs += count 22 | if received_logs % 10000 == 0: 23 | print(f"Total received logs: {received_logs}") 24 | return ExportLogsServiceResponse() 25 | 26 | @app.route("/metrics") 27 | def metrics(): 28 | print("Metrics endpoint called. Returning: {received_logs}") 29 | return jsonify({"received_logs": received_logs}) 30 | 31 | 32 | def start_flask(): 33 | app.run(host="0.0.0.0", port=FLASK_PORT) 34 | 35 | def serve(): 36 | try: 37 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) 38 | logs_service_pb2_grpc.add_LogsServiceServicer_to_server(FakeLogsExporter(), server) 39 | server.add_insecure_port(f"[::]:{GRPC_PORT}") 40 | server.start() 41 | print(f"Fake OTLP gRPC server started on port {GRPC_PORT}") 42 | server.wait_for_termination() 43 | except Exception as e: 44 | print(f"Error starting gRPC server: {e}") 45 | raise 46 | 47 | if __name__ == '__main__': 48 | Thread(target=start_flask).start() 49 | print("About to start gRPC server") 50 | serve() 51 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/backend/readme.md: -------------------------------------------------------------------------------- 1 | # Backend Service 2 | 3 | The backend service acts as a destination for exported telemetry data. It 4 | currently supports OTLP/gRPC on port `5317`, counts the logs it receives, and 5 | exposes these count at the `:5000/metrics` endpoint. 6 | 7 | ## Planned Enhancements 8 | 9 | - A Null Sink to discard all incoming data. 10 | - A mock service that introduces configurable latency and tracks incoming 11 | requests. 12 | - A fully functional backend to validate end-to-end pipeline integrity (allowing 13 | vendor-specific forks to extend functionality). 14 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/backend/requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio 2 | flask 3 | opentelemetry-proto -------------------------------------------------------------------------------- /tools/pipeline_perf_test/load_generator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13-slim@sha256:56a11364ffe0fee3bd60af6d6d5209eba8a99c2c16dc4c7c5861dc06261503cc 2 | 3 | WORKDIR /app 4 | 5 | # Install dependencies 6 | COPY requirements.txt . 7 | RUN pip install --no-cache-dir -r requirements.txt 8 | 9 | # Copy the load generator code 10 | COPY *.py ./ 11 | 12 | # Set environment variables 13 | ENV PYTHONUNBUFFERED=1 14 | ENV OTLP_ENDPOINT=localhost:4317 15 | 16 | # Command to run the load generator 17 | ENTRYPOINT ["python", "loadgen.py"] 18 | CMD ["--duration", "30"] -------------------------------------------------------------------------------- /tools/pipeline_perf_test/load_generator/loadgen-manifest.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: otel-loadgen 5 | labels: 6 | app: loadgen 7 | spec: 8 | ttlSecondsAfterFinished: 100 9 | template: 10 | metadata: 11 | labels: 12 | app: loadgen 13 | spec: 14 | restartPolicy: Never 15 | containers: 16 | - name: loadgen 17 | image: otel-loadgen:latest 18 | imagePullPolicy: IfNotPresent # Use the local image 19 | env: 20 | - name: OTLP_ENDPOINT 21 | value: "otel-collector-service:4317" 22 | args: 23 | - "--duration" 24 | - "{{DURATION}}" # This will be replaced by the orchestrator 25 | resources: 26 | requests: 27 | cpu: "2" 28 | memory: 1000Mi 29 | limits: 30 | cpu: "4" 31 | memory: 1000Mi 32 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/load_generator/readme.md: -------------------------------------------------------------------------------- 1 | # Load Generator 2 | 3 | A simple Python script that continuously sends OTLP logs for a specified 4 | duration. At the end of the run, it outputs the total count of logs sent to 5 | stdout, which can be parsed to determine the number of logs sent. 6 | 7 | ## Future Enhancements 8 | 9 | - Utilize language-specific OpenTelemetry SDKs. 10 | - Integrate load generation tools like Locust or custom telemetry generators. 11 | - Extend the script to support configurable options such as log size and length. 12 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/load_generator/requirements.txt: -------------------------------------------------------------------------------- 1 | docker 2 | psutil 3 | requests 4 | kubernetes 5 | pyyaml 6 | grpcio 7 | opentelemetry-proto -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the lib package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the core package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/component/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the core.component package.""" 2 | 3 | from .component import Component, ComponentPhase 4 | from .component_data import ComponentData 5 | 6 | __all__ = ["Component", "ComponentPhase", "ComponentData"] 7 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/component/component_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | component_data.py 3 | 4 | This module defines the `ComponentData` class, which encapsulates monitoring-related 5 | data collected from a system component during test execution. This includes both 6 | metrics (as arbitrary key-value pairs) and runtime information, allowing consistent 7 | representation of component state for reporting, analysis, or serialization. 8 | 9 | The `ComponentData` class provides a `from_component` factory method to construct 10 | instances directly from a `Component` and a `TestExecutionContext`. 11 | 12 | Typical usage: 13 | data = ComponentData.from_component(component, context) 14 | """ 15 | 16 | from dataclasses import dataclass, field 17 | from typing import Any, Dict, TYPE_CHECKING 18 | 19 | from ..context.test_contexts import TestExecutionContext 20 | from ..runtime.runtime import Runtime 21 | 22 | if TYPE_CHECKING: 23 | from .component import Component 24 | 25 | 26 | @dataclass 27 | class ComponentData: 28 | """The class holds data about a component including arbitrary runtime data and metrics""" 29 | 30 | metrics: Dict[str, Any] = field(default_factory=dict) 31 | runtime: Runtime = None 32 | 33 | @classmethod 34 | def from_component( 35 | cls, component: "Component", context: TestExecutionContext 36 | ) -> "ComponentData": 37 | """Create a ComponentData instance from a component and context.""" 38 | return cls( 39 | metrics=component.collect_monitoring_data(context), 40 | runtime=component.runtime, 41 | ) 42 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/context/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the core.context package.""" 2 | 3 | from .base import BaseContext 4 | from ..context.component_hook_context import ( 5 | ComponentHookContext, 6 | HookableComponentPhase, 7 | ) 8 | from ..context.test_element_hook_context import ( 9 | TestElementHookContext, 10 | HookableTestPhase, 11 | ) 12 | from .test_contexts import ( 13 | TestSuiteContext, 14 | TestExecutionContext, 15 | TestStepContext, 16 | TestFrameworkElementContext, 17 | ) 18 | 19 | __all__ = [ 20 | "BaseContext", 21 | "ComponentHookContext", 22 | "HookableComponentPhase", 23 | "TestSuiteContext", 24 | "TestExecutionContext", 25 | "TestStepContext", 26 | "TestFrameworkElementContext", 27 | "TestElementHookContext", 28 | "HookableTestPhase", 29 | ] 30 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the core.runtime package.""" 2 | 3 | from ..runtime.runtime import Runtime 4 | 5 | __all__ = ["Runtime"] 6 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/strategies/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the core.strategies package.""" 2 | 3 | from .monitoring_strategy import MonitoringStrategy, MonitoringStrategyConfig 4 | from .deployment_strategy import DeploymentStrategy, DeploymentStrategyConfig 5 | from .configuration_strategy import ConfigurationStrategy, ConfigurationStrategyConfig 6 | from .execution_strategy import ExecutionStrategy, ExecutionStrategyConfig 7 | from .reporting_strategy import ( 8 | ReportingStrategy, 9 | DestinationStrategy, 10 | FormatStrategy, 11 | ReportingStrategyConfig, 12 | ) 13 | 14 | __all__ = [ 15 | "MonitoringStrategyConfig", 16 | "MonitoringStrategy", 17 | "DeploymentStrategyConfig", 18 | "DeploymentStrategy", 19 | "ConfigurationStrategyConfig", 20 | "ConfigurationStrategy", 21 | "ExecutionStrategyConfig", 22 | "ExecutionStrategy", 23 | "ReportingStrategyConfig", 24 | "ReportingStrategy", 25 | "DestinationStrategy", 26 | "FormatStrategy", 27 | ] 28 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/strategies/hook_strategy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module: hook_strategy 3 | 4 | This module defines the `HookStrategy` abstract base class, which provides a unified 5 | interface for implementing custom hooks that can be executed at various stages of 6 | a testbed or component lifecycle. 7 | 8 | Hook strategies encapsulate behavior that can be injected into specific points in the 9 | execution flow, such as setup, teardown, or validation phases. This allows for modular 10 | and reusable logic that interacts with the runtime context without modifying core 11 | execution strategies. 12 | 13 | 14 | Classes: 15 | HookStrategyConfig(BaseModel): Base model for Hook Strategy config, passed to strategy init. 16 | HookStrategy (ABC): Abstract interface for executing a hook with access to runtime context. 17 | """ 18 | 19 | from typing import Optional 20 | from abc import ABC, abstractmethod 21 | 22 | from pydantic import BaseModel 23 | 24 | from ..context.base import BaseContext 25 | 26 | 27 | class HookStrategyConfig(BaseModel): 28 | """Base model for Execution Strategy config, passed to strategy init.""" 29 | 30 | 31 | class HookStrategy(ABC): 32 | @abstractmethod 33 | def __init__(self, config: HookStrategyConfig): 34 | """All hook strategies must be initialized with a config object.""" 35 | 36 | @abstractmethod 37 | def execute(self, ctx: BaseContext) -> None: 38 | """Execute the hook and pass it the current context.""" 39 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/test_framework/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the core.test_framework package.""" 2 | 3 | from .test_data import TestData 4 | from .test_definition import TestDefinition 5 | from .test_step import TestStep, TestStepActionConfig, TestStepAction 6 | from .test_suite import TestSuite 7 | from .test_element import TestFrameworkElement, TestLifecyclePhase 8 | 9 | __all__ = [ 10 | "TestData", 11 | "TestFrameworkElement", 12 | "TestSuite", 13 | "TestDefinition", 14 | "TestStep", 15 | "TestStepActionConfig", 16 | "TestStepAction", 17 | "TestLifecyclePhase", 18 | ] 19 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/core/test_framework/test_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module: test_data 3 | 4 | This module defines the `TestData` class, a container for storing information about a test run. 5 | It aggregates execution context and per-component data to support reporting, validation, and 6 | post-execution analysis. 7 | 8 | The `TestData` object is typically populated during or after test execution and passed to 9 | reporting strategies or other consumers that require a structured view of test results. 10 | 11 | Use cases include: 12 | - Collecting metrics or artifacts from individual test components. 13 | - Generating structured reports based on the test execution context. 14 | - Providing a unified data interface to post-processing tools or hooks. 15 | 16 | Classes: 17 | TestData: Stores the test execution context and associated component-level data. 18 | """ 19 | 20 | from dataclasses import dataclass, field 21 | from typing import Dict 22 | 23 | from ..component.component_data import ComponentData 24 | from ..context.test_contexts import TestExecutionContext 25 | 26 | 27 | @dataclass 28 | class TestData: 29 | """This class holds data about the test run, generally to be consumed by a reporting strategy.""" 30 | 31 | context: TestExecutionContext 32 | component_data: Dict[str, ComponentData] = field(default_factory=dict) 33 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/impl/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the impl package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/impl/component/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the impl.component package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/impl/strategies/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the impl.strategies package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/impl/strategies/deployment/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the impl.strategies.execution package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/process/__init__.py: -------------------------------------------------------------------------------- 1 | from .stats import ProcessStats 2 | 3 | __all__ = [ 4 | "ProcessStats", 5 | ] 6 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/process/deployed_process/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import DeployedProcess 2 | from .docker import DockerProcess 3 | from .kubernetes import K8sDeployedResource 4 | 5 | __all__ = [ 6 | "DeployedProcess", 7 | "DockerProcess", 8 | "K8sDeployedResource", 9 | ] 10 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/process/deployed_process/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | base.py 3 | 4 | Defines the abstract base class `DeployedProcess` for managing deployed system components 5 | (e.g., backend services, load generators, or collectors) in a performance testing environment. 6 | 7 | This class serves as the foundation for concrete implementations that manage specific deployment 8 | targets or environments (e.g., Docker containers, Kubernetes pods). It encapsulates shared 9 | functionality such as process monitoring and statistics collection via the `ProcessStats` utility. 10 | 11 | Key Components: 12 | - `process_type`: A string identifier for the type of deployed process. 13 | - `stats`: A `ProcessStats` object for tracking resource usage and performance metrics. 14 | - `start_monitoring(interval)`: Begin monitoring the process at a specified time interval (to be 15 | implemented by subclasses). 16 | - `stop_monitoring()`: Stop the monitoring process (to be implemented by subclasses). 17 | - `shutdown()`: Gracefully shut down the deployed process (to be implemented by subclasses). 18 | 19 | Intended to be subclassed with environment-specific logic for process lifecycle management. 20 | """ 21 | from ..stats import ProcessStats 22 | 23 | class DeployedProcess: 24 | """Base class for managing deployed processes""" 25 | 26 | def __init__(self, process_type: str): 27 | self.process_type = process_type 28 | self.stats = ProcessStats() 29 | 30 | def get_stats(self) -> ProcessStats: 31 | """Get the process stats object for the process""" 32 | return self.stats 33 | 34 | def start_monitoring(self, interval: float) -> None: 35 | """Initialize process monitoring""" 36 | pass 37 | 38 | def stop_monitoring(self) -> None: 39 | """Stop process monitoring""" 40 | pass 41 | 42 | def shutdown(self) -> None: 43 | """Gracefully shutdown the process""" 44 | pass 45 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/process/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the process.utils package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/report/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the report package.""" 2 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/runner/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the lib.runner package.""" 2 | 3 | from .registry import ( 4 | deployment_registry, 5 | monitoring_registry, 6 | reporting_registry, 7 | configuration_registry, 8 | execution_registry, 9 | hook_registry, 10 | test_step_action_registry, 11 | ) 12 | from .schema.loader import load_config_from_file, load_config_from_string 13 | 14 | __all__ = [ 15 | "deployment_registry", 16 | "monitoring_registry", 17 | "reporting_registry", 18 | "configuration_registry", 19 | "execution_registry", 20 | "hook_registry", 21 | "test_step_action_registry", 22 | "load_config_from_file", 23 | "load_config_from_string", 24 | ] 25 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/runner/schema/__init__.py: -------------------------------------------------------------------------------- 1 | """Initialization for the lib.runner.schema package.""" 2 | 3 | from .loader import load_config_from_file, load_config_from_string 4 | from .test_config import TestStepConfig, TestDefinitionConfig, TestSuiteConfig 5 | 6 | __all__ = [ 7 | "load_config_from_file", 8 | "load_config_from_string", 9 | "TestStepConfig", 10 | "TestDefinitionConfig", 11 | "TestSuiteConfig" 12 | ] 13 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/runner/schema/hook_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | hook_config.py 3 | 4 | This module defines the configuration schema for managing lifecycle hooks 5 | attached to components or test phases in the framework. 6 | 7 | Key Elements: 8 | - HooksConfig: A Pydantic model representing pre- and post-phase hooks that can 9 | be applied to a component or test. Each phase supports a list of hook wrappers 10 | and an associated strategy for how hooks should be added (e.g., append or replace). 11 | 12 | Types: 13 | - HookAddStrategy: A literal type defining allowed strategies for modifying the 14 | existing hook list ("append" or "replace"). 15 | 16 | Usage: 17 | - Use `HooksConfig` as part of higher-level configurations (e.g., test steps, 18 | test definitions, components) to declaratively define lifecycle hooks. 19 | - Hooks are wrapped using `HookWrapper` to defer instantiation until build time. 20 | 21 | This configuration model enables flexible and declarative control over how hooks 22 | are injected into different phases of execution. 23 | """ 24 | 25 | from typing import List, Literal 26 | 27 | from pydantic import BaseModel, Field 28 | 29 | from ..wrappers import HookWrapper 30 | 31 | 32 | HookAddStrategy = Literal["append", "replace"] 33 | 34 | 35 | class HooksConfig(BaseModel): 36 | """Base configuration model that specifies a hook to set on a component or test""" 37 | 38 | pre: List[HookWrapper] = Field(default_factory=list) 39 | pre_strategy: HookAddStrategy = "append" 40 | post: List[HookWrapper] = Field(default_factory=list) 41 | post_strategy: HookAddStrategy = "append" 42 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/lib/runner/schema/loader.py: -------------------------------------------------------------------------------- 1 | """ 2 | loader.py 3 | 4 | This module provides utility functions for loading a `TestSuiteConfig` from YAML sources. 5 | 6 | Functions: 7 | - load_config_from_file: Reads a YAML file from disk and parses it into a `TestSuiteConfig`. 8 | - load_config_from_string: Parses a YAML string directly into a `TestSuiteConfig`. 9 | 10 | These utilities support configuration-driven workflows, allowing test suites to be 11 | defined declaratively in YAML and loaded at runtime using Pydantic model validation. 12 | """ 13 | 14 | import yaml 15 | from pathlib import Path 16 | from .test_config import TestSuiteConfig 17 | 18 | 19 | def load_config_from_file(path: str | Path) -> TestSuiteConfig: 20 | """Loads and parses a TestSuiteConfig from a YAML file.""" 21 | with open(path, "r", encoding="utf-8") as f: 22 | data = yaml.safe_load(f) 23 | return TestSuiteConfig.model_validate(data) 24 | 25 | 26 | def load_config_from_string(yaml_str: str) -> TestSuiteConfig: 27 | """Loads and parses a TestSuiteConfig from a YAML string.""" 28 | data = yaml.safe_load(yaml_str) 29 | return TestSuiteConfig.model_validate(data) 30 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/orchestrator/requirements.txt: -------------------------------------------------------------------------------- 1 | # Dependencies for the pipeline performance test orchestrator 2 | 3 | # General dependencies 4 | requests>=2.25.0 5 | pyyaml>=5.4.1 6 | pydantic>=2.11.4 7 | 8 | # Docker management 9 | docker>=5.0.0 10 | psutil>=5.8.0 11 | 12 | # Kubernetes management 13 | kubernetes>=23.6.0 14 | 15 | # Other dependencies 16 | opentelemetry-proto>=1.12.0 -------------------------------------------------------------------------------- /tools/pipeline_perf_test/system_under_test/otel-collector/collector-config-with-batch-processor.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:4317 6 | 7 | processors: 8 | batch: 9 | 10 | exporters: 11 | debug: 12 | verbosity: detailed 13 | otlp: 14 | endpoint: backend-service:5317 15 | tls: 16 | insecure: true 17 | 18 | service: 19 | pipelines: 20 | traces: 21 | receivers: [otlp] 22 | exporters: [debug] 23 | metrics: 24 | receivers: [otlp] 25 | exporters: [debug] 26 | logs: 27 | receivers: [otlp] 28 | processors: [batch] 29 | exporters: [otlp] 30 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/system_under_test/otel-collector/collector-config.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:4317 6 | 7 | exporters: 8 | debug: 9 | verbosity: detailed 10 | otlp: 11 | endpoint: backend-service:5317 12 | tls: 13 | insecure: true 14 | 15 | service: 16 | pipelines: 17 | traces: 18 | receivers: [otlp] 19 | exporters: [debug] 20 | metrics: 21 | receivers: [otlp] 22 | exporters: [debug] 23 | logs: 24 | receivers: [otlp] 25 | exporters: [otlp] 26 | -------------------------------------------------------------------------------- /tools/pipeline_perf_test/system_under_test/readme.md: -------------------------------------------------------------------------------- 1 | # System Under Test 2 | 3 | The pipeline being tested -eg: OTLP or OTAP pipeline. 4 | 5 | - Starts with the latest OTel Collector image and evolves to support 6 | - Custom Collector builds and config variations 7 | - Multi-instance topologies via Docker Compose or Kubernetes 8 | -------------------------------------------------------------------------------- /versions.yaml: -------------------------------------------------------------------------------- 1 | # Copyright The OpenTelemetry Authors 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | module-sets: 5 | beta: 6 | version: v0.35.0 7 | modules: 8 | - github.com/open-telemetry/otel-arrow 9 | - github.com/open-telemetry/otel-arrow/collector/cmd/otelarrowcol 10 | --------------------------------------------------------------------------------