├── .github ├── dependabot.yaml └── workflows │ ├── govulncheck.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .versions ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── cmd └── benthos │ └── main.go ├── config ├── template_examples │ ├── input_sqs_example.yaml │ ├── input_stdin_uppercase.yaml │ ├── output_dead_letter.yaml │ ├── processor_hydration.yaml │ ├── processor_log_and_drop.yaml │ ├── processor_log_message.yaml │ └── processor_plugin_alias.yaml └── test │ ├── bloblang │ ├── also_tests_boolean_operands.yaml │ ├── boolean_operands.yaml │ ├── cities.blobl │ ├── cities_test.yaml │ ├── csv.yaml │ ├── csv_formatter.blobl │ ├── csv_formatter_test.yaml │ ├── env.yaml │ ├── fans.yaml │ ├── github_releases.blobl │ ├── github_releases_test.yaml │ ├── literals.yaml │ ├── message_expansion.yaml │ ├── walk_json.yaml │ └── windowed.yaml │ ├── deduplicate.yaml │ ├── deduplicate_by_batch.yaml │ ├── env_var_stuff.yaml │ ├── files │ ├── input.txt │ └── output.txt │ ├── files_for_content.yaml │ ├── filters.yaml │ ├── infile_resource_mock.yaml │ ├── json_contains_predicate.yaml │ ├── mock_http_proc.yaml │ ├── mock_http_proc_path.yaml │ ├── resources │ ├── other_mappings.yaml │ ├── other_mappings_benthos_test.yaml │ └── some_mappings.yaml │ ├── structured_metadata.yaml │ ├── unit_test_example.yaml │ └── unit_test_example_benthos_test.yaml ├── go.mod ├── go.sum ├── icon.png ├── internal ├── api │ ├── api.go │ ├── api_test.go │ ├── config.go │ ├── docs.go │ ├── dynamic_crud.go │ ├── dynamic_crud_test.go │ └── package.go ├── autoretry │ ├── auto_retry_list.go │ └── auto_retry_list_test.go ├── batch │ ├── combined_ack_func.go │ ├── combined_ack_func_test.go │ ├── count.go │ ├── count_test.go │ ├── error.go │ ├── error_test.go │ ├── package.go │ └── policy │ │ ├── batchconfig │ │ └── config.go │ │ ├── docs.go │ │ ├── package.go │ │ ├── policy.go │ │ └── policy_test.go ├── bloblang │ ├── environment.go │ ├── field │ │ ├── expression.go │ │ ├── expression_test.go │ │ ├── package.go │ │ └── resolver.go │ ├── mapping │ │ ├── assignment.go │ │ ├── executor.go │ │ ├── executor_test.go │ │ ├── package.go │ │ ├── statement.go │ │ └── target.go │ ├── package.go │ ├── package_test.go │ ├── parser │ │ ├── combinators.go │ │ ├── combinators_test.go │ │ ├── context.go │ │ ├── context_test.go │ │ ├── dot_env_parser.go │ │ ├── dot_env_parser_test.go │ │ ├── error.go │ │ ├── error_test.go │ │ ├── field_parser.go │ │ ├── field_parser_test.go │ │ ├── mapping_parser.go │ │ ├── mapping_parser_test.go │ │ ├── query_arithmetic_parser.go │ │ ├── query_arithmetic_parser_test.go │ │ ├── query_expression_parser.go │ │ ├── query_expression_parser_test.go │ │ ├── query_function_parser.go │ │ ├── query_function_parser_test.go │ │ ├── query_literal_parser.go │ │ ├── query_literal_parser_test.go │ │ ├── query_method_parser_test.go │ │ ├── query_parser.go │ │ ├── query_parser_test.go │ │ ├── root_expression_parser.go │ │ └── root_expression_parser_test.go │ ├── plugins │ │ └── bloblang.go │ └── query │ │ ├── arithmetic.go │ │ ├── arithmetic_test.go │ │ ├── docs.go │ │ ├── errors.go │ │ ├── errors_test.go │ │ ├── expression.go │ │ ├── expression_test.go │ │ ├── function_ctor.go │ │ ├── function_set.go │ │ ├── function_set_test.go │ │ ├── functions.go │ │ ├── functions_test.go │ │ ├── iterator.go │ │ ├── iterator_test.go │ │ ├── literals.go │ │ ├── literals_test.go │ │ ├── method_ctor.go │ │ ├── method_set.go │ │ ├── method_set_test.go │ │ ├── methods.go │ │ ├── methods_numbers.go │ │ ├── methods_strings.go │ │ ├── methods_structured.go │ │ ├── methods_structured_test.go │ │ ├── methods_test.go │ │ ├── package.go │ │ ├── params.go │ │ ├── params_test.go │ │ ├── parsed_test.go │ │ └── target.go ├── bundle │ ├── buffers.go │ ├── caches.go │ ├── environment.go │ ├── environment_test.go │ ├── inputs.go │ ├── metrics.go │ ├── outputs.go │ ├── package.go │ ├── processors.go │ ├── ratelimits.go │ ├── scanners.go │ ├── tracers.go │ └── tracing │ │ ├── bundle.go │ │ ├── bundle_test.go │ │ ├── events.go │ │ ├── input.go │ │ ├── output.go │ │ └── processor.go ├── cli │ ├── app.go │ ├── app_test.go │ ├── app_test │ │ └── app_test.go │ ├── blobl │ │ ├── cli.go │ │ ├── resources │ │ │ └── bloblang_editor_page.html │ │ └── server.go │ ├── common │ │ ├── logger.go │ │ ├── manager.go │ │ ├── opts.go │ │ ├── reader.go │ │ ├── run_flags.go │ │ ├── service.go │ │ └── swappable.go │ ├── create.go │ ├── create_test.go │ ├── echo.go │ ├── echo_test.go │ ├── lint.go │ ├── lint_test.go │ ├── list.go │ ├── run.go │ ├── streams.go │ ├── streams_test.go │ ├── studio │ │ ├── cli.go │ │ ├── logger.go │ │ ├── metrics │ │ │ ├── observed.go │ │ │ └── tracker.go │ │ ├── pull.go │ │ ├── pull_runner.go │ │ ├── pull_runner_test.go │ │ ├── pull_session_fs.go │ │ ├── pull_session_tracker.go │ │ ├── sync_schema.go │ │ ├── sync_schema_test.go │ │ └── tracing │ │ │ └── observed.go │ ├── template │ │ ├── cli.go │ │ └── lint.go │ └── test │ │ ├── case.go │ │ ├── case_test.go │ │ ├── cli.go │ │ ├── command.go │ │ ├── command_test.go │ │ ├── definition.go │ │ ├── definition_test.go │ │ ├── processors_provider.go │ │ └── processors_provider_test.go ├── codec │ ├── reader.go │ ├── reader_test.go │ ├── skip_group_reader.go │ ├── skip_group_reader_test.go │ └── writer.go ├── component │ ├── buffer │ │ ├── config.go │ │ ├── interface.go │ │ ├── memory_buffer_test.go │ │ ├── stream.go │ │ └── stream_test.go │ ├── cache │ │ ├── cache_metrics.go │ │ ├── cache_metrics_test.go │ │ ├── config.go │ │ └── interface.go │ ├── connection.go │ ├── errors.go │ ├── input │ │ ├── async_cut_off.go │ │ ├── async_preserver.go │ │ ├── async_preserver_test.go │ │ ├── async_reader.go │ │ ├── async_reader_test.go │ │ ├── batcher │ │ │ ├── batcher.go │ │ │ └── batcher_test.go │ │ ├── config.go │ │ ├── config │ │ │ └── config.go │ │ ├── interface.go │ │ ├── processors │ │ │ └── append.go │ │ ├── wrap_with_pipeline.go │ │ └── wrap_with_pipeline_test.go │ ├── interop │ │ └── interop.go │ ├── metrics │ │ ├── combine.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── dud_type.go │ │ ├── local.go │ │ ├── local_test.go │ │ ├── mapping.go │ │ ├── mapping_test.go │ │ ├── namespaced.go │ │ ├── namespaced_test.go │ │ ├── type.go │ │ └── vector_util.go │ ├── observability.go │ ├── output │ │ ├── async_writer.go │ │ ├── async_writer_test.go │ │ ├── batched_send.go │ │ ├── batched_send_test.go │ │ ├── batcher │ │ │ ├── batcher.go │ │ │ └── batcher_test.go │ │ ├── config.go │ │ ├── interface.go │ │ ├── not_batched.go │ │ ├── not_batched_test.go │ │ ├── processors │ │ │ └── append.go │ │ ├── wrap_with_pipeline.go │ │ └── wrap_with_pipeline_test.go │ ├── processor │ │ ├── auto_observed.go │ │ ├── auto_observed_test.go │ │ ├── config.go │ │ ├── error.go │ │ ├── execute.go │ │ ├── execute_test.go │ │ └── interface.go │ ├── ratelimit │ │ ├── config.go │ │ ├── interface.go │ │ ├── rate_limit_metrics.go │ │ └── rate_limit_metrics_test.go │ ├── scanner │ │ ├── config.go │ │ ├── interface.go │ │ └── testutil │ │ │ └── testutil.go │ ├── testutil │ │ └── from_yaml.go │ └── tracer │ │ └── config.go ├── config │ ├── config_test.go │ ├── env_vars.go │ ├── env_vars_test.go │ ├── lint.go │ ├── reader.go │ ├── reader_test.go │ ├── resource_reader.go │ ├── resource_reader_test.go │ ├── schema.go │ ├── schema │ │ └── schema.go │ ├── stream_reader.go │ ├── stream_reader_test.go │ ├── test │ │ ├── case.go │ │ ├── docs.go │ │ ├── input.go │ │ ├── output.go │ │ └── output_test.go │ ├── watcher.go │ ├── watcher_test.go │ └── watcher_wasm.go ├── cuegen │ ├── README.md │ ├── component.go │ ├── config.go │ ├── cue.go │ ├── identifiers.go │ └── schema.go ├── docs │ ├── benchmark_test.go │ ├── bloblang.go │ ├── bloblang_test.go │ ├── component.go │ ├── component_markdown.go │ ├── config.go │ ├── config_test.go │ ├── field.go │ ├── field_template.go │ ├── field_test.go │ ├── format_any.go │ ├── format_yaml.go │ ├── format_yaml_path.go │ ├── format_yaml_path_test.go │ ├── format_yaml_test.go │ ├── interop │ │ └── interop.go │ ├── json_schema.go │ ├── metrics_mapping.go │ ├── package.go │ ├── parsed.go │ ├── registry.go │ ├── walk_any.go │ ├── walk_test.go │ └── walk_yaml.go ├── filepath │ ├── glob.go │ ├── glob_test.go │ └── ifs │ │ ├── http.go │ │ ├── os.go │ │ └── os_test.go ├── httpclient │ ├── auth_oauth2.go │ ├── client.go │ ├── client_test.go │ ├── config.go │ ├── config_test.go │ ├── errors.go │ ├── errors_test.go │ ├── logger.go │ ├── logger_test.go │ ├── request.go │ └── request_test.go ├── httpserver │ ├── basic_auth.go │ ├── cors.go │ └── cors_test.go ├── impl │ ├── io │ │ ├── bloblang.go │ │ ├── bloblang_examples_test.go │ │ ├── bloblang_test.go │ │ ├── cache_file.go │ │ ├── cache_file_test.go │ │ ├── input_csv.go │ │ ├── input_csv_integration_test.go │ │ ├── input_csv_test.go │ │ ├── input_dynamic.go │ │ ├── input_dynamic_fan_in.go │ │ ├── input_dynamic_fan_in_test.go │ │ ├── input_dynamic_test.go │ │ ├── input_file.go │ │ ├── input_file_test.go │ │ ├── input_http_client.go │ │ ├── input_http_client_test.go │ │ ├── input_http_server.go │ │ ├── input_http_server_test.go │ │ ├── input_socket.go │ │ ├── input_socket_server.go │ │ ├── input_socket_server_test.go │ │ ├── input_socket_test.go │ │ ├── input_stdin.go │ │ ├── input_stdin_test.go │ │ ├── input_subprocess.go │ │ ├── input_subprocess_test.go │ │ ├── input_websocket.go │ │ ├── input_websocket_test.go │ │ ├── metrics_json_api.go │ │ ├── output_dynamic.go │ │ ├── output_dynamic_fan_out.go │ │ ├── output_dynamic_fan_out_test.go │ │ ├── output_dynamic_test.go │ │ ├── output_file.go │ │ ├── output_http_client.go │ │ ├── output_http_client_test.go │ │ ├── output_http_server.go │ │ ├── output_http_server_test.go │ │ ├── output_socket.go │ │ ├── output_socket_test.go │ │ ├── output_stdout.go │ │ ├── output_subprocess.go │ │ ├── output_subprocess_test.go │ │ ├── output_websocket.go │ │ ├── output_websocket_test.go │ │ ├── package.go │ │ ├── processor_command.go │ │ ├── processor_command_test.go │ │ ├── processor_http.go │ │ ├── processor_http_test.go │ │ ├── processor_subprocess.go │ │ └── processor_subprocess_test.go │ └── pure │ │ ├── algorithms.go │ │ ├── bloblang_encoding.go │ │ ├── bloblang_encoding_test.go │ │ ├── bloblang_examples_test.go │ │ ├── bloblang_general.go │ │ ├── bloblang_general_test.go │ │ ├── bloblang_numbers.go │ │ ├── bloblang_objects.go │ │ ├── bloblang_objects_test.go │ │ ├── bloblang_string.go │ │ ├── bloblang_string_test.go │ │ ├── bloblang_time.go │ │ ├── bloblang_time_test.go │ │ ├── buffer_memory.go │ │ ├── buffer_memory_test.go │ │ ├── buffer_none.go │ │ ├── buffer_system_window.go │ │ ├── buffer_system_window_test.go │ │ ├── cache_integration_test.go │ │ ├── cache_lru.go │ │ ├── cache_lru_test.go │ │ ├── cache_memory.go │ │ ├── cache_memory_test.go │ │ ├── cache_multilevel.go │ │ ├── cache_multilevel_test.go │ │ ├── cache_noop.go │ │ ├── cache_noop_test.go │ │ ├── cache_ttlru.go │ │ ├── cache_ttlru_test.go │ │ ├── extended │ │ ├── zstd.go │ │ └── zstd_test.go │ │ ├── input_batched.go │ │ ├── input_batched_test.go │ │ ├── input_broker.go │ │ ├── input_broker_fan_in.go │ │ ├── input_broker_fan_in_test.go │ │ ├── input_broker_test.go │ │ ├── input_generate.go │ │ ├── input_generate_test.go │ │ ├── input_inproc.go │ │ ├── input_inproc_test.go │ │ ├── input_read_until.go │ │ ├── input_read_until_test.go │ │ ├── input_resource.go │ │ ├── input_resource_test.go │ │ ├── input_sequence.go │ │ ├── input_sequence_test.go │ │ ├── metrics_logger.go │ │ ├── metrics_none.go │ │ ├── output_broker.go │ │ ├── output_broker_fan_out.go │ │ ├── output_broker_fan_out_sequential.go │ │ ├── output_broker_fan_out_sequential_test.go │ │ ├── output_broker_fan_out_test.go │ │ ├── output_broker_greedy.go │ │ ├── output_broker_greedy_test.go │ │ ├── output_broker_round_robin.go │ │ ├── output_broker_round_robin_test.go │ │ ├── output_broker_test.go │ │ ├── output_cache.go │ │ ├── output_cache_test.go │ │ ├── output_drop.go │ │ ├── output_drop_on.go │ │ ├── output_drop_on_test.go │ │ ├── output_fallback.go │ │ ├── output_fallback_test.go │ │ ├── output_inproc.go │ │ ├── output_inproc_test.go │ │ ├── output_reject.go │ │ ├── output_reject_errored.go │ │ ├── output_reject_errored_test.go │ │ ├── output_resource.go │ │ ├── output_resource_test.go │ │ ├── output_retry.go │ │ ├── output_retry_test.go │ │ ├── output_switch.go │ │ ├── output_switch_test.go │ │ ├── output_sync_response.go │ │ ├── output_sync_response_test.go │ │ ├── package.go │ │ ├── processor_archive.go │ │ ├── processor_archive_test.go │ │ ├── processor_benchmark.go │ │ ├── processor_benchmark_test.go │ │ ├── processor_bloblang.go │ │ ├── processor_bloblang_test.go │ │ ├── processor_bounds_check.go │ │ ├── processor_bounds_check_test.go │ │ ├── processor_branch.go │ │ ├── processor_branch_test.go │ │ ├── processor_cache.go │ │ ├── processor_cache_test.go │ │ ├── processor_cached.go │ │ ├── processor_cached_test.go │ │ ├── processor_catch.go │ │ ├── processor_catch_test.go │ │ ├── processor_compress.go │ │ ├── processor_compress_test.go │ │ ├── processor_crash.go │ │ ├── processor_decompress.go │ │ ├── processor_decompress_test.go │ │ ├── processor_dedupe.go │ │ ├── processor_dedupe_test.go │ │ ├── processor_for_each.go │ │ ├── processor_for_each_test.go │ │ ├── processor_grok.go │ │ ├── processor_grok_test.go │ │ ├── processor_group_by.go │ │ ├── processor_group_by_test.go │ │ ├── processor_group_by_value.go │ │ ├── processor_group_by_value_test.go │ │ ├── processor_insert_part.go │ │ ├── processor_insert_part_test.go │ │ ├── processor_jmespath.go │ │ ├── processor_jmespath_test.go │ │ ├── processor_jq.go │ │ ├── processor_jq_test.go │ │ ├── processor_jsonschema.go │ │ ├── processor_jsonschema_test.go │ │ ├── processor_log.go │ │ ├── processor_log_test.go │ │ ├── processor_mapping.go │ │ ├── processor_mapping_test.go │ │ ├── processor_metric.go │ │ ├── processor_metric_test.go │ │ ├── processor_mutation.go │ │ ├── processor_mutation_test.go │ │ ├── processor_noop.go │ │ ├── processor_parallel.go │ │ ├── processor_parallel_test.go │ │ ├── processor_parse_log.go │ │ ├── processor_parse_log_test.go │ │ ├── processor_processors.go │ │ ├── processor_processors_test.go │ │ ├── processor_rate_limit.go │ │ ├── processor_rate_limit_test.go │ │ ├── processor_resource.go │ │ ├── processor_resource_test.go │ │ ├── processor_retry.go │ │ ├── processor_retry_test.go │ │ ├── processor_select_parts.go │ │ ├── processor_select_parts_test.go │ │ ├── processor_sleep.go │ │ ├── processor_sleep_test.go │ │ ├── processor_split.go │ │ ├── processor_split_test.go │ │ ├── processor_switch.go │ │ ├── processor_switch_test.go │ │ ├── processor_sync_response.go │ │ ├── processor_try.go │ │ ├── processor_try_test.go │ │ ├── processor_unarchive.go │ │ ├── processor_unarchive_test.go │ │ ├── processor_while.go │ │ ├── processor_while_test.go │ │ ├── processor_workflow.go │ │ ├── processor_workflow_branch_map.go │ │ ├── processor_workflow_test.go │ │ ├── rate_limit_local.go │ │ ├── rate_limit_local_test.go │ │ ├── scanner_chunker.go │ │ ├── scanner_chunker_test.go │ │ ├── scanner_csv.go │ │ ├── scanner_csv_test.go │ │ ├── scanner_decompress.go │ │ ├── scanner_decompress_test.go │ │ ├── scanner_json.go │ │ ├── scanner_json_test.go │ │ ├── scanner_lines.go │ │ ├── scanner_lines_test.go │ │ ├── scanner_re_match.go │ │ ├── scanner_re_match_test.go │ │ ├── scanner_skip_bom.go │ │ ├── scanner_skip_bom_test.go │ │ ├── scanner_switch.go │ │ ├── scanner_switch_test.go │ │ ├── scanner_tar.go │ │ ├── scanner_tar_test.go │ │ ├── scanner_to_the_end.go │ │ ├── scanner_to_the_end_test.go │ │ └── tracer_none.go ├── jsonschema │ ├── stream_schema.go │ └── stream_schema_test.go ├── log │ ├── config.go │ ├── docs.go │ ├── interface.go │ ├── logrus.go │ ├── logrus_test.go │ ├── slog.go │ ├── slog_test.go │ ├── tee.go │ ├── tee_test.go │ ├── testutil │ │ └── mock_log.go │ └── wrap.go ├── manager │ ├── config.go │ ├── config_test.go │ ├── docs.go │ ├── initialization_test.go │ ├── input_wrapper.go │ ├── input_wrapper_test.go │ ├── live_resources.go │ ├── mock │ │ ├── cache.go │ │ ├── cache_test.go │ │ ├── input.go │ │ ├── input_test.go │ │ ├── manager.go │ │ ├── manager_test.go │ │ ├── output.go │ │ ├── output_test.go │ │ ├── processor.go │ │ ├── processor_test.go │ │ ├── ratelimit.go │ │ └── ratelimit_test.go │ ├── output_wrapper.go │ ├── output_wrapper_test.go │ ├── package.go │ ├── type.go │ ├── type_stream_test.go │ └── type_test.go ├── message │ ├── data.go │ ├── data_test.go │ ├── errors.go │ ├── message.go │ ├── message_test.go │ ├── part.go │ ├── part_test.go │ ├── part_with_context_test.go │ ├── sort_group.go │ ├── sort_group_test.go │ ├── transaction.go │ ├── util.go │ └── util_test.go ├── metadata │ ├── exclude_filter.go │ ├── exclude_filter_test.go │ ├── include_filter.go │ └── include_filter_test.go ├── old │ └── util │ │ └── throttle │ │ ├── package.go │ │ ├── type.go │ │ └── type_test.go ├── periodic │ ├── periodic.go │ └── periodic_test.go ├── pipeline │ ├── config_test.go │ ├── constructor.go │ ├── package.go │ ├── pool.go │ ├── pool_test.go │ ├── processor.go │ └── processor_test.go ├── retries │ └── retries.go ├── stream │ ├── config.go │ ├── config_test.go │ ├── docs.go │ ├── manager │ │ ├── api.go │ │ ├── api_test.go │ │ ├── package.go │ │ ├── type.go │ │ ├── type_stress_test.go │ │ └── type_test.go │ ├── type.go │ └── type_test.go ├── template │ ├── config.go │ ├── template.go │ └── template_test.go ├── tls │ ├── docs.go │ ├── package.go │ ├── type.go │ └── type_test.go ├── tracing │ ├── otel.go │ ├── otel_test.go │ ├── package.go │ ├── span.go │ └── v2 │ │ ├── otel.go │ │ ├── otel_test.go │ │ ├── package.go │ │ └── span.go ├── transaction │ ├── benchmarks_test.go │ ├── result_store.go │ ├── result_store_test.go │ ├── tracked.go │ └── tracked_test.go └── value │ ├── errors.go │ ├── type_helpers.go │ └── type_helpers_test.go ├── licenses └── third_party.md ├── public ├── bloblang │ ├── arguments.go │ ├── arguments_test.go │ ├── context.go │ ├── context_test.go │ ├── environment.go │ ├── environment_test.go │ ├── environment_unwrapper.go │ ├── example_plugins_v2_test.go │ ├── executor.go │ ├── executor_test.go │ ├── executor_unwrapper.go │ ├── function.go │ ├── method.go │ ├── method_test.go │ ├── package.go │ ├── parse_error.go │ ├── parse_error_test.go │ ├── spec.go │ ├── spec_test.go │ ├── util.go │ └── view.go ├── components │ ├── io │ │ └── package.go │ └── pure │ │ ├── extended │ │ └── package.go │ │ └── package.go ├── service │ ├── benchmark_test.go │ ├── buffer.go │ ├── buffer_test.go │ ├── cache.go │ ├── cache_test.go │ ├── chaos_test.go │ ├── codec │ │ ├── scanner.go │ │ └── scanner_test.go │ ├── component_config_linter.go │ ├── component_config_linter_test.go │ ├── config.go │ ├── config_backoff.go │ ├── config_backoff_test.go │ ├── config_batch_policy.go │ ├── config_batch_policy_test.go │ ├── config_bloblang.go │ ├── config_bloblang_test.go │ ├── config_docs.go │ ├── config_extract_tracing.go │ ├── config_extract_tracing_test.go │ ├── config_http.go │ ├── config_inject_tracing.go │ ├── config_inject_tracing_test.go │ ├── config_input.go │ ├── config_input_test.go │ ├── config_interpolated_string.go │ ├── config_interpolated_string_test.go │ ├── config_max_in_flight.go │ ├── config_metadata_filter.go │ ├── config_metadata_filter_test.go │ ├── config_output.go │ ├── config_output_test.go │ ├── config_processor.go │ ├── config_processor_test.go │ ├── config_scanner.go │ ├── config_test.go │ ├── config_tls.go │ ├── config_url.go │ ├── config_urls_test.go │ ├── config_util.go │ ├── environment.go │ ├── environment_schema.go │ ├── environment_test.go │ ├── errors.go │ ├── errors_test.go │ ├── example_buffer_plugin_test.go │ ├── example_cache_plugin_test.go │ ├── example_input_plugin_test.go │ ├── example_output_batched_plugin_test.go │ ├── example_output_plugin_test.go │ ├── example_processor_plugin_test.go │ ├── example_rate_limit_plugin_test.go │ ├── example_stream_builder_yaml_test.go │ ├── input.go │ ├── input_auto_retry.go │ ├── input_auto_retry_batched.go │ ├── input_auto_retry_batched_test.go │ ├── input_auto_retry_test.go │ ├── input_max_in_flight.go │ ├── input_max_in_flight_test.go │ ├── input_test.go │ ├── integration │ │ ├── cache_test_definitions.go │ │ ├── cache_test_helpers.go │ │ ├── stream_benchmark_definitions.go │ │ ├── stream_test_definitions.go │ │ └── stream_test_helpers.go │ ├── interpolated_string.go │ ├── interpolated_string_test.go │ ├── lints.go │ ├── linttype_string.go │ ├── logger.go │ ├── logger_test.go │ ├── message.go │ ├── message_batch_blobl.go │ ├── message_batch_blobl_test.go │ ├── message_test.go │ ├── metrics.go │ ├── metrics_test.go │ ├── output.go │ ├── output_test.go │ ├── package.go │ ├── plugins.go │ ├── plugins_test.go │ ├── processor.go │ ├── processor_test.go │ ├── rate_limit.go │ ├── rate_limit_test.go │ ├── resource_builder.go │ ├── resource_builder_test.go │ ├── resources.go │ ├── resources_test.go │ ├── scanner.go │ ├── service.go │ ├── service_test.go │ ├── servicetest │ │ ├── service.go │ │ └── service_test.go │ ├── stream.go │ ├── stream_builder.go │ ├── stream_builder_test.go │ ├── stream_builder_test │ │ └── stream_builder_test.go │ ├── stream_config_linter.go │ ├── stream_config_linter_test.go │ ├── stream_config_marshaller.go │ ├── stream_config_marshaller_test.go │ ├── stream_config_walker.go │ ├── stream_config_walker_test.go │ ├── stream_schema.go │ ├── stream_schema_test.go │ ├── stream_template_tester.go │ ├── tracing.go │ ├── tracing_test.go │ └── util.go └── wasm │ ├── README.md │ ├── examples │ ├── rust │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ └── louder.rs │ └── tinygo │ │ ├── README.md │ │ └── main.go │ └── tinygo │ ├── package.go │ └── tinygo.go └── resources └── scripts ├── add_copyright.sh ├── third_party.md.tpl └── third_party_licenses.sh /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | groups: 8 | production-dependencies: 9 | dependency-type: "production" 10 | development-dependencies: 11 | dependency-type: "development" 12 | open-pull-requests-limit: 10 13 | - package-ecosystem: "github-actions" 14 | directory: "/" 15 | schedule: 16 | interval: "weekly" 17 | -------------------------------------------------------------------------------- /.github/workflows/govulncheck.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Vulnerability Check 3 | 4 | on: 5 | schedule: 6 | - cron: '0 1 * * *' # run at 1 AM UTC 7 | workflow_dispatch: 8 | 9 | jobs: 10 | vulnerability-check: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v4 15 | 16 | - name: Run Go Vulnerability Check 17 | uses: golang/govulncheck-action@v1 18 | with: 19 | go-version-input: stable 20 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | schedule: 9 | - cron: '0 0/2 * * *' # Every two hours 10 | 11 | jobs: 12 | test: 13 | if: ${{ github.repository == 'redpanda-data/benthos' || github.event_name != 'schedule' }} 14 | runs-on: ubuntu-latest 15 | env: 16 | CGO_ENABLED: 0 17 | steps: 18 | 19 | - name: Checkout code 20 | uses: actions/checkout@v4 21 | 22 | - name: Install Go 23 | uses: actions/setup-go@v5 24 | with: 25 | go-version: stable 26 | 27 | - name: Deps 28 | run: make deps && git diff-index --quiet HEAD || { >&2 echo "Stale go.{mod,sum} detected. This can be fixed with 'make deps'."; exit 1; } 29 | 30 | - name: Test 31 | run: make test 32 | 33 | - name: Test WASM build 34 | run: make build-wasm 35 | 36 | golangci-lint: 37 | if: ${{ github.repository == 'redpanda-data/benthos' || github.event_name != 'schedule' }} 38 | runs-on: ubuntu-latest 39 | env: 40 | CGO_ENABLED: 0 41 | steps: 42 | 43 | - name: Checkout code 44 | uses: actions/checkout@v4 45 | 46 | - name: Install Go 47 | uses: actions/setup-go@v5 48 | with: 49 | go-version: stable 50 | 51 | - name: Set version env variables 52 | run: | 53 | cat .versions >> $GITHUB_ENV 54 | 55 | - name: Lint 56 | uses: golangci/golangci-lint-action@v8 57 | with: 58 | version: "v${{env.GOLANGCI_LINT_VERSION}}" 59 | args: "cmd/... internal/... public/..." 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | target 3 | vendor 4 | tmp 5 | site 6 | .tags 7 | .DS_Store 8 | TODO.md 9 | release_notes.md 10 | .idea 11 | .vscode 12 | .op 13 | benthos -------------------------------------------------------------------------------- /.versions: -------------------------------------------------------------------------------- 1 | GOLANGCI_LINT_VERSION=2.1.2 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2025 Redpanda Data, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: deps test test-race fmt lint 2 | 3 | GOMAXPROCS ?= 1 4 | 5 | build: 6 | @go build ./cmd/benthos 7 | 8 | build-wasm: 9 | @GOOS=js GOARCH=wasm go build -o benthos.wasm ./cmd/benthos 10 | 11 | export GOBIN ?= $(CURDIR)/bin 12 | export PATH := $(GOBIN):$(PATH) 13 | 14 | include .versions 15 | 16 | install-tools: 17 | @go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v$(GOLANGCI_LINT_VERSION) 18 | 19 | install: 20 | @go install ./cmd/benthos 21 | 22 | deps: 23 | @go mod tidy 24 | 25 | fmt: 26 | @golangci-lint fmt cmd/... internal/... public/... 27 | @go mod tidy 28 | 29 | lint: 30 | @golangci-lint run cmd/... internal/... public/... 31 | 32 | test: 33 | @go test -timeout 3m ./... 34 | @go run ./cmd/benthos template lint $(TEMPLATE_FILES) 35 | @go run ./cmd/benthos test ./config/test/... 36 | 37 | test-race: 38 | @go test -timeout 3m -race ./... 39 | 40 | generate: 41 | @go generate ./... 42 | -------------------------------------------------------------------------------- /cmd/benthos/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package main 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/public/service" 9 | 10 | // Import all plugins defined within the repo. 11 | _ "github.com/redpanda-data/benthos/v4/public/components/io" 12 | _ "github.com/redpanda-data/benthos/v4/public/components/pure" 13 | _ "github.com/redpanda-data/benthos/v4/public/components/pure/extended" 14 | ) 15 | 16 | func main() { 17 | service.RunCLI(context.Background()) 18 | } 19 | -------------------------------------------------------------------------------- /config/template_examples/input_sqs_example.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sqs_list 2 | type: input 3 | 4 | fields: 5 | - name: urls 6 | type: string 7 | kind: list 8 | - name: region 9 | type: string 10 | default: us-east-1 11 | 12 | mapping: | 13 | root.broker.inputs = this.urls.map_each(url -> { 14 | "aws_sqs": { 15 | "url": url, 16 | "region": this.region, 17 | } 18 | }) 19 | 20 | tests: 21 | - name: urls array 22 | config: 23 | urls: 24 | - https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue1 25 | - https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue2 26 | expected: 27 | broker: 28 | inputs: 29 | - aws_sqs: 30 | url: https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue1 31 | region: us-east-1 32 | - aws_sqs: 33 | url: https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue2 34 | region: us-east-1 35 | -------------------------------------------------------------------------------- /config/template_examples/input_stdin_uppercase.yaml: -------------------------------------------------------------------------------- 1 | name: stdin_uppercase 2 | type: input 3 | status: experimental 4 | categories: [ Pointless ] 5 | summary: Reads messages from stdin but uppercases everything for some reason. 6 | 7 | mapping: | 8 | root.stdin = {} 9 | root.processors = [] 10 | root.processors."-".bloblang = """ 11 | root = content().uppercase().string() 12 | """.trim() 13 | 14 | metrics_mapping: | 15 | map decrement_processor { 16 | let start_index = this.index_of("processor") 17 | let prefix = this.slice(0, $start_index) 18 | let suffix = this.slice($start_index) 19 | 20 | let index = $suffix.split(".").1.number().floor() 21 | 22 | root = $prefix + if $index == 0 { 23 | $suffix.replace_all("processor.0.", "mapping.") 24 | } else { 25 | $suffix.re_replace_all("processor\\.[0-9]+\\.", "processor.%v.".format($index - 1)) 26 | } 27 | } 28 | 29 | root = if this.contains("processor") { 30 | this.apply("decrement_processor") 31 | } 32 | 33 | tests: 34 | - name: no fields 35 | config: {} 36 | expected: 37 | stdin: {} 38 | processors: 39 | - bloblang: "root = content().uppercase().string()" 40 | -------------------------------------------------------------------------------- /config/template_examples/output_dead_letter.yaml: -------------------------------------------------------------------------------- 1 | name: dead_letter 2 | type: output 3 | status: experimental 4 | categories: [Utility] 5 | summary: Route to a dead letter queue on output failure 6 | fields: 7 | - name: max_retries 8 | description: Max times to try before routing to the dead letter 9 | type: int 10 | - name: output 11 | description: Regular output to route messages to. 12 | type: unknown 13 | - name: path 14 | description: file to save undeliverable messages to 15 | type: string 16 | mapping: | 17 | root.fallback = [] 18 | 19 | # Regular Output 20 | root.fallback."-".retry.max_retries = this.max_retries 21 | root.fallback."0".retry.output = this.output 22 | 23 | # Dead Letter Output 24 | root.fallback."-".file.path = this.path 25 | root.fallback."1".file.codec = "lines" 26 | tests: 27 | - name: Basic Unknown 28 | config: 29 | max_retries: 5 30 | output: 31 | http_client: 32 | url: http://localhost:0 33 | path: dead.log 34 | expected: 35 | fallback: 36 | - retry: 37 | max_retries: 5 38 | output: 39 | http_client: 40 | url: http://localhost:0 41 | - file: 42 | path: dead.log 43 | codec: lines 44 | -------------------------------------------------------------------------------- /config/template_examples/processor_log_and_drop.yaml: -------------------------------------------------------------------------------- 1 | name: log_and_drop 2 | type: processor 3 | categories: [ Utility ] 4 | summary: A common lossy error handling pattern. 5 | description: If a message has failed in a previous processor this one will log the error and the contents of the message and then drop it. This is a common pattern when working with data that isn't considered important. 6 | 7 | fields: [] 8 | 9 | mapping: | 10 | root.catch = [ 11 | { 12 | "log": { 13 | "level": "ERROR", 14 | "fields": { 15 | "content": "${! content() }" 16 | }, 17 | "message": "${! error() }" 18 | } 19 | }, 20 | { 21 | "bloblang": "root = deleted()" 22 | } 23 | ] 24 | 25 | metrics_mapping: | 26 | root = if this.has_suffix("1.dropped") { 27 | this.replace_all("1.dropped", "dropped") 28 | } else { deleted() } 29 | 30 | tests: 31 | - name: No fields 32 | config: {} 33 | expected: 34 | catch: 35 | - log: 36 | level: ERROR 37 | fields: 38 | content: "${! content() }" 39 | message: "${! error() }" 40 | - bloblang: root = deleted() 41 | -------------------------------------------------------------------------------- /config/template_examples/processor_log_message.yaml: -------------------------------------------------------------------------------- 1 | name: log_message 2 | type: processor 3 | summary: Print a log line that shows the contents of a message. 4 | 5 | fields: 6 | - name: level 7 | description: The level to log at. 8 | type: string 9 | default: INFO 10 | 11 | mapping: | 12 | root.log.level = this.level 13 | root.log.message = "${! content() }" 14 | root.log.fields.metadata = "${! meta() }" 15 | root.log.fields.error = "${! error() }" 16 | -------------------------------------------------------------------------------- /config/template_examples/processor_plugin_alias.yaml: -------------------------------------------------------------------------------- 1 | name: plugin_alias 2 | type: processor 3 | status: experimental 4 | summary: This is a test template to check that plugin aliases work. 5 | 6 | fields: 7 | - name: url 8 | description: the url of the thing. 9 | type: string 10 | default: http://defaultschemas.example.com 11 | 12 | mapping: 'root.schema_registry_decode.url = this.url' 13 | 14 | tests: 15 | - name: Basic fields 16 | config: 17 | url: 'http://schemas.example.com' 18 | expected: 19 | schema_registry_decode: 20 | url: 'http://schemas.example.com' 21 | 22 | - name: Use Default 23 | config: {} 24 | expected: 25 | schema_registry_decode: 26 | url: 'http://defaultschemas.example.com' 27 | -------------------------------------------------------------------------------- /config/test/bloblang/also_tests_boolean_operands.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: neither exists 3 | target_processors: ./boolean_operands.yaml#/pipeline/processors 4 | input_batch: 5 | - content: '{"none":"of the target values"}' 6 | - content: '{"first":true}' 7 | - content: '{"first":false}' 8 | - content: '{"first":true,"second":true}' 9 | output_batches: 10 | - - content_equals: '{"ands":"failed","ors":"failed"}' 11 | - content_equals: '{"ands":"failed","ors":true}' 12 | - content_equals: '{"ands":false,"ors":"failed"}' 13 | - content_equals: '{"ands":true,"ors":true}' 14 | -------------------------------------------------------------------------------- /config/test/bloblang/boolean_operands.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | ands = (first && second).catch("failed") 5 | ors = (first || second).catch("failed") 6 | 7 | tests: 8 | - name: neither exists 9 | target_processors: /pipeline/processors 10 | input_batch: 11 | - content: '{"none":"of the target values"}' 12 | - content: '{"first":true}' 13 | - content: '{"first":false}' 14 | - content: '{"first":true,"second":true}' 15 | output_batches: 16 | - - content_equals: '{"ands":"failed","ors":"failed"}' 17 | - content_equals: '{"ands":"failed","ors":true}' 18 | - content_equals: '{"ands":false,"ors":"failed"}' 19 | - content_equals: '{"ands":true,"ors":true}' 20 | -------------------------------------------------------------------------------- /config/test/bloblang/cities.blobl: -------------------------------------------------------------------------------- 1 | root.Cities = this.locations. 2 | filter(loc -> loc.state == "WA"). 3 | map_each(loc -> loc.name). 4 | sort().join(", ") -------------------------------------------------------------------------------- /config/test/bloblang/cities_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: test cities mapping 3 | target_mapping: './cities.blobl' 4 | environment: {} 5 | input_batch: 6 | - content: | 7 | { 8 | "locations": [ 9 | {"name": "Seattle", "state": "WA"}, 10 | {"name": "New York", "state": "NY"}, 11 | {"name": "Bellevue", "state": "WA"}, 12 | {"name": "Olympia", "state": "WA"} 13 | ] 14 | } 15 | output_batches: 16 | - 17 | - json_equals: {"Cities": "Bellevue, Olympia, Seattle"} -------------------------------------------------------------------------------- /config/test/bloblang/csv.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | root = content().string().split("\n").enumerated().map_each(match { 5 | index == 0 => deleted() # Drop the first line 6 | _ => match value.trim() { 7 | this.length() == 0 => deleted() # Drop empty lines 8 | _ => this.split(",") # Split the remaining by comma 9 | } 10 | }).map_each( 11 | # Then do something cool like sum each row 12 | this.map_each(this.trim().number(0)).sum() 13 | ) 14 | 15 | tests: 16 | - name: Bloblang CSV test 17 | environment: {} 18 | target_processors: /pipeline/processors 19 | input_batch: 20 | - content: | 21 | cat1,cat2,cat3 22 | 1,2,3 23 | 7,11,23 24 | 89,23,2 25 | output_batches: 26 | - - content_equals: '[6,41,114]' 27 | 28 | - name: Bloblang CSV test with whitespace 29 | environment: {} 30 | target_processors: /pipeline/processors 31 | input_batch: 32 | - content: | 33 | cat1, cat2,cat3 34 | 35 | 1, 2,3 36 | 7,11 ,23 37 | 38 | 89 , 23 ,2 39 | output_batches: 40 | - - content_equals: '[6,41,114]' -------------------------------------------------------------------------------- /config/test/bloblang/csv_formatter.blobl: -------------------------------------------------------------------------------- 1 | let header_row = this.0.keys().sort().join(",") 2 | 3 | root = $header_row + "\n" + this.map_each(element -> element.key_values(). 4 | sort_by(item -> item.key). 5 | map_each(item -> item.value.string()). 6 | join(",") 7 | ).join("\n") 8 | -------------------------------------------------------------------------------- /config/test/bloblang/csv_formatter_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: Consistent objects 3 | target_mapping: './csv_formatter.blobl' 4 | input_batch: 5 | - content: | 6 | [ 7 | { 8 | "foo": "hello world", 9 | "baz": 110, 10 | "bar": "bar value", 11 | "buz": false 12 | }, 13 | { 14 | "foo": "hello world 2", 15 | "bar": "bar value 2", 16 | "baz": 220, 17 | "buz": true 18 | }, 19 | { 20 | "foo": "hello world 3", 21 | "bar": "bar value 3", 22 | "baz": 330, 23 | "buz": true 24 | } 25 | ] 26 | output_batches: 27 | - 28 | - content_equals: |- 29 | bar,baz,buz,foo 30 | bar value,110,false,hello world 31 | bar value 2,220,true,hello world 2 32 | bar value 3,330,true,hello world 3 33 | 34 | - name: Empty 35 | target_mapping: './csv_formatter.blobl' 36 | input_batch: 37 | - content: '[]' 38 | output_batches: 39 | - 40 | - bloblang: 'error() == "failed assignment (line 1): expected object value, got null from field `this.0`"' 41 | -------------------------------------------------------------------------------- /config/test/bloblang/env.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | foo_env = env("FOO") 5 | bar_env = env("BAR") 6 | 7 | tests: 8 | - name: both exist 9 | target_processors: /pipeline/processors 10 | environment: 11 | FOO: fooval 12 | BAR: barval 13 | input_batch: 14 | - content: '{}' 15 | output_batches: 16 | - - content_equals: '{"bar_env":"barval","foo_env":"fooval"}' 17 | 18 | - name: foo exists 19 | target_processors: /pipeline/processors 20 | environment: 21 | FOO: fooval 22 | input_batch: 23 | - content: '{}' 24 | output_batches: 25 | - - content_equals: '{"bar_env":null,"foo_env":"fooval"}' 26 | 27 | - name: neither exists 28 | target_processors: /pipeline/processors 29 | environment: {} 30 | input_batch: 31 | - content: '{}' 32 | output_batches: 33 | - - content_equals: '{"bar_env":null,"foo_env":null}' 34 | 35 | -------------------------------------------------------------------------------- /config/test/bloblang/fans.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - mutation: | 4 | root.fans = this.fans.filter(fan -> fan.obsession > 0.5) 5 | 6 | tests: 7 | - name: Bloblang fans test 8 | input_batch: 9 | - json_content: 10 | id: foo 11 | fans: 12 | - {"name":"bev","obsession":0.57} 13 | - {"name":"grace","obsession":0.21} 14 | - {"name":"ali","obsession":0.89} 15 | - {"name":"vic","obsession":0.43} 16 | output_batches: 17 | - - json_equals: 18 | id: foo 19 | fans: 20 | - {"name":"bev","obsession":0.57} 21 | - {"name":"ali","obsession":0.89} 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /config/test/bloblang/github_releases.blobl: -------------------------------------------------------------------------------- 1 | root = this.map_each(release -> release.assets.map_each(asset -> { 2 | "source": "github", 3 | "dist": asset.name.re_replace_all("^benthos-?((lambda_)|_)[0-9\\.]+(-rc[0-9]+)?_([^\\.]+).*", "$2$4"), 4 | "download_count": asset.download_count, 5 | "version": release.tag_name.trim("v"), 6 | }).filter(asset -> asset.dist != "checksums")).flatten() -------------------------------------------------------------------------------- /config/test/bloblang/literals.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | root = { 5 | "1": "1", 6 | "2": if env("FOO") == "ENABLED" { 7 | "foo" 8 | }, 9 | "3": if this.count > 5 { 10 | this.count 11 | } else { 12 | deleted() 13 | }, 14 | "4": [ 15 | "1", 16 | if env("FOO") == "ENABLED" { 17 | "foo" 18 | }, 19 | if this.count > 5 { 20 | this.count 21 | } else { 22 | deleted() 23 | }, 24 | "4" 25 | ] 26 | } 27 | 28 | tests: 29 | - name: With foos 30 | target_processors: /pipeline/processors 31 | environment: 32 | FOO: ENABLED 33 | input_batch: 34 | - content: '{"count":10}' 35 | - content: '{"count":3}' 36 | output_batches: 37 | - - content_equals: '{"1":"1","2":"foo","3":10,"4":["1","foo",10,"4"]}' 38 | - content_equals: '{"1":"1","2":"foo","4":["1","foo","4"]}' 39 | 40 | - name: Without foos 41 | target_processors: /pipeline/processors 42 | environment: 43 | FOO: DISABLED 44 | input_batch: 45 | - content: '{"count":10}' 46 | - content: '{"count":3}' 47 | output_batches: 48 | - - content_equals: '{"1":"1","3":10,"4":["1",10,"4"]}' 49 | - content_equals: '{"1":"1","4":["1","4"]}' 50 | -------------------------------------------------------------------------------- /config/test/bloblang/message_expansion.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | let doc_root = this.without("items") 5 | root = items.map_each($doc_root.merge(this)) 6 | - unarchive: 7 | format: json_array 8 | 9 | tests: 10 | - name: Sample object 11 | target_processors: /pipeline/processors 12 | input_batch: 13 | - content: | 14 | { 15 | "id": "foobar", 16 | "items": [ 17 | {"content":"foo"}, 18 | {"content":"bar"}, 19 | {"content":"baz"} 20 | ] 21 | } 22 | output_batches: 23 | - - content_equals: '{"content":"foo","id":"foobar"}' 24 | - content_equals: '{"content":"bar","id":"foobar"}' 25 | - content_equals: '{"content":"baz","id":"foobar"}' 26 | -------------------------------------------------------------------------------- /config/test/bloblang/walk_json.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | map unescape_values { 5 | root = match { 6 | this.type() == "object" => this.map_each(this.value.apply("unescape_values")), 7 | this.type() == "array" => this.map_each(this.apply("unescape_values")), 8 | this.type() == "string" => this.unescape_html(), 9 | this.type() == "bytes" => this.unescape_html(), 10 | _ => this, 11 | } 12 | } 13 | root = this.or(content()).apply("unescape_values") 14 | 15 | tests: 16 | - name: Just a string 17 | target_processors: /pipeline/processors 18 | input_batch: 19 | - content: 'foo & bar' 20 | output_batches: 21 | - - content_equals: 'foo & bar' 22 | 23 | - name: Just an array 24 | target_processors: /pipeline/processors 25 | input_batch: 26 | - content: '["foo & bar",10,"1 < 2"]' 27 | output_batches: 28 | - - content_equals: '["foo & bar",10,"1 < 2"]' 29 | 30 | - name: Just an object 31 | target_processors: /pipeline/processors 32 | input_batch: 33 | - content: '{"first":"foo & bar","second":10,"third":"1 < 2"}' 34 | output_batches: 35 | - - content_equals: '{"first":"foo & bar","second":10,"third":"1 < 2"}' 36 | 37 | - name: Nested object 38 | target_processors: /pipeline/processors 39 | input_batch: 40 | - content: '{"first":{"nested":"foo & bar"},"second":10,"third":"1 < 2"}' 41 | output_batches: 42 | - - content_equals: '{"first":{"nested":"foo & bar"},"second":10,"third":"1 < 2"}' 43 | 44 | - name: Nested object with array 45 | target_processors: /pipeline/processors 46 | input_batch: 47 | - content: '{"first":{"nested":"foo & bar"},"second":10,"third":["1 < 2",{"also_nested":"2 > 1"}]}' 48 | output_batches: 49 | - - content_equals: '{"first":{"nested":"foo & bar"},"second":10,"third":["1 < 2",{"also_nested":"2 > 1"}]}' 50 | -------------------------------------------------------------------------------- /config/test/bloblang/windowed.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: | 4 | root = this 5 | doc.count = json("doc.count").from_all().sum() 6 | doc.max = json("doc.count").from_all().fold(0, match { 7 | tally < value => value 8 | _ => tally 9 | }) 10 | 11 | # Drop all documents except the first. 12 | root = match { 13 | batch_index() > 0 => deleted() 14 | } 15 | 16 | tests: 17 | - name: Bloblang windowed functions test 18 | environment: {} 19 | target_processors: /pipeline/processors 20 | input_batch: 21 | - content: '{"doc":{"count":243,"contents":"foobar 1"}}' 22 | - content: '{"doc":{"count":71,"contents":"foobar 2"}}' 23 | - content: '{"doc":{"count":10,"contents":"foobar 3"}}' 24 | - content: '{"doc":{"count":333,"contents":"foobar 4"}}' 25 | - content: '{"doc":{"count":164,"contents":"foobar 5"}}' 26 | output_batches: 27 | - - content_equals: | 28 | {"doc":{"contents":"foobar 1","count":821,"max":333}} -------------------------------------------------------------------------------- /config/test/deduplicate.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - dedupe: 4 | cache: local 5 | key: ${! content() } 6 | 7 | cache_resources: 8 | - label: local 9 | memory: 10 | default_ttl: 1m 11 | 12 | tests: 13 | - name: de-duplicate across batches 14 | input_batches: 15 | - 16 | - content: '1' 17 | - content: '2' 18 | - content: '3' 19 | - content: '4' 20 | - content: '3' 21 | - content: '3' 22 | - content: '3' 23 | - 24 | - content: '4' 25 | - content: '1' 26 | - content: '1' 27 | - content: '3' 28 | - content: '4' 29 | - content: '4' 30 | - content: '2' 31 | - content: '1' 32 | output_batches: 33 | - 34 | - content_equals: 1 35 | - content_equals: 2 36 | - content_equals: 3 37 | - content_equals: 4 38 | -------------------------------------------------------------------------------- /config/test/deduplicate_by_batch.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - mapping: | 4 | meta batch_tag = if batch_index() == 0 { 5 | nanoid(10) 6 | } 7 | - dedupe: 8 | cache: local 9 | key: ${! meta("batch_tag").from(0) + content() } 10 | 11 | cache_resources: 12 | - label: local 13 | memory: 14 | default_ttl: 1m 15 | 16 | tests: 17 | - name: de-duplicate by batches 18 | input_batches: 19 | - 20 | - content: '1' 21 | - content: '2' 22 | - content: '3' 23 | - content: '4' 24 | - content: '3' 25 | - content: '3' 26 | - content: '3' 27 | - 28 | - content: '4' 29 | - content: '1' 30 | - content: '1' 31 | - content: '3' 32 | - content: '4' 33 | - content: '4' 34 | - content: '2' 35 | - content: '1' 36 | output_batches: 37 | - 38 | - content_equals: 1 39 | - content_equals: 2 40 | - content_equals: 3 41 | - content_equals: 4 42 | - 43 | - content_equals: 4 44 | - content_equals: 1 45 | - content_equals: 3 46 | - content_equals: 2 47 | -------------------------------------------------------------------------------- /config/test/env_var_stuff.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - mutation: | 4 | root.foo = "${BENTHOS_TEST_FOO:woof}" 5 | root.bar = env("BENTHOS_TEST_BAR").or("meow") 6 | 7 | tests: 8 | - name: only defaults 9 | environment: {} 10 | input_batch: 11 | - content: '{"id":"1"}' 12 | output_batches: 13 | - 14 | - json_equals: { "id": "1", "foo": "woof", "bar": "meow" } 15 | 16 | - name: both defined 17 | environment: 18 | BENTHOS_TEST_FOO: quack 19 | BENTHOS_TEST_BAR: moo 20 | input_batch: 21 | - content: '{"id":"1"}' 22 | output_batches: 23 | - 24 | - json_equals: { "id": "1", "foo": "quack", "bar": "moo" } 25 | 26 | - name: both defined again 27 | environment: 28 | BENTHOS_TEST_FOO: tweet 29 | BENTHOS_TEST_BAR: neigh 30 | input_batch: 31 | - content: '{"id":"1"}' 32 | output_batches: 33 | - 34 | - json_equals: { "id": "1", "foo": "tweet", "bar": "neigh" } -------------------------------------------------------------------------------- /config/test/files/input.txt: -------------------------------------------------------------------------------- 1 | hello world 2 | 3 | this file 4 | 5 | is a test input 6 | 7 | and it lives in a file because 8 | 9 | it's very large and would 10 | 11 | look ugly if it were inline in the test 12 | -------------------------------------------------------------------------------- /config/test/files/output.txt: -------------------------------------------------------------------------------- 1 | HELLO WORLD 2 | 3 | THIS FILE 4 | 5 | IS A TEST INPUT 6 | 7 | AND IT LIVES IN A FILE BECAUSE 8 | 9 | IT'S VERY LARGE AND WOULD 10 | 11 | LOOK UGLY IF IT WERE INLINE IN THE TEST 12 | -------------------------------------------------------------------------------- /config/test/files_for_content.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = content().uppercase()' 4 | 5 | tests: 6 | - name: should be uppercased 7 | input_batch: 8 | - file_content: ./files/input.txt 9 | output_batches: 10 | - - file_equals: ./files/output.txt 11 | -------------------------------------------------------------------------------- /config/test/filters.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = if content().contains("delete me") { deleted() }' 4 | 5 | tests: 6 | - name: delete one of one message 7 | input_batch: 8 | - content: "hello world delete me please" 9 | 10 | - name: delete all messages 11 | input_batch: 12 | - content: "hello world delete me please" 13 | - content: "hello world 2 delete me please" 14 | - content: "hello world 3 delete me please" 15 | - content: "hello world 4 delete me please" 16 | 17 | - name: delete some messages 18 | input_batch: 19 | - content: "hello world delete me please" 20 | - content: "hello world 2" 21 | - content: "hello world 3 delete me please" 22 | - content: "hello world 4" 23 | output_batches: 24 | - - content_equals: "hello world 2" 25 | - content_equals: "hello world 4" 26 | -------------------------------------------------------------------------------- /config/test/infile_resource_mock.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - resource: http_submit 4 | 5 | processor_resources: 6 | - label: http_submit 7 | http: 8 | url: http://nonexistant.foo/ 9 | verb: POST 10 | 11 | tests: 12 | - name: test_case 13 | target_processors: /pipeline/processors 14 | mocks: 15 | http_submit: 16 | mapping: 'root = {"abc": 123}' 17 | input_batch: 18 | - json_content: 19 | foo: bar 20 | output_batches: 21 | - - json_equals: 22 | abc: 123 23 | bloblang: '!errored()' 24 | -------------------------------------------------------------------------------- /config/test/json_contains_predicate.yaml: -------------------------------------------------------------------------------- 1 | processor_resources: 2 | - label: woof_drop 3 | mapping: | 4 | root = if this.resource."service.name" == "woof" { deleted() } 5 | 6 | tests: 7 | - name: woof drop test 8 | target_processors: 'woof_drop' 9 | input_batch: 10 | - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"aaa","service.name":"meow"}}' 11 | - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"bbb","service.name":"woof"}}' 12 | - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"ccc","service.name":"quack"}}' 13 | output_batches: 14 | - 15 | - json_contains: { "resource": { "cloud.platform": "aws_eks", "host.id": "aaa" } } 16 | - json_contains: { "resource": { "cloud.platform": "aws_eks", "host.id": "ccc" } } 17 | -------------------------------------------------------------------------------- /config/test/mock_http_proc.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = "simon says: " + content()' 4 | - label: get_foobar_api 5 | http: 6 | url: http://example.com/foobar 7 | verb: GET 8 | - bloblang: 'root = content().uppercase()' 9 | 10 | tests: 11 | - name: mocks the http proc 12 | mocks: 13 | get_foobar_api: 14 | bloblang: 'root = content().string() + " this is some mock content"' 15 | input_batch: 16 | - content: "hello world" 17 | output_batches: 18 | - - content_equals: "SIMON SAYS: HELLO WORLD THIS IS SOME MOCK CONTENT" 19 | 20 | - name: mocks the http proc and also adds another processor to expose error 21 | mocks: 22 | get_foobar_api: 23 | bloblang: 'root = throw("the processor failed")' 24 | /pipeline/processors/-: 25 | bloblang: | 26 | root.content = content().string() 27 | root.error = error() 28 | input_batch: 29 | - content: "hello world" 30 | output_batches: 31 | - - json_equals: 32 | content: 'SIMON SAYS: HELLO WORLD' 33 | error: 'failed assignment (line 1): the processor failed' 34 | -------------------------------------------------------------------------------- /config/test/mock_http_proc_path.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | processors: 3 | - bloblang: 'root = "simon says: " + content()' 4 | - http: 5 | url: http://example.com/foobar 6 | verb: GET 7 | - bloblang: 'root = content().uppercase()' 8 | 9 | tests: 10 | - name: mocks the http proc 11 | mocks: 12 | /pipeline/processors/1: 13 | bloblang: 'root = content().string() + " this is some mock content"' 14 | input_batch: 15 | - content: "hello world" 16 | output_batches: 17 | - - content_equals: "SIMON SAYS: HELLO WORLD THIS IS SOME MOCK CONTENT" 18 | 19 | - name: mocks the http proc and also adds another processor to expose error 20 | mocks: 21 | /pipeline/processors/1: 22 | bloblang: 'root = throw("the processor failed")' 23 | /pipeline/processors/-: 24 | bloblang: | 25 | root.content = content().string() 26 | root.error = error() 27 | input_batch: 28 | - content: "hello world" 29 | output_batches: 30 | - - json_equals: 31 | content: 'SIMON SAYS: HELLO WORLD' 32 | error: 'failed assignment (line 1): the processor failed' 33 | -------------------------------------------------------------------------------- /config/test/resources/other_mappings.yaml: -------------------------------------------------------------------------------- 1 | processor_resources: 2 | - label: prefix 3 | bloblang: 'root = "bar " + content()' 4 | 5 | - label: upper 6 | bloblang: 'root = content().uppercase()' 7 | -------------------------------------------------------------------------------- /config/test/resources/other_mappings_benthos_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: run all resources 3 | target_processors: '/processor_resources' 4 | input_batch: 5 | - content: 'example content' 6 | output_batches: 7 | - 8 | - content_equals: BAR EXAMPLE CONTENT 9 | 10 | - name: run just prefix 11 | target_processors: '/processor_resources/0' 12 | input_batch: 13 | - content: 'example content' 14 | output_batches: 15 | - 16 | - content_equals: bar example content 17 | 18 | - name: run just upper 19 | target_processors: '/processor_resources/1' 20 | input_batch: 21 | - content: 'example content' 22 | output_batches: 23 | - 24 | - content_equals: EXAMPLE CONTENT 25 | -------------------------------------------------------------------------------- /config/test/resources/some_mappings.yaml: -------------------------------------------------------------------------------- 1 | processor_resources: 2 | - label: prefix 3 | bloblang: 'root = "foo " + content()' 4 | 5 | - label: upper 6 | bloblang: 'root = content().uppercase()' 7 | 8 | tests: 9 | - name: run all resources 10 | target_processors: '/processor_resources' 11 | input_batch: 12 | - content: 'example content' 13 | output_batches: 14 | - 15 | - content_equals: FOO EXAMPLE CONTENT 16 | 17 | - name: run just prefix 18 | target_processors: '/processor_resources/0' 19 | input_batch: 20 | - content: 'example content' 21 | output_batches: 22 | - 23 | - content_equals: foo example content 24 | 25 | - name: run just upper 26 | target_processors: '/processor_resources/1' 27 | input_batch: 28 | - content: 'example content' 29 | output_batches: 30 | - 31 | - content_equals: EXAMPLE CONTENT 32 | -------------------------------------------------------------------------------- /config/test/structured_metadata.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | stdin: 3 | codec: lines 4 | pipeline: 5 | processors: 6 | - mapping: | 7 | meta foo = { "a": "hello" } 8 | meta bar = { "b": { "c": "hello" } } 9 | meta baz = [ { "a": "hello" }, { "b": { "c": "hello" } } ] 10 | output: 11 | stdout: 12 | codec: lines 13 | 14 | tests: 15 | - name: Should not fail 16 | input_batch: 17 | - content: hello 18 | output_batches: 19 | - - metadata_equals: 20 | foo: { "a": "hello" } 21 | bar: { "b": { "c": "hello" } } 22 | baz: [ { "a": "hello" }, { "b": { "c": "hello" } } ] 23 | -------------------------------------------------------------------------------- /config/test/unit_test_example.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | generate: 3 | mapping: 'root.id = uuid_v4()' 4 | 5 | pipeline: 6 | processors: 7 | - mapping: 'root = "%vend".format(content().uppercase().string())' 8 | 9 | output: 10 | drop: {} 11 | -------------------------------------------------------------------------------- /config/test/unit_test_example_benthos_test.yaml: -------------------------------------------------------------------------------- 1 | tests: 2 | - name: example test 3 | target_processors: '/pipeline/processors' 4 | environment: {} 5 | input_batch: 6 | - content: 'example content' 7 | metadata: 8 | example_key: example metadata value 9 | output_batches: 10 | - 11 | - content_equals: EXAMPLE CONTENTend 12 | metadata_equals: 13 | example_key: example metadata value 14 | 15 | - name: empty message test 16 | target_processors: '/pipeline/processors' 17 | environment: {} 18 | input_batch: 19 | - content: '' 20 | metadata: 21 | example_key: example metadata value 22 | output_batches: 23 | - 24 | - content_equals: end 25 | metadata_equals: 26 | example_key: example metadata value 27 | -------------------------------------------------------------------------------- /icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redpanda-data/benthos/c2963df78610b6e53ffdbb1605a755832f1ec5e4/icon.png -------------------------------------------------------------------------------- /internal/api/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package api implements a type used for creating the Benthos HTTP API. 4 | package api 5 | 6 | import ( 7 | "github.com/gorilla/mux" 8 | ) 9 | 10 | // GetMuxRoute returns a *mux.Route (the result of calling .Path or .PathPrefix 11 | // on the provided router), where in cases where the path ends in a slash it 12 | // will be treated as a prefix. This isn't ideal but it's as close as we can 13 | // realistically get to the http.ServeMux behaviour with added path variables. 14 | // 15 | // NOTE: Eventually we can move back to http.ServeMux once 16 | // https://github.com/golang/go/issues/61410 is available, and that'll allow us 17 | // to make all paths explicit. 18 | func GetMuxRoute(gMux *mux.Router, path string) *mux.Route { 19 | if path != "" && path[len(path)-1] == '/' { 20 | return gMux.PathPrefix(path) 21 | } 22 | return gMux.Path(path) 23 | } 24 | -------------------------------------------------------------------------------- /internal/batch/combined_ack_func.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package batch 4 | 5 | import ( 6 | "context" 7 | "sync" 8 | ) 9 | 10 | // AckFunc is a common function signature for acknowledging receipt of messages. 11 | type AckFunc func(context.Context, error) error 12 | 13 | // CombinedAcker creates a single ack func closure that aggregates one or more 14 | // derived closures such that only once each derived closure is called the 15 | // singular ack func will trigger. If at least one derived closure receives an 16 | // error the singular ack func will send the first non-nil error received. 17 | type CombinedAcker struct { 18 | mut sync.Mutex 19 | remainingAcks int 20 | err error 21 | root AckFunc 22 | } 23 | 24 | // NewCombinedAcker creates an aggregated that derives one or more ack funcs 25 | // that, once all of which have been called, the provided root ack func is 26 | // called. 27 | func NewCombinedAcker(aFn AckFunc) *CombinedAcker { 28 | return &CombinedAcker{ 29 | remainingAcks: 0, 30 | root: aFn, 31 | } 32 | } 33 | 34 | // Derive creates a new ack func that must be called before the origin ack func 35 | // will be called. It is invalid to derive an ack func after any other 36 | // previously derived funcs have been called. 37 | func (c *CombinedAcker) Derive() AckFunc { 38 | c.mut.Lock() 39 | c.remainingAcks++ 40 | c.mut.Unlock() 41 | 42 | var decrementOnce sync.Once 43 | return func(ctx context.Context, ackErr error) (err error) { 44 | decrementOnce.Do(func() { 45 | c.mut.Lock() 46 | c.remainingAcks-- 47 | remaining := c.remainingAcks 48 | if ackErr != nil { 49 | c.err = ackErr 50 | } 51 | ackErr = c.err 52 | c.mut.Unlock() 53 | if remaining == 0 { 54 | err = c.root(ctx, ackErr) 55 | } 56 | }) 57 | return 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /internal/batch/count.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package batch 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/message" 9 | ) 10 | 11 | type batchedCountKeyType int 12 | 13 | const batchedCountKey batchedCountKeyType = iota 14 | 15 | // CtxCollapsedCount attempts to extract the actual number of messages that were 16 | // collapsed into the resulting message part. This value could be greater than 1 17 | // when users configure processors that archive batched message parts. 18 | func CtxCollapsedCount(ctx context.Context) int { 19 | if v, ok := ctx.Value(batchedCountKey).(int); ok { 20 | return v 21 | } 22 | return 1 23 | } 24 | 25 | // MessageCollapsedCount attempts to extract the actual number of messages that 26 | // were combined into the resulting batched message parts. This value could 27 | // differ from message.Len() when users configure processors that archive 28 | // batched message parts. 29 | func MessageCollapsedCount(m message.Batch) int { 30 | total := 0 31 | _ = m.Iter(func(i int, p *message.Part) error { 32 | total += CtxCollapsedCount(message.GetContext(p)) 33 | return nil 34 | }) 35 | return total 36 | } 37 | 38 | // CtxWithCollapsedCount returns a message part with a context indicating that this 39 | // message is the result of collapsing a number of messages. This allows 40 | // downstream components to know how many total messages were combined. 41 | func CtxWithCollapsedCount(ctx context.Context, count int) context.Context { 42 | base := 1 43 | if v, ok := ctx.Value(batchedCountKey).(int); ok { 44 | base = v 45 | } 46 | 47 | // The new length is the previous length plus the total messages put into 48 | // this batch (minus one to prevent double counting the original part). 49 | return context.WithValue(ctx, batchedCountKey, base-1+count) 50 | } 51 | -------------------------------------------------------------------------------- /internal/batch/count_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package batch 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | ) 12 | 13 | func TestCount(t *testing.T) { 14 | p1 := message.GetContext(message.NewPart([]byte("foo bar"))) 15 | 16 | p2 := CtxWithCollapsedCount(p1, 2) 17 | p3 := CtxWithCollapsedCount(p2, 3) 18 | p4 := CtxWithCollapsedCount(p1, 4) 19 | 20 | assert.Equal(t, 1, CtxCollapsedCount(p1)) 21 | assert.Equal(t, 2, CtxCollapsedCount(p2)) 22 | assert.Equal(t, 4, CtxCollapsedCount(p3)) 23 | assert.Equal(t, 4, CtxCollapsedCount(p4)) 24 | } 25 | 26 | func TestMessageCount(t *testing.T) { 27 | m := message.QuickBatch([][]byte{ 28 | []byte("FOO"), 29 | []byte("BAR"), 30 | []byte("BAZ"), 31 | }) 32 | 33 | assert.Equal(t, 3, MessageCollapsedCount(m)) 34 | } 35 | -------------------------------------------------------------------------------- /internal/batch/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package batch contains internal utilities for interacting with message 4 | // batches. 5 | package batch 6 | -------------------------------------------------------------------------------- /internal/batch/policy/batchconfig/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package batchconfig 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/component/processor" 7 | ) 8 | 9 | // Config contains configuration parameters for a batch policy. 10 | type Config struct { 11 | ByteSize int `json:"byte_size" yaml:"byte_size"` 12 | Count int `json:"count" yaml:"count"` 13 | Check string `json:"check" yaml:"check"` 14 | Period string `json:"period" yaml:"period"` 15 | Processors []processor.Config `json:"processors" yaml:"processors"` 16 | } 17 | 18 | // NewConfig creates a default PolicyConfig. 19 | func NewConfig() Config { 20 | return Config{ 21 | ByteSize: 0, 22 | Count: 0, 23 | Check: "", 24 | Period: "", 25 | Processors: []processor.Config{}, 26 | } 27 | } 28 | 29 | // IsNoop returns true if this batch policy configuration does nothing. 30 | func (p Config) IsNoop() bool { 31 | if p.ByteSize > 0 { 32 | return false 33 | } 34 | if p.Count > 1 { 35 | return false 36 | } 37 | if p.Check != "" { 38 | return false 39 | } 40 | if p.Period != "" { 41 | return false 42 | } 43 | if len(p.Processors) > 0 { 44 | return false 45 | } 46 | return true 47 | } 48 | 49 | // IsLimited returns true if there's any limit on the batching policy. 50 | func (p Config) IsLimited() bool { 51 | if p.ByteSize > 0 { 52 | return true 53 | } 54 | if p.Count > 0 { 55 | return true 56 | } 57 | if p.Period != "" { 58 | return true 59 | } 60 | if p.Check != "" { 61 | return true 62 | } 63 | return false 64 | } 65 | 66 | // IsHardLimited returns true if there's a realistic limit on the batching 67 | // policy, where checks are not included. 68 | func (p Config) IsHardLimited() bool { 69 | if p.ByteSize > 0 { 70 | return true 71 | } 72 | if p.Count > 0 { 73 | return true 74 | } 75 | if p.Period != "" { 76 | return true 77 | } 78 | return false 79 | } 80 | -------------------------------------------------------------------------------- /internal/batch/policy/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package policy provides tooling for creating and executing Benthos message 4 | // batch policies. 5 | package policy 6 | -------------------------------------------------------------------------------- /internal/bloblang/field/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package field implements a bloblang interpolation function templating syntax 4 | // used in some dynamic fields within Benthos. Only the query (right-hand side) 5 | // part of the bloblang spec is supported within interpolation functions. 6 | package field 7 | -------------------------------------------------------------------------------- /internal/bloblang/mapping/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package mapping provides a parser for the full bloblang mapping spec. 4 | package mapping 5 | -------------------------------------------------------------------------------- /internal/bloblang/mapping/target.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mapping 4 | 5 | // TargetType represents a mapping target type, which is a destination for a 6 | // query result to be mapped into a message. 7 | type TargetType int 8 | 9 | // TargetTypes. 10 | const ( 11 | TargetMetadata TargetType = iota 12 | TargetValue 13 | TargetVariable 14 | ) 15 | 16 | // TargetPath represents a target type and segmented path that a query function 17 | // references. An empty path indicates the root of the type is targeted. 18 | type TargetPath struct { 19 | Type TargetType 20 | Path []string 21 | } 22 | 23 | // NewTargetPath constructs a new target path from a type and zero or more path 24 | // segments. 25 | func NewTargetPath(t TargetType, path ...string) TargetPath { 26 | return TargetPath{ 27 | Type: t, 28 | Path: path, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /internal/bloblang/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package bloblang 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/bloblang/plugins" 7 | ) 8 | 9 | func init() { 10 | if err := plugins.Register(); err != nil { 11 | panic(err) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /internal/bloblang/parser/dot_env_parser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package parser 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | var dotEnvParser = func() Func[map[string]string] { 10 | assignmentParser := Sequence( 11 | NotInSet('=', ' ', '\n', '#'), 12 | Optional(SpacesAndTabs), 13 | charEquals, 14 | Optional(SpacesAndTabs), 15 | Optional(OneOf(TripleQuoteString, QuotedString, NotInSet('#', ' ', '\n'))), 16 | Optional(SpacesAndTabs), 17 | ) 18 | 19 | envFileParser := Delimited( 20 | Expect(OneOf( 21 | assignmentParser, 22 | ZeroedFuncAs[string, []string](SpacesAndTabs), 23 | ZeroedFuncAs[any, []string](EmptyLine), 24 | ZeroedFuncAs[any, []string](EndOfInput), 25 | ZeroedFuncAs[[]any, []string](Sequence( 26 | FuncAsAny(charHash), 27 | FuncAsAny(Optional(UntilFail(NotChar('\n')))), 28 | )), 29 | ), "Environment variable assignment"), 30 | NewlineAllowComment, 31 | ) 32 | 33 | return func(input []rune) Result[map[string]string] { 34 | res := envFileParser(input) 35 | if res.Err != nil { 36 | return Fail[map[string]string](res.Err, input) 37 | } 38 | vars := map[string]string{} 39 | for _, line := range res.Payload.Primary { 40 | if len(line) != 6 { 41 | continue 42 | } 43 | vars[line[0]] = line[4] 44 | } 45 | return Success(vars, res.Remaining) 46 | } 47 | }() 48 | 49 | // ParseDotEnvFile attempts to parse a .env file containing environment variable 50 | // assignments, and returns either a map of key/value assignments or an error. 51 | func ParseDotEnvFile(envFileBytes []byte) (map[string]string, error) { 52 | input := string(envFileBytes) 53 | res := dotEnvParser([]rune(input)) 54 | if res.Err != nil { 55 | line, _ := LineAndColOf([]rune(input), res.Err.Input) 56 | return nil, fmt.Errorf("%v: %v", line, res.Err.ErrorAtPositionStructured("", []rune(input))) 57 | } 58 | return res.Payload, nil 59 | } 60 | -------------------------------------------------------------------------------- /internal/bloblang/parser/query_parser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package parser 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/bloblang/query" 7 | ) 8 | 9 | func queryParser(pCtx Context) Func[query.Function] { 10 | rootParser := parseWithTails(Expect( 11 | OneOf( 12 | matchExpressionParser(pCtx), 13 | ifExpressionParser(pCtx), 14 | lambdaExpressionParser(pCtx), 15 | bracketsExpressionParser(pCtx), 16 | literalValueParser(pCtx), 17 | functionParser(pCtx), 18 | metadataReferenceParser, 19 | variableReferenceParser, 20 | fieldReferenceRootParser(pCtx), 21 | ), 22 | "query", 23 | ), pCtx) 24 | return func(input []rune) Result[query.Function] { 25 | res := SpacesAndTabs(input) 26 | return arithmeticParser(rootParser)(res.Remaining) 27 | } 28 | } 29 | 30 | func tryParseQuery(expr string) (query.Function, *Error) { 31 | res := queryParser(Context{ 32 | Functions: query.AllFunctions, 33 | Methods: query.AllMethods, 34 | })([]rune(expr)) 35 | if res.Err != nil { 36 | return nil, res.Err 37 | } 38 | return res.Payload, nil 39 | } 40 | -------------------------------------------------------------------------------- /internal/bundle/tracing/processor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package tracing 4 | 5 | import ( 6 | "context" 7 | "sync/atomic" 8 | 9 | iprocessor "github.com/redpanda-data/benthos/v4/internal/component/processor" 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | ) 12 | 13 | type tracedProcessor struct { 14 | e *events 15 | errCtr *uint64 16 | wrapped iprocessor.V1 17 | } 18 | 19 | func traceProcessor(e *events, errCtr *uint64, p iprocessor.V1) iprocessor.V1 { 20 | t := &tracedProcessor{ 21 | e: e, 22 | errCtr: errCtr, 23 | wrapped: p, 24 | } 25 | return t 26 | } 27 | 28 | func (t *tracedProcessor) UnwrapProc() iprocessor.V1 { 29 | return t.wrapped 30 | } 31 | 32 | func (t *tracedProcessor) ProcessBatch(ctx context.Context, m message.Batch) ([]message.Batch, error) { 33 | if !t.e.IsEnabled() { 34 | return t.wrapped.ProcessBatch(ctx, m) 35 | } 36 | 37 | prevErrs := make([]error, m.Len()) 38 | _ = m.Iter(func(i int, part *message.Part) error { 39 | t.e.Add(EventConsumeOf(part)) 40 | prevErrs[i] = part.ErrorGet() 41 | return nil 42 | }) 43 | 44 | outMsgs, res := t.wrapped.ProcessBatch(ctx, m) 45 | for _, outMsg := range outMsgs { 46 | _ = outMsg.Iter(func(i int, part *message.Part) error { 47 | t.e.Add(EventProduceOf(part)) 48 | fail := part.ErrorGet() 49 | if fail == nil { 50 | return nil 51 | } 52 | // TODO: Improve mechanism for tracking the introduction of errors? 53 | if len(prevErrs) <= i || prevErrs[i] == fail { 54 | return nil 55 | } 56 | _ = atomic.AddUint64(t.errCtr, 1) 57 | t.e.Add(EventErrorOf(fail)) 58 | return nil 59 | }) 60 | } 61 | if len(outMsgs) == 0 { 62 | // TODO: Find a better way of locating deletes (using batch index tracking). 63 | t.e.Add(EventDeleteOf()) 64 | } 65 | 66 | return outMsgs, res 67 | } 68 | 69 | func (t *tracedProcessor) Close(ctx context.Context) error { 70 | return t.wrapped.Close(ctx) 71 | } 72 | -------------------------------------------------------------------------------- /internal/cli/common/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package common 4 | 5 | import ( 6 | "strings" 7 | 8 | "github.com/urfave/cli/v2" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/config" 11 | "github.com/redpanda-data/benthos/v4/internal/filepath/ifs" 12 | "github.com/redpanda-data/benthos/v4/internal/log" 13 | ) 14 | 15 | // CreateLogger from a CLI context and a stream config. 16 | func CreateLogger(c *cli.Context, opts *CLIOpts, conf config.Type, streamsMode bool) (logger log.Modular, err error) { 17 | if overrideLogLevel := opts.RootFlags.GetLogLevel(c); overrideLogLevel != "" { 18 | conf.Logger.LogLevel = strings.ToUpper(overrideLogLevel) 19 | } 20 | 21 | defaultStream := opts.Stdout 22 | if !streamsMode && conf.Output.Type == "stdout" { 23 | defaultStream = opts.Stderr 24 | } 25 | if logger, err = log.New(defaultStream, ifs.OS(), conf.Logger); err != nil { 26 | return 27 | } 28 | if logger, err = opts.OnLoggerInit(logger); err != nil { 29 | return 30 | } 31 | return 32 | } 33 | -------------------------------------------------------------------------------- /internal/cli/common/reader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package common 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/config" 7 | "github.com/redpanda-data/benthos/v4/internal/docs" 8 | "github.com/redpanda-data/benthos/v4/internal/filepath/ifs" 9 | 10 | "github.com/urfave/cli/v2" 11 | ) 12 | 13 | // ReadConfig attempts to read a general service wide config via a returned 14 | // config.Reader based on input CLI flags. This includes applying any config 15 | // overrides expressed by the --set flag. 16 | func ReadConfig(c *cli.Context, cliOpts *CLIOpts, streamsMode bool) (mainPath string, inferred bool, conf *config.Reader) { 17 | path := cliOpts.RootFlags.GetConfig(c) 18 | if path == "" { 19 | // Iterate default config paths 20 | for _, dpath := range cliOpts.ConfigSearchPaths { 21 | if _, err := ifs.OS().Stat(dpath); err == nil { 22 | inferred = true 23 | path = dpath 24 | break 25 | } 26 | } 27 | } 28 | 29 | lintConf := docs.NewLintConfig(cliOpts.Environment) 30 | 31 | opts := []config.OptFunc{ 32 | config.OptSetFullSpec(cliOpts.MainConfigSpecCtor), 33 | config.OptAddOverrides(cliOpts.RootFlags.GetSet(c)...), 34 | config.OptTestSuffix("_benthos_test"), 35 | config.OptSetLintConfig(lintConf), 36 | } 37 | if streamsMode { 38 | opts = append(opts, config.OptSetStreamPaths(c.Args().Slice()...)) 39 | } 40 | if cliOpts.SecretAccessFn != nil { 41 | opts = append(opts, config.OptUseEnvLookupFunc(cliOpts.SecretAccessFn)) 42 | } 43 | return path, inferred, config.NewReader(path, cliOpts.RootFlags.GetResources(c), opts...) 44 | } 45 | -------------------------------------------------------------------------------- /internal/cli/run.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package cli 4 | 5 | import ( 6 | "errors" 7 | 8 | "github.com/urfave/cli/v2" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/cli/common" 11 | ) 12 | 13 | func runCliCommand(opts *common.CLIOpts) *cli.Command { 14 | flags := common.RunFlags(opts, false) 15 | flags = append(flags, common.EnvFileAndTemplateFlags(opts, false)...) 16 | 17 | return &cli.Command{ 18 | Name: "run", 19 | Usage: opts.ExecTemplate("Run {{.ProductName}} in normal mode against a specified config file"), 20 | Flags: flags, 21 | Before: func(c *cli.Context) error { 22 | return common.PreApplyEnvFilesAndTemplates(c, opts) 23 | }, 24 | Description: opts.ExecTemplate(` 25 | Run a {{.ProductName}} config. 26 | 27 | {{.BinaryName}} run ./foo.yaml`)[1:], 28 | Action: func(c *cli.Context) error { 29 | if c.Args().Len() > 0 { 30 | if c.Args().Len() > 1 || opts.RootFlags.Config != "" { 31 | return errors.New("a maximum of one config must be specified with the run command") 32 | } 33 | opts.RootFlags.Config = c.Args().First() 34 | } 35 | return common.RunService(c, opts, false) 36 | }, 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal/cli/studio/cli.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package studio 4 | 5 | import ( 6 | "github.com/urfave/cli/v2" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/cli/common" 9 | ) 10 | 11 | // CliCommand is a cli.Command definition for interacting with Benthos studio. 12 | func CliCommand(cliOpts *common.CLIOpts) *cli.Command { 13 | flags := []cli.Flag{ 14 | &cli.StringFlag{ 15 | Name: "endpoint", 16 | Aliases: []string{"e"}, 17 | Value: "https://studio.benthos.dev", 18 | Usage: "Specify the URL of the Benthos studio server to connect to.", 19 | }, 20 | } 21 | 22 | return &cli.Command{ 23 | Name: "studio", 24 | Usage: "Interact with Benthos studio (https://studio.benthos.dev)", 25 | Flags: flags, 26 | Hidden: true, 27 | Description: ` 28 | EXPERIMENTAL: This subcommand is experimental and therefore are subject to 29 | change outside of major version releases.`[1:], 30 | Subcommands: []*cli.Command{ 31 | syncSchemaCommand(cliOpts), 32 | pullCommand(cliOpts), 33 | }, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/cli/studio/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package studio 4 | 5 | import ( 6 | "sync/atomic" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/log" 9 | ) 10 | 11 | var _ log.Modular = &hotSwapLogger{} 12 | 13 | type hotSwapLogger struct { 14 | lPtr atomic.Pointer[log.Modular] 15 | } 16 | 17 | func (h *hotSwapLogger) swap(l log.Modular) { 18 | h.lPtr.Store(&l) 19 | } 20 | 21 | func (h *hotSwapLogger) WithFields(fields map[string]string) log.Modular { 22 | return (*h.lPtr.Load()).WithFields(fields) 23 | } 24 | 25 | func (h *hotSwapLogger) With(keyValues ...any) log.Modular { 26 | return (*h.lPtr.Load()).With(keyValues...) 27 | } 28 | 29 | func (h *hotSwapLogger) Fatal(format string, v ...any) { 30 | (*h.lPtr.Load()).Fatal(format, v...) 31 | } 32 | 33 | func (h *hotSwapLogger) Error(format string, v ...any) { 34 | (*h.lPtr.Load()).Error(format, v...) 35 | } 36 | 37 | func (h *hotSwapLogger) Warn(format string, v ...any) { 38 | (*h.lPtr.Load()).Warn(format, v...) 39 | } 40 | 41 | func (h *hotSwapLogger) Info(format string, v ...any) { 42 | (*h.lPtr.Load()).Info(format, v...) 43 | } 44 | 45 | func (h *hotSwapLogger) Debug(format string, v ...any) { 46 | (*h.lPtr.Load()).Debug(format, v...) 47 | } 48 | 49 | func (h *hotSwapLogger) Trace(format string, v ...any) { 50 | (*h.lPtr.Load()).Trace(format, v...) 51 | } 52 | -------------------------------------------------------------------------------- /internal/cli/studio/metrics/observed.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package metrics 4 | 5 | // ObservedInput is a subset of input metrics that we're interested in. 6 | type ObservedInput struct { 7 | Received int64 `json:"received"` 8 | } 9 | 10 | // ObservedOutput is a subset of output metrics that we're interested in. 11 | type ObservedOutput struct { 12 | Sent int64 `json:"sent"` 13 | Error int64 `json:"error"` 14 | } 15 | 16 | // ObservedProcessor is a subset of processor metrics that we're interested in. 17 | type ObservedProcessor struct { 18 | Received int64 `json:"received"` 19 | Sent int64 `json:"sent"` 20 | Error int64 `json:"error"` 21 | } 22 | 23 | // Observed is a subset of typical Benthos metrics collected by streams that 24 | // we're interested in for studios purposes. 25 | type Observed struct { 26 | Input map[string]ObservedInput `json:"input"` 27 | Processor map[string]ObservedProcessor `json:"processor"` 28 | Output map[string]ObservedOutput `json:"output"` 29 | } 30 | 31 | func newObserved() *Observed { 32 | return &Observed{ 33 | Input: map[string]ObservedInput{}, 34 | Processor: map[string]ObservedProcessor{}, 35 | Output: map[string]ObservedOutput{}, 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /internal/cli/template/cli.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package template 4 | 5 | import ( 6 | "github.com/urfave/cli/v2" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/cli/common" 9 | ) 10 | 11 | // CliCommand is a cli.Command definition for interacting with templates. 12 | func CliCommand(opts *common.CLIOpts) *cli.Command { 13 | return &cli.Command{ 14 | Name: "template", 15 | Usage: opts.ExecTemplate("Interact and generate {{.ProductName}} templates"), 16 | Description: opts.ExecTemplate(` 17 | EXPERIMENTAL: This subcommand, and templates in general, are experimental and 18 | therefore are subject to change outside of major version releases. 19 | 20 | Allows linting and generating {{.ProductName}} templates. 21 | 22 | {{.BinaryName}} template lint ./path/to/templates/... 23 | 24 | For more information check out the docs at: 25 | {{.DocumentationURL}}/configuration/templating`)[1:], 26 | Subcommands: []*cli.Command{ 27 | lintCliCommand(opts), 28 | }, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /internal/cli/test/definition.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package test 4 | 5 | import ( 6 | "fmt" 7 | "path/filepath" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/bundle" 10 | "github.com/redpanda-data/benthos/v4/internal/config/test" 11 | "github.com/redpanda-data/benthos/v4/internal/docs" 12 | "github.com/redpanda-data/benthos/v4/internal/filepath/ifs" 13 | "github.com/redpanda-data/benthos/v4/internal/log" 14 | ) 15 | 16 | // Execute the test definition. 17 | func Execute(env *bundle.Environment, confSpec docs.FieldSpecs, cases []test.Case, testFilePath string, resourcesPaths []string, logger log.Modular) ([]CaseFailure, error) { 18 | procsProvider := NewProcessorsProvider(testFilePath, resourcesPaths, confSpec, env, logger) 19 | 20 | dir := filepath.Dir(testFilePath) 21 | 22 | var totalFailures []CaseFailure 23 | for i, c := range cases { 24 | cleanupEnv := setEnvironment(c.Environment) 25 | failures, err := ExecuteFrom(ifs.OS(), dir, c, procsProvider) 26 | if err != nil { 27 | cleanupEnv() 28 | return nil, fmt.Errorf("test case %v failed: %v", i, err) 29 | } 30 | totalFailures = append(totalFailures, failures...) 31 | cleanupEnv() 32 | } 33 | 34 | return totalFailures, nil 35 | } 36 | -------------------------------------------------------------------------------- /internal/component/buffer/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package buffer 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/message" 9 | ) 10 | 11 | // Streamed is an interface implemented by all buffer types that provides stream 12 | // based methods. 13 | type Streamed interface { 14 | // TransactionChan returns a channel used for consuming transactions from 15 | // this type. Every transaction received must be resolved before another 16 | // transaction will be sent. 17 | TransactionChan() <-chan message.Transaction 18 | 19 | // Consume starts the type receiving transactions from a Transactor. 20 | Consume(<-chan message.Transaction) error 21 | 22 | // TriggerStopConsuming instructs the buffer to cut off the producer it is 23 | // consuming from. It will then enter a mode whereby messages can only be 24 | // read, and when the buffer is empty it will shut down. 25 | TriggerStopConsuming() 26 | 27 | // TriggerCloseNow triggers the shut down of this component but should not 28 | // block the calling goroutine. 29 | TriggerCloseNow() 30 | 31 | // WaitForClose is a blocking call to wait until the component has finished 32 | // shutting down and cleaning up resources. 33 | WaitForClose(ctx context.Context) error 34 | } 35 | -------------------------------------------------------------------------------- /internal/component/buffer/memory_buffer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package buffer 4 | 5 | import ( 6 | "context" 7 | "sync" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/component" 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | ) 12 | 13 | type memoryBuffer struct { 14 | messages chan message.Batch 15 | endOfInputChan chan struct{} 16 | closeOnce sync.Once 17 | } 18 | 19 | func newMemoryBuffer(n int) *memoryBuffer { 20 | return &memoryBuffer{ 21 | messages: make(chan message.Batch, n), 22 | endOfInputChan: make(chan struct{}), 23 | } 24 | } 25 | 26 | func (m *memoryBuffer) Read(ctx context.Context) (message.Batch, AckFunc, error) { 27 | select { 28 | case msg := <-m.messages: 29 | return msg, func(c context.Context, e error) error { 30 | return nil 31 | }, nil 32 | case <-ctx.Done(): 33 | return nil, nil, ctx.Err() 34 | case <-m.endOfInputChan: 35 | // Input has ended, so return ErrEndOfBuffer if our buffer is empty. 36 | select { 37 | case msg := <-m.messages: 38 | return msg, func(c context.Context, e error) error { 39 | // YOLO: Drop messages that are nacked 40 | return nil 41 | }, nil 42 | default: 43 | return nil, nil, component.ErrTypeClosed 44 | } 45 | } 46 | } 47 | 48 | func (m *memoryBuffer) Write(ctx context.Context, msg message.Batch, aFn AckFunc) error { 49 | select { 50 | case m.messages <- msg: 51 | if err := aFn(context.Background(), nil); err != nil { 52 | return err 53 | } 54 | case <-ctx.Done(): 55 | return ctx.Err() 56 | } 57 | return nil 58 | } 59 | 60 | func (m *memoryBuffer) EndOfInput() { 61 | m.closeOnce.Do(func() { 62 | close(m.endOfInputChan) 63 | }) 64 | } 65 | 66 | func (m *memoryBuffer) Close(ctx context.Context) error { 67 | // Nothing to clean up 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /internal/component/cache/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package cache 4 | 5 | import ( 6 | "context" 7 | "time" 8 | ) 9 | 10 | // TTLItem contains a value to cache along with an optional TTL. 11 | type TTLItem struct { 12 | Value []byte 13 | TTL *time.Duration 14 | } 15 | 16 | // V1 Defines a common interface of cache implementations. 17 | type V1 interface { 18 | // Get attempts to locate and return a cached value by its key, returns an 19 | // error if the key does not exist or if the command fails. 20 | Get(ctx context.Context, key string) ([]byte, error) 21 | 22 | // Set attempts to set the value of a key, returns an error if the command 23 | // fails. 24 | Set(ctx context.Context, key string, value []byte, ttl *time.Duration) error 25 | 26 | // SetMulti attempts to set the value of multiple keys, returns an error if 27 | // any of the keys fail. 28 | SetMulti(ctx context.Context, items map[string]TTLItem) error 29 | 30 | // Add attempts to set the value of a key only if the key does not already 31 | // exist, returns an error if the key already exists or if the command 32 | // fails. 33 | Add(ctx context.Context, key string, value []byte, ttl *time.Duration) error 34 | 35 | // Delete attempts to remove a key. Returns an error if a failure occurs. 36 | Delete(ctx context.Context, key string) error 37 | 38 | // Close the component, blocks until either the underlying resources are 39 | // cleaned up or the context is cancelled. Returns an error if the context 40 | // is cancelled. 41 | Close(ctx context.Context) error 42 | } 43 | -------------------------------------------------------------------------------- /internal/component/input/processors/append.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package processors 4 | 5 | import ( 6 | "fmt" 7 | "strconv" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/bundle" 10 | "github.com/redpanda-data/benthos/v4/internal/component/input" 11 | "github.com/redpanda-data/benthos/v4/internal/component/processor" 12 | "github.com/redpanda-data/benthos/v4/internal/pipeline" 13 | ) 14 | 15 | // AppendFromConfig takes a variant arg of pipeline constructor functions and 16 | // returns a new slice of them where the processors of the provided input 17 | // configuration will also be initialized. 18 | func AppendFromConfig(conf input.Config, mgr bundle.NewManagement, pipelines ...processor.PipelineConstructorFunc) []processor.PipelineConstructorFunc { 19 | if len(conf.Processors) > 0 { 20 | pipelines = append([]processor.PipelineConstructorFunc{func() (processor.Pipeline, error) { 21 | processors := make([]processor.V1, len(conf.Processors)) 22 | for j, procConf := range conf.Processors { 23 | newMgr := mgr.IntoPath("processors", strconv.Itoa(j)) 24 | var err error 25 | processors[j], err = newMgr.NewProcessor(procConf) 26 | if err != nil { 27 | return nil, fmt.Errorf("failed to create processor '%v': %v", procConf.Type, err) 28 | } 29 | } 30 | return pipeline.NewProcessor(processors...), nil 31 | }}, pipelines...) 32 | } 33 | return pipelines 34 | } 35 | 36 | // WrapConstructor provides a way to define an input constructor without 37 | // manually initializing processors of the config. 38 | func WrapConstructor(fn func(input.Config, bundle.NewManagement) (input.Streamed, error)) bundle.InputConstructor { 39 | return func(c input.Config, nm bundle.NewManagement) (input.Streamed, error) { 40 | i, err := fn(c, nm) 41 | if err != nil { 42 | return nil, err 43 | } 44 | pcf := AppendFromConfig(c, nm) 45 | return input.WrapWithPipelines(i, pcf...) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/component/metrics/vector_util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package metrics 4 | 5 | type fCounterVec struct { 6 | f func(...string) StatCounter 7 | } 8 | 9 | func (f *fCounterVec) With(labels ...string) StatCounter { 10 | return f.f(labels...) 11 | } 12 | 13 | // FakeCounterVec returns a counter vec implementation that ignores labels. 14 | func FakeCounterVec(f func(...string) StatCounter) StatCounterVec { 15 | return &fCounterVec{ 16 | f: f, 17 | } 18 | } 19 | 20 | //------------------------------------------------------------------------------ 21 | 22 | type fTimerVec struct { 23 | f func(...string) StatTimer 24 | } 25 | 26 | func (f *fTimerVec) With(labels ...string) StatTimer { 27 | return f.f(labels...) 28 | } 29 | 30 | // FakeTimerVec returns a timer vec implementation that ignores labels. 31 | func FakeTimerVec(f func(...string) StatTimer) StatTimerVec { 32 | return &fTimerVec{ 33 | f: f, 34 | } 35 | } 36 | 37 | //------------------------------------------------------------------------------ 38 | 39 | type fGaugeVec struct { 40 | f func(...string) StatGauge 41 | } 42 | 43 | func (f *fGaugeVec) With(labels ...string) StatGauge { 44 | return f.f(labels...) 45 | } 46 | 47 | // FakeGaugeVec returns a gauge vec implementation that ignores labels. 48 | func FakeGaugeVec(f func(...string) StatGauge) StatGaugeVec { 49 | return &fGaugeVec{ 50 | f: f, 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /internal/component/observability.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package component 4 | 5 | import ( 6 | "go.opentelemetry.io/otel/trace" 7 | "go.opentelemetry.io/otel/trace/noop" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/component/metrics" 10 | "github.com/redpanda-data/benthos/v4/internal/log" 11 | ) 12 | 13 | // Observability is an interface implemented by components that provide a range 14 | // of observability APIs to components. This is primarily done the service-wide 15 | // managers. 16 | type Observability interface { 17 | Metrics() metrics.Type 18 | Logger() log.Modular 19 | Tracer() trace.TracerProvider 20 | Path() []string 21 | Label() string 22 | } 23 | 24 | type mockObs struct{} 25 | 26 | func (m mockObs) Metrics() metrics.Type { 27 | return metrics.Noop() 28 | } 29 | 30 | func (m mockObs) Logger() log.Modular { 31 | return log.Noop() 32 | } 33 | 34 | func (m mockObs) Tracer() trace.TracerProvider { 35 | return noop.NewTracerProvider() 36 | } 37 | 38 | func (m mockObs) Path() []string { 39 | return nil 40 | } 41 | 42 | func (m mockObs) Label() string { 43 | return "" 44 | } 45 | 46 | // NoopObservability returns an implementation of Observability that does 47 | // nothing. 48 | func NoopObservability() Observability { 49 | return mockObs{} 50 | } 51 | -------------------------------------------------------------------------------- /internal/component/output/batched_send.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package output 4 | 5 | import ( 6 | "errors" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/batch" 9 | "github.com/redpanda-data/benthos/v4/internal/component" 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | ) 12 | 13 | // Returns true if the error should break a batch send loop. 14 | func sendErrIsFatal(err error) bool { 15 | if errors.Is(err, component.ErrTypeClosed) { 16 | return true 17 | } 18 | if errors.Is(err, component.ErrNotConnected) { 19 | return true 20 | } 21 | if errors.Is(err, component.ErrTimeout) { 22 | return true 23 | } 24 | return false 25 | } 26 | 27 | // IterateBatchedSend executes a closure fn on each message of a batch, where 28 | // the closure is expected to attempt a send and return an error. If an error is 29 | // returned then it is added to a batch error in order to support index specific 30 | // error handling. 31 | // 32 | // However, if a fatal error is returned such as a connection loss or shut down 33 | // then it is returned immediately. 34 | func IterateBatchedSend(msg message.Batch, fn func(int, *message.Part) error) error { 35 | if msg.Len() == 1 { 36 | return fn(0, msg.Get(0)) 37 | } 38 | var batchErr *batch.Error 39 | if err := msg.Iter(func(i int, p *message.Part) error { 40 | tmpErr := fn(i, p) 41 | if tmpErr != nil { 42 | if sendErrIsFatal(tmpErr) { 43 | return tmpErr 44 | } 45 | if batchErr == nil { 46 | batchErr = batch.NewError(msg, tmpErr) 47 | } 48 | batchErr.Failed(i, tmpErr) 49 | } 50 | return nil 51 | }); err != nil { 52 | return err 53 | } 54 | if batchErr != nil { 55 | return batchErr 56 | } 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /internal/component/output/processors/append.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package processors 4 | 5 | import ( 6 | "strconv" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/bundle" 9 | "github.com/redpanda-data/benthos/v4/internal/component/output" 10 | "github.com/redpanda-data/benthos/v4/internal/component/processor" 11 | "github.com/redpanda-data/benthos/v4/internal/pipeline" 12 | ) 13 | 14 | // AppendFromConfig takes a variant arg of pipeline constructor functions and 15 | // returns a new slice of them where the processors of the provided output 16 | // configuration will also be initialized. 17 | func AppendFromConfig(conf output.Config, mgr bundle.NewManagement, pipelines ...processor.PipelineConstructorFunc) []processor.PipelineConstructorFunc { 18 | if len(conf.Processors) > 0 { 19 | pipelines = append(pipelines, []processor.PipelineConstructorFunc{func() (processor.Pipeline, error) { 20 | processors := make([]processor.V1, len(conf.Processors)) 21 | for j, procConf := range conf.Processors { 22 | var err error 23 | pMgr := mgr.IntoPath("processors", strconv.Itoa(j)) 24 | processors[j], err = pMgr.NewProcessor(procConf) 25 | if err != nil { 26 | return nil, err 27 | } 28 | } 29 | return pipeline.NewProcessor(processors...), nil 30 | }}...) 31 | } 32 | return pipelines 33 | } 34 | 35 | // WrapConstructor provides a way to define an output constructor without 36 | // manually initializing processors of the config. 37 | func WrapConstructor(fn func(output.Config, bundle.NewManagement) (output.Streamed, error)) bundle.OutputConstructor { 38 | return func(c output.Config, nm bundle.NewManagement, pcf ...processor.PipelineConstructorFunc) (output.Streamed, error) { 39 | o, err := fn(c, nm) 40 | if err != nil { 41 | return nil, err 42 | } 43 | pcf = AppendFromConfig(c, nm, pcf...) 44 | return output.WrapWithPipelines(o, pcf...) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /internal/component/processor/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package processor 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/message" 7 | "github.com/redpanda-data/benthos/v4/internal/tracing" 8 | ) 9 | 10 | // MarkErr marks a message part as having failed. This includes modifying 11 | // metadata to contain this error as well as adding the error to a tracing span 12 | // if the message has one. 13 | func MarkErr(part *message.Part, span *tracing.Span, err error) { 14 | if err == nil { 15 | return 16 | } 17 | if part != nil { 18 | part.ErrorSet(err) 19 | } 20 | if span == nil && part != nil { 21 | span = tracing.GetActiveSpan(part) 22 | } 23 | if span != nil { 24 | span.SetTag("error", "true") 25 | span.LogKV( 26 | "event", "error", 27 | "type", err.Error(), 28 | ) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /internal/component/ratelimit/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package ratelimit 4 | 5 | import ( 6 | "context" 7 | "time" 8 | ) 9 | 10 | // V1 is a common interface implemented by rate limits. 11 | type V1 interface { 12 | // Access the rate limited resource. Returns a duration or an error if the 13 | // rate limit check fails. The returned duration is either zero (meaning the 14 | // resource may be accessed) or a reasonable length of time to wait before 15 | // requesting again. 16 | Access(ctx context.Context) (time.Duration, error) 17 | 18 | // Close the component, blocks until either the underlying resources are 19 | // cleaned up or the context is cancelled. Returns an error if the context 20 | // is cancelled. 21 | Close(ctx context.Context) error 22 | } 23 | -------------------------------------------------------------------------------- /internal/component/ratelimit/rate_limit_metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package ratelimit 4 | 5 | import ( 6 | "context" 7 | "time" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/component/metrics" 10 | ) 11 | 12 | type metricsRateLimit struct { 13 | r V1 14 | 15 | mChecked metrics.StatCounter 16 | mLimited metrics.StatCounter 17 | mErr metrics.StatCounter 18 | } 19 | 20 | // MetricsForRateLimit wraps a ratelimit.V2 with a struct that implements 21 | // types.RateLimit. 22 | func MetricsForRateLimit(r V1, stats metrics.Type) V1 { 23 | return &metricsRateLimit{ 24 | r: r, 25 | 26 | mChecked: stats.GetCounter("rate_limit_checked"), 27 | mLimited: stats.GetCounter("rate_limit_triggered"), 28 | mErr: stats.GetCounter("rate_limit_error"), 29 | } 30 | } 31 | 32 | func (r *metricsRateLimit) Access(ctx context.Context) (time.Duration, error) { 33 | r.mChecked.Incr(1) 34 | tout, err := r.r.Access(ctx) 35 | if err != nil { 36 | r.mErr.Incr(1) 37 | } else if tout > 0 { 38 | r.mLimited.Incr(1) 39 | } 40 | return tout, err 41 | } 42 | 43 | func (r *metricsRateLimit) Close(ctx context.Context) error { 44 | return r.r.Close(ctx) 45 | } 46 | -------------------------------------------------------------------------------- /internal/component/ratelimit/rate_limit_metrics_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package ratelimit 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/redpanda-data/benthos/v4/internal/component/metrics" 13 | ) 14 | 15 | type closableRateLimit struct { 16 | closed bool 17 | } 18 | 19 | func (c *closableRateLimit) Access(ctx context.Context) (time.Duration, error) { 20 | return 0, nil 21 | } 22 | 23 | func (c *closableRateLimit) Close(ctx context.Context) error { 24 | c.closed = true 25 | return nil 26 | } 27 | 28 | func TestRateLimitAirGapShutdown(t *testing.T) { 29 | rl := &closableRateLimit{} 30 | agrl := MetricsForRateLimit(rl, metrics.Noop()) 31 | 32 | err := agrl.Close(t.Context()) 33 | assert.NoError(t, err) 34 | assert.True(t, rl.closed) 35 | } 36 | -------------------------------------------------------------------------------- /internal/component/scanner/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package scanner 4 | 5 | import ( 6 | "fmt" 7 | 8 | "gopkg.in/yaml.v3" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/docs" 11 | ) 12 | 13 | // Config is the all encompassing configuration struct for all scanner types. 14 | type Config struct { 15 | Type string 16 | Plugin any 17 | } 18 | 19 | // FromAny returns a scanner config from a parsed config, yaml node or map. 20 | func FromAny(prov docs.Provider, value any) (conf Config, err error) { 21 | switch t := value.(type) { 22 | case Config: 23 | return t, nil 24 | case *yaml.Node: 25 | return fromYAML(prov, t) 26 | case map[string]any: 27 | return fromMap(prov, t) 28 | } 29 | err = fmt.Errorf("unexpected value, expected object, got %T", value) 30 | return 31 | } 32 | 33 | func fromMap(prov docs.Provider, value map[string]any) (conf Config, err error) { 34 | if conf.Type, _, err = docs.GetInferenceCandidateFromMap(prov, docs.TypeScanner, value); err != nil { 35 | err = docs.NewLintError(0, docs.LintComponentNotFound, err) 36 | return 37 | } 38 | 39 | if p, exists := value[conf.Type]; exists { 40 | conf.Plugin = p 41 | } else if p, exists := value["plugin"]; exists { 42 | conf.Plugin = p 43 | } 44 | return 45 | } 46 | 47 | func fromYAML(prov docs.Provider, value *yaml.Node) (conf Config, err error) { 48 | if conf.Type, _, err = docs.GetInferenceCandidateFromYAML(prov, docs.TypeScanner, value); err != nil { 49 | err = docs.NewLintError(value.Line, docs.LintComponentNotFound, err) 50 | return 51 | } 52 | 53 | pluginNode, err := docs.GetPluginConfigYAML(conf.Type, value) 54 | if err != nil { 55 | err = docs.NewLintError(value.Line, docs.LintFailedRead, err) 56 | return 57 | } 58 | 59 | conf.Plugin = &pluginNode 60 | return 61 | } 62 | -------------------------------------------------------------------------------- /internal/component/scanner/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package scanner 4 | 5 | import ( 6 | "context" 7 | "io" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/message" 10 | ) 11 | 12 | // AckFn is a function provided to a scanner that it should call once the 13 | // derived io.ReadCloser is fully consumed. 14 | type AckFn func(context.Context, error) error 15 | 16 | // Scanner is an interface implemented by all scanner implementations once a 17 | // creator has instantiated it on a byte stream. 18 | type Scanner interface { 19 | Next(context.Context) (message.Batch, AckFn, error) 20 | Close(context.Context) error 21 | } 22 | 23 | // SourceDetails contains exclusively optional information which could be used 24 | // by codec implementations in order to determine the underlying data format. 25 | type SourceDetails struct { 26 | Name string 27 | } 28 | 29 | // Creator is an interface implemented by all scanners, which allows components 30 | // to construct a scanner from an unbounded io.ReadCloser. 31 | type Creator interface { 32 | Create(rdr io.ReadCloser, aFn AckFn, details SourceDetails) (Scanner, error) 33 | Close(context.Context) error 34 | } 35 | -------------------------------------------------------------------------------- /internal/config/test/docs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package test 4 | 5 | import ( 6 | "fmt" 7 | 8 | "gopkg.in/yaml.v3" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/docs" 11 | ) 12 | 13 | const fieldTests = "tests" 14 | 15 | // ConfigSpec returns a configuration spec for a template. 16 | func ConfigSpec() docs.FieldSpec { 17 | return docs.FieldObject(fieldTests, "A list of one or more unit tests to execute.").Array().WithChildren(caseFields()...).Optional() 18 | } 19 | 20 | // FromAny returns a Case slice from a yaml node or parsed config. 21 | func FromAny(v any) ([]Case, error) { 22 | if t, ok := v.(*yaml.Node); ok { 23 | var tmp struct { 24 | Tests []yaml.Node 25 | } 26 | if err := t.Decode(&tmp); err != nil { 27 | return nil, err 28 | } 29 | var cases []Case 30 | for i, v := range tmp.Tests { 31 | pConf, err := caseFields().ParsedConfigFromAny(&v) 32 | if err != nil { 33 | return nil, fmt.Errorf("%v: %w", i, err) 34 | } 35 | c, err := CaseFromParsed(pConf) 36 | if err != nil { 37 | return nil, fmt.Errorf("%v: %w", i, err) 38 | } 39 | cases = append(cases, c) 40 | } 41 | return cases, nil 42 | } 43 | 44 | pConf, err := ConfigSpec().ParsedConfigFromAny(v) 45 | if err != nil { 46 | return nil, err 47 | } 48 | return FromParsed(pConf) 49 | } 50 | 51 | // FromParsed extracts a Case slice from a parsed config. 52 | func FromParsed(pConf *docs.ParsedConfig) ([]Case, error) { 53 | if !pConf.Contains(fieldTests) { 54 | return nil, nil 55 | } 56 | 57 | oList, err := pConf.FieldObjectList(fieldTests) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | var cases []Case 63 | for i, pc := range oList { 64 | c, err := CaseFromParsed(pc) 65 | if err != nil { 66 | return nil, fmt.Errorf("%v: %w", i, err) 67 | } 68 | cases = append(cases, c) 69 | } 70 | return cases, nil 71 | } 72 | -------------------------------------------------------------------------------- /internal/config/watcher_wasm.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | //go:build wasm 4 | 5 | package config 6 | 7 | import ( 8 | "errors" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/bundle" 11 | ) 12 | 13 | // BeginFileWatching does nothing in WASM builds as it is not supported. Sorry! 14 | func (r *Reader) BeginFileWatching(mgr bundle.NewManagement, strict bool) error { 15 | return errors.New("file watching is disabled in WASM builds") 16 | } 17 | 18 | // noReread is a no-op in WASM builds as the file watcher is not supported. 19 | func noReread(err error) error { 20 | return err 21 | } 22 | -------------------------------------------------------------------------------- /internal/cuegen/README.md: -------------------------------------------------------------------------------- 1 | # CUE AST tips 2 | 3 | ## Adding optional fields to structs 4 | 5 | Code: 6 | 7 | ```go 8 | requiredField := &ast.Field { 9 | Label: ast.NewIdent("username"), 10 | Value: ast.NewIdent("string"), 11 | } 12 | optionalField := &ast.Field { 13 | Label: ast.NewIdent("age"), 14 | Value: ast.NewIdent("uint"), 15 | } 16 | 17 | ast.NewStruct( 18 | requiredField, 19 | 20 | optionalField.Label, 21 | token.OPTION, 22 | optionalField.Value, 23 | ) 24 | ``` 25 | 26 | ## Given a struct, create a disjunction 27 | 28 | In other words, given: 29 | 30 | ```cue 31 | #AllInputs: { 32 | http_client: {url: string} 33 | generate: {mapping: string} 34 | file: {paths: [...string]} 35 | } 36 | ``` 37 | 38 | Generate a type `#Input` that conforms to: 39 | 40 | ```cue 41 | #Input: {http_client: {url: string}} | {generate: {mapping: string}} | {file: {paths: [...string]}} 42 | ``` 43 | 44 | This can be done using the `or` built-in function in Cue and field comprehension: 45 | 46 | ```cue 47 | #Input: or([for name, config in #AllInputs {(name): config}]) 48 | ``` 49 | 50 | Expressing that using the `ast` package looks like this: 51 | 52 | Code: 53 | 54 | ```go 55 | collectionIdent := ast.NewIdent("#AllInputs") 56 | disjunctionIdent := ast.NewIdent("#Input") 57 | 58 | &ast.Field{ 59 | Label: disjunctionIdent, 60 | Value: ast.NewCall(ast.NewIdent("or"), ast.NewList(&ast.Comprehension{ 61 | Clauses: []ast.Clause{ 62 | &ast.ForClause{ 63 | Key: ast.NewIdent("name"), 64 | Value: ast.NewIdent("config"), 65 | Source: collectionIdent, 66 | }, 67 | }, 68 | Value: ast.NewStruct(&ast.Field{ 69 | Label: interpolateIdent(ast.NewIdent("name")), 70 | Value: ast.NewIdent("config"), 71 | }), 72 | })), 73 | }, 74 | ``` 75 | -------------------------------------------------------------------------------- /internal/cuegen/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package cuegen 4 | 5 | import ( 6 | "cuelang.org/go/cue/ast" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/config/schema" 9 | ) 10 | 11 | func doConfig(sch schema.Full) ([]ast.Decl, error) { 12 | members, err := doFieldSpecs(sch.Config) 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | return []ast.Decl{ 18 | &ast.Field{ 19 | Label: identConfig, 20 | Value: ast.NewStruct(members...), 21 | }, 22 | }, nil 23 | } 24 | -------------------------------------------------------------------------------- /internal/cuegen/identifiers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package cuegen 4 | 5 | import "cuelang.org/go/cue/ast" 6 | 7 | var ( 8 | identConfig = ast.NewIdent("#Config") 9 | 10 | identInputDisjunction = ast.NewIdent("#Input") 11 | identInputCollection = ast.NewIdent("#AllInputs") 12 | 13 | identOutputDisjunction = ast.NewIdent("#Output") 14 | identOutputCollection = ast.NewIdent("#AllOutputs") 15 | 16 | identProcessorDisjunction = ast.NewIdent("#Processor") 17 | identProcessorCollection = ast.NewIdent("#AllProcessors") 18 | 19 | identRateLimitDisjunction = ast.NewIdent("#RateLimit") 20 | identRateLimitCollection = ast.NewIdent("#AllRateLimits") 21 | 22 | identBufferDisjunction = ast.NewIdent("#Buffer") 23 | identBufferCollection = ast.NewIdent("#AllBuffers") 24 | 25 | identCacheDisjunction = ast.NewIdent("#Cache") 26 | identCacheCollection = ast.NewIdent("#AllCaches") 27 | 28 | identMetricDisjunction = ast.NewIdent("#Metric") 29 | identMetricCollection = ast.NewIdent("#AllMetrics") 30 | 31 | identTracerDisjunction = ast.NewIdent("#Tracer") 32 | identTracerCollection = ast.NewIdent("#AllTracers") 33 | 34 | identScannerDisjunction = ast.NewIdent("#Scanner") 35 | identScannerCollection = ast.NewIdent("#AllScanners") 36 | ) 37 | -------------------------------------------------------------------------------- /internal/docs/bloblang.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package docs 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/public/bloblang" 7 | ) 8 | 9 | // LintBloblangMapping is function for linting a config field expected to be a 10 | // bloblang mapping. 11 | func LintBloblangMapping(ctx LintContext, line, col int, v any) []Lint { 12 | str, ok := v.(string) 13 | if !ok { 14 | return nil 15 | } 16 | if str == "" { 17 | return nil 18 | } 19 | _, err := ctx.conf.BloblangEnv.Parse(str) 20 | if err == nil { 21 | return nil 22 | } 23 | if mErr, ok := err.(*bloblang.ParseError); ok { 24 | lint := NewLintError(line+mErr.Line-1, LintBadBloblang, mErr) 25 | lint.Column = col + mErr.Column 26 | return []Lint{lint} 27 | } 28 | return []Lint{NewLintError(line, LintBadBloblang, err)} 29 | } 30 | 31 | // LintBloblangField is function for linting a config field expected to be an 32 | // interpolation string. 33 | func LintBloblangField(ctx LintContext, line, col int, v any) []Lint { 34 | str, ok := v.(string) 35 | if !ok { 36 | return nil 37 | } 38 | if str == "" { 39 | return nil 40 | } 41 | err := ctx.conf.BloblangEnv.CheckInterpolatedString(str) 42 | if err == nil { 43 | return nil 44 | } 45 | if mErr, ok := err.(*bloblang.ParseError); ok { 46 | lint := NewLintError(line+mErr.Line-1, LintBadBloblang, mErr) 47 | lint.Column = col + mErr.Column 48 | return []Lint{lint} 49 | } 50 | return []Lint{NewLintError(line, LintBadBloblang, err)} 51 | } 52 | -------------------------------------------------------------------------------- /internal/docs/field_template.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package docs 4 | 5 | // DeprecatedFieldsTemplate is an old (now unused) template for generating 6 | // documentation. It has been replace with public methods for exporting template 7 | // data, allowing you to use whichever template suits your needs. 8 | // TODO: V5 Remove this 9 | func DeprecatedFieldsTemplate(lintableExamples bool) string { 10 | // Use trailing whitespace below to render line breaks in Asciidoc 11 | return `{{define "field_docs" -}} 12 | {{range $i, $field := .Fields -}} 13 | === ` + "`{{$field.FullName}}`" + ` 14 | 15 | {{$field.Description}} 16 | {{if $field.IsSecret -}} 17 | 18 | [CAUTION] 19 | ==== 20 | This field contains sensitive information that usually shouldn't be added to a config directly, read our xref:configuration:secrets.adoc[secrets page for more info]. 21 | ==== 22 | 23 | {{end -}} 24 | {{if $field.IsInterpolated -}} 25 | This field supports xref:configuration:interpolation.adoc#bloblang-queries[interpolation functions]. 26 | {{end}} 27 | 28 | *Type*: ` + "`{{$field.Type}}`" + ` 29 | 30 | {{if gt (len $field.DefaultMarshalled) 0}}*Default*: ` + "`{{$field.DefaultMarshalled}}`" + ` 31 | {{end -}} 32 | {{if gt (len $field.Version) 0}}Requires version {{$field.Version}} or newer 33 | {{end -}} 34 | {{if gt (len $field.AnnotatedOptions) 0}} 35 | |=== 36 | | Option | Summary 37 | 38 | {{range $j, $option := $field.AnnotatedOptions -}} 39 | | ` + "`{{index $option 0}}`" + ` 40 | | {{index $option 1}} 41 | {{end}} 42 | |=== 43 | {{else if gt (len $field.Options) 0}} 44 | Options: 45 | {{range $j, $option := $field.Options -}} 46 | {{if ne $j 0}}, {{end}}` + "`{{$option}}`" + ` 47 | {{end}}. 48 | {{end}} 49 | {{if gt (len $field.Examples) 0 -}} 50 | ` + "```yml" + ` 51 | # Examples 52 | 53 | {{range $j, $example := $field.ExamplesMarshalled -}} 54 | {{if ne $j 0}} 55 | {{end}}{{$example}}{{end -}} 56 | ` + "```" + ` 57 | 58 | {{end -}} 59 | {{end -}} 60 | {{end -}}` 61 | } 62 | -------------------------------------------------------------------------------- /internal/docs/interop/interop.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package interop 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/docs" 7 | "github.com/redpanda-data/benthos/v4/public/service" 8 | ) 9 | 10 | // Unwrap a public *service.ConfigField type into an internal docs.FieldSpec. 11 | // This is useful in situations where a config spec needs to be shared by new 12 | // components built by the service package at the same time as older components 13 | // using the internal APIs directly. 14 | // 15 | // In these cases we want the canonical spec to be made with the service package 16 | // but still extract a docs.FieldSpec from it. 17 | func Unwrap(f *service.ConfigField) docs.FieldSpec { 18 | return f.XUnwrapper().(interface { 19 | Unwrap() docs.FieldSpec 20 | }).Unwrap() 21 | } 22 | -------------------------------------------------------------------------------- /internal/docs/metrics_mapping.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package docs 4 | 5 | // MetricsMappingFieldSpec is a field spec that describes a Bloblang mapping for 6 | // renaming metrics. 7 | func MetricsMappingFieldSpec(name string) FieldSpec { 8 | examples := []any{ 9 | `this.replace("input", "source").replace("output", "sink")`, 10 | `root = if ![ 11 | "input_received", 12 | "input_latency", 13 | "output_sent" 14 | ].contains(this) { deleted() }`, 15 | } 16 | summary := "An optional xref:guides:bloblang/about.adoc[Bloblang mapping] that allows you to rename or prevent certain metrics paths from being exported. For more information check out the xref:components:metrics/about.adoc#metric-mapping[metrics documentation]. When metric paths are created, renamed and dropped a trace log is written, enabling TRACE level logging is therefore a good way to diagnose path mappings." 17 | return FieldBloblang(name, summary, examples...).HasDefault("") 18 | } 19 | -------------------------------------------------------------------------------- /internal/docs/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package docs provides useful functions for creating documentation from 4 | // Benthos components 5 | package docs 6 | -------------------------------------------------------------------------------- /internal/filepath/ifs/os_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package ifs 4 | 5 | import ( 6 | "errors" 7 | "io/fs" 8 | "testing" 9 | "testing/fstest" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | type testFS struct { 15 | fstest.MapFS 16 | } 17 | 18 | func (t testFS) MkdirAll(path string, perm fs.FileMode) error { 19 | return errors.New("not implemented") 20 | } 21 | 22 | func (t testFS) OpenFile(name string, flag int, perm fs.FileMode) (fs.File, error) { 23 | return nil, errors.New("not implemented") 24 | } 25 | 26 | func (t testFS) Remove(name string) error { 27 | return errors.New("not implemented") 28 | } 29 | 30 | func TestOSAccess(t *testing.T) { 31 | var fs FS = testFS{} 32 | 33 | require.False(t, IsOS(fs)) 34 | 35 | fs = OS() 36 | 37 | require.True(t, IsOS(fs)) 38 | } 39 | -------------------------------------------------------------------------------- /internal/httpclient/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package httpclient 4 | 5 | import ( 6 | "fmt" 7 | "strings" 8 | ) 9 | 10 | // ErrUnexpectedHTTPRes is an error returned when an HTTP request returned an 11 | // unexpected response. 12 | type ErrUnexpectedHTTPRes struct { 13 | Code int 14 | S string 15 | Body []byte 16 | } 17 | 18 | // Error returns the Error string. 19 | func (e ErrUnexpectedHTTPRes) Error() string { 20 | body := strings.ReplaceAll(string(e.Body), "\n", "") 21 | return fmt.Sprintf("HTTP request returned unexpected response code (%v): %v, Error: %v", e.Code, e.S, body) 22 | } 23 | -------------------------------------------------------------------------------- /internal/httpclient/errors_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package httpclient 4 | 5 | import "testing" 6 | 7 | func TestHTTPError(t *testing.T) { 8 | err := ErrUnexpectedHTTPRes{ 9 | Code: 0, 10 | S: "test str", 11 | Body: []byte("test body str"), 12 | } 13 | 14 | exp, act := `HTTP request returned unexpected response code (0): test str, Error: test body str`, err.Error() 15 | if exp != act { 16 | t.Errorf("Wrong Error() from ErrUnexpectedHTTPRes: %v != %v", exp, act) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /internal/httpclient/request_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package httpclient 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/redpanda-data/benthos/v4/public/service" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestSingleMessageHeaders(t *testing.T) { 15 | spec := service.NewConfigSpec().Field(ConfigField("GET", false)) 16 | parsed, err := spec.ParseYAML(` 17 | url: example.com/foo 18 | headers: 19 | "Content-Type": "foo" 20 | metadata: 21 | include_prefixes: [ "more_" ] 22 | `, nil) 23 | require.NoError(t, err) 24 | 25 | oldConf, err := ConfigFromParsed(parsed) 26 | require.NoError(t, err) 27 | 28 | reqCreator, err := RequestCreatorFromOldConfig(oldConf, service.MockResources()) 29 | require.NoError(t, err) 30 | 31 | part := service.NewMessage([]byte("hello world")) 32 | part.MetaSetMut("more_bar", "barvalue") 33 | part.MetaSetMut("ignore_baz", "bazvalue") 34 | 35 | b := service.MessageBatch{part} 36 | 37 | req, err := reqCreator.Create(b) 38 | require.NoError(t, err) 39 | 40 | assert.Equal(t, []string{"foo"}, req.Header.Values("Content-Type")) 41 | assert.Equal(t, []string{"barvalue"}, req.Header.Values("more_bar")) 42 | assert.Equal(t, []string(nil), req.Header.Values("ignore_baz")) 43 | } 44 | -------------------------------------------------------------------------------- /internal/impl/io/cache_file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package io 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/public/service" 12 | ) 13 | 14 | func TestFileCache(t *testing.T) { 15 | dir := t.TempDir() 16 | 17 | tCtx := t.Context() 18 | c := newFileCache(dir, service.MockResources()) 19 | 20 | _, err := c.Get(tCtx, "foo") 21 | assert.Equal(t, service.ErrKeyNotFound, err) 22 | 23 | require.NoError(t, c.Set(tCtx, "foo", []byte("1"), nil)) 24 | 25 | act, err := c.Get(tCtx, "foo") 26 | require.NoError(t, err) 27 | assert.Equal(t, "1", string(act)) 28 | 29 | require.NoError(t, c.Add(tCtx, "bar", []byte("2"), nil)) 30 | 31 | act, err = c.Get(tCtx, "bar") 32 | require.NoError(t, err) 33 | assert.Equal(t, "2", string(act)) 34 | 35 | assert.Equal(t, service.ErrKeyAlreadyExists, c.Add(tCtx, "foo", []byte("2"), nil)) 36 | 37 | require.NoError(t, c.Set(tCtx, "foo", []byte("3"), nil)) 38 | 39 | act, err = c.Get(tCtx, "foo") 40 | require.NoError(t, err) 41 | assert.Equal(t, "3", string(act)) 42 | 43 | require.NoError(t, c.Delete(tCtx, "foo")) 44 | 45 | _, err = c.Get(tCtx, "foo") 46 | assert.Equal(t, service.ErrKeyNotFound, err) 47 | } 48 | -------------------------------------------------------------------------------- /internal/impl/io/input_stdin_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package io_test 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/redpanda-data/benthos/v4/internal/component/input" 13 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 14 | ) 15 | 16 | func TestSTDINClose(t *testing.T) { 17 | conf := input.NewConfig() 18 | conf.Type = "stdin" 19 | s, err := mock.NewManager().NewInput(conf) 20 | require.NoError(t, err) 21 | 22 | ctx, done := context.WithTimeout(t.Context(), time.Second*20) 23 | defer done() 24 | 25 | s.TriggerStopConsuming() 26 | require.NoError(t, s.WaitForClose(ctx)) 27 | } 28 | -------------------------------------------------------------------------------- /internal/impl/io/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package io contains component implementations that have a small dependency 4 | // footprint (mostly standard library) and interact with external systems via 5 | // the filesystem and/or network sockets. 6 | package io 7 | -------------------------------------------------------------------------------- /internal/impl/pure/bloblang_encoding_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "fmt" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/redpanda-data/benthos/v4/public/bloblang" 13 | ) 14 | 15 | func TestCompressionDecompression(t *testing.T) { 16 | seen := map[string]struct{}{} 17 | for _, alg := range []string{`flate`, `gzip`, `pgzip`, `lz4`, `snappy`, `zlib`} { 18 | exec, err := bloblang.Parse(fmt.Sprintf(`root = this.compress(algorithm: "%v")`, alg)) 19 | require.NoError(t, err) 20 | 21 | input := []byte("hello world this is a really long string") 22 | 23 | compressed, err := exec.Query(input) 24 | require.NoError(t, err) 25 | 26 | compressedBytes, ok := compressed.([]byte) 27 | require.True(t, ok) 28 | 29 | _, exists := seen[string(compressedBytes)] 30 | require.False(t, exists) 31 | seen[string(compressedBytes)] = struct{}{} 32 | 33 | assert.NotEqual(t, input, compressed) 34 | assert.Greater(t, len(compressedBytes), 1) 35 | 36 | exec, err = bloblang.Parse(fmt.Sprintf(`root = this.decompress(algorithm: "%v")`, alg)) 37 | require.NoError(t, err) 38 | 39 | decompressed, err := exec.Query(compressed) 40 | require.NoError(t, err) 41 | 42 | assert.Equal(t, input, decompressed) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/impl/pure/bloblang_string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "fmt" 7 | "net/url" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/bloblang/query" 10 | "github.com/redpanda-data/benthos/v4/public/bloblang" 11 | ) 12 | 13 | func init() { 14 | bloblang.MustRegisterMethodV2("parse_form_url_encoded", 15 | bloblang.NewPluginSpec(). 16 | Category(query.MethodCategoryParsing). 17 | Description(`Attempts to parse a url-encoded query string (from an x-www-form-urlencoded request body) and returns a structured result.`). 18 | Example("", `root.values = this.body.parse_form_url_encoded()`, 19 | [2]string{ 20 | `{"body":"noise=meow&animal=cat&fur=orange&fur=fluffy"}`, 21 | `{"values":{"animal":"cat","fur":["orange","fluffy"],"noise":"meow"}}`, 22 | }, 23 | ), 24 | func(args *bloblang.ParsedParams) (bloblang.Method, error) { 25 | return bloblang.StringMethod(func(data string) (any, error) { 26 | values, err := url.ParseQuery(data) 27 | if err != nil { 28 | return nil, fmt.Errorf("failed to parse value as url-encoded data: %w", err) 29 | } 30 | return urlValuesToMap(values), nil 31 | }), nil 32 | }) 33 | } 34 | 35 | func urlValuesToMap(values url.Values) map[string]any { 36 | root := make(map[string]any, len(values)) 37 | 38 | for k, v := range values { 39 | if len(v) == 1 { 40 | root[k] = v[0] 41 | } else { 42 | elements := make([]any, 0, len(v)) 43 | for _, e := range v { 44 | elements = append(elements, e) 45 | } 46 | root[k] = elements 47 | } 48 | } 49 | 50 | return root 51 | } 52 | -------------------------------------------------------------------------------- /internal/impl/pure/bloblang_string_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/bloblang/query" 12 | "github.com/redpanda-data/benthos/v4/internal/value" 13 | ) 14 | 15 | func TestParseUrlencoded(t *testing.T) { 16 | testCases := []struct { 17 | name string 18 | method string 19 | target any 20 | args []any 21 | exp any 22 | }{ 23 | { 24 | name: "simple parsing", 25 | method: "parse_form_url_encoded", 26 | target: "username=example", 27 | args: []any{}, 28 | exp: map[string]any{"username": "example"}, 29 | }, 30 | { 31 | name: "parsing multiple values under the same key", 32 | method: "parse_form_url_encoded", 33 | target: "usernames=userA&usernames=userB", 34 | args: []any{}, 35 | exp: map[string]any{"usernames": []any{"userA", "userB"}}, 36 | }, 37 | { 38 | name: "decodes data correctly", 39 | method: "parse_form_url_encoded", 40 | target: "email=example%40email.com", 41 | args: []any{}, 42 | exp: map[string]any{"email": "example@email.com"}, 43 | }, 44 | } 45 | 46 | for _, test := range testCases { 47 | test := test 48 | t.Run(test.name, func(t *testing.T) { 49 | targetClone := value.IClone(test.target) 50 | argsClone := value.IClone(test.args).([]any) 51 | 52 | fn, err := query.InitMethodHelper(test.method, query.NewLiteralFunction("", targetClone), argsClone...) 53 | require.NoError(t, err) 54 | 55 | res, err := fn.Exec(query.FunctionContext{ 56 | Maps: map[string]query.Function{}, 57 | Index: 0, 58 | MsgBatch: nil, 59 | }) 60 | require.NoError(t, err) 61 | 62 | assert.Equal(t, test.exp, res) 63 | assert.Equal(t, test.target, targetClone) 64 | assert.Equal(t, test.args, argsClone) 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/impl/pure/buffer_none.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "errors" 7 | 8 | "github.com/redpanda-data/benthos/v4/public/service" 9 | ) 10 | 11 | func init() { 12 | service.MustRegisterBatchBuffer( 13 | "none", 14 | service.NewConfigSpec(). 15 | Stable(). 16 | Summary(`Do not buffer messages. This is the default and most resilient configuration.`). 17 | Description(`Selecting no buffer means the output layer is directly coupled with the input layer. This is the safest and lowest latency option since acknowledgements from at-least-once protocols can be propagated all the way from the output protocol to the input protocol. 18 | 19 | If the output layer is hit with back pressure it will propagate all the way to the input layer, and further up the data stream. If you need to relieve your pipeline of this back pressure consider using a more robust buffering solution such as Kafka before resorting to alternatives.`). 20 | Field(service.NewObjectField("").Default(map[string]any{})), 21 | func(conf *service.ParsedConfig, mgr *service.Resources) (service.BatchBuffer, error) { 22 | return nil, errors.New("not implemented") 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /internal/impl/pure/cache_integration_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/redpanda-data/benthos/v4/public/service/integration" 9 | ) 10 | 11 | func TestIntegrationMultilevelCache(t *testing.T) { 12 | integration.CheckSkip(t) 13 | 14 | t.Parallel() 15 | 16 | template := ` 17 | cache_resources: 18 | - label: testcache 19 | multilevel: [ first, second ] 20 | - label: first 21 | memory: {} 22 | - label: second 23 | memory: {} 24 | ` 25 | suite := integration.CacheTests( 26 | integration.CacheTestOpenClose(), 27 | integration.CacheTestMissingKey(), 28 | integration.CacheTestDoubleAdd(), 29 | integration.CacheTestDelete(), 30 | integration.CacheTestGetAndSet(50), 31 | ) 32 | suite.Run(t, template) 33 | } 34 | -------------------------------------------------------------------------------- /internal/impl/pure/cache_noop_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/public/service" 12 | ) 13 | 14 | func TestNoopCacheStandard(t *testing.T) { 15 | t.Parallel() 16 | 17 | resources := service.MockResources() 18 | 19 | c := noopMemCache("TestNoopCacheStandard", resources.Logger()) 20 | 21 | err := c.Set(t.Context(), "foo", []byte("bar"), nil) 22 | require.NoError(t, err) 23 | 24 | value, err := c.Get(t.Context(), "foo") 25 | require.EqualError(t, err, "key does not exist") 26 | 27 | assert.Nil(t, value) 28 | } 29 | -------------------------------------------------------------------------------- /internal/impl/pure/extended/zstd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package extended 4 | 5 | import ( 6 | "io" 7 | 8 | "github.com/klauspost/compress/zstd" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/impl/pure" 11 | ) 12 | 13 | var _ = pure.AddKnownCompressionAlgorithm("zstd", pure.KnownCompressionAlgorithm{ 14 | CompressWriter: func(level int, w io.Writer) (io.Writer, error) { 15 | aw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) 16 | if err != nil { 17 | return nil, err 18 | } 19 | return &pure.CombinedWriteCloser{Primary: aw, Sink: w}, nil 20 | }, 21 | DecompressReader: func(r io.Reader) (io.Reader, error) { 22 | ar, err := zstd.NewReader(r) 23 | if err != nil { 24 | return nil, err 25 | } 26 | return &pure.CombinedReadCloser{Primary: ar, Source: r}, nil 27 | }, 28 | }) 29 | -------------------------------------------------------------------------------- /internal/impl/pure/extended/zstd_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package extended 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/public/bloblang" 12 | ) 13 | 14 | func TestZstdCompressionDecompression(t *testing.T) { 15 | exec, err := bloblang.Parse(`root = this.compress(algorithm: "zstd")`) 16 | require.NoError(t, err) 17 | 18 | input := []byte("hello world this is a really long string") 19 | 20 | compressed, err := exec.Query(input) 21 | require.NoError(t, err) 22 | 23 | assert.NotEqual(t, input, compressed) 24 | assert.Greater(t, len(compressed.([]byte)), 1) 25 | 26 | exec, err = bloblang.Parse(`root = this.decompress(algorithm: "zstd")`) 27 | require.NoError(t, err) 28 | 29 | decompressed, err := exec.Query(compressed) 30 | require.NoError(t, err) 31 | 32 | assert.Equal(t, input, decompressed) 33 | } 34 | -------------------------------------------------------------------------------- /internal/impl/pure/input_batched.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/component/interop" 7 | "github.com/redpanda-data/benthos/v4/public/service" 8 | ) 9 | 10 | func batchedInputConfig() *service.ConfigSpec { 11 | spec := service.NewConfigSpec(). 12 | Stable(). 13 | Categories("Utility"). 14 | Summary("Consumes data from a child input and applies a batching policy to the stream."). 15 | Description(`Batching at the input level is sometimes useful for processing across micro-batches, and can also sometimes be a useful performance trick. However, most inputs are fine without it so unless you have a specific plan for batching this component is not worth using.`). 16 | Field(service.NewInputField("child").Description("The child input.")). 17 | Field(service.NewBatchPolicyField("policy")). 18 | Version("4.11.0") 19 | return spec 20 | } 21 | 22 | func init() { 23 | service.MustRegisterBatchInput( 24 | "batched", batchedInputConfig(), 25 | func(conf *service.ParsedConfig, mgr *service.Resources) (service.BatchInput, error) { 26 | child, err := conf.FieldInput("child") 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | batcherPol, err := conf.FieldBatchPolicy("policy") 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | batcher, err := batcherPol.NewBatcher(mgr) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | child = child.BatchedWith(batcher) 42 | sChild := interop.UnwrapOwnedInput(child) 43 | return interop.NewUnwrapInternalInput(sChild), nil 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /internal/impl/pure/input_inproc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/redpanda-data/benthos/v4/internal/manager" 13 | "github.com/redpanda-data/benthos/v4/internal/message" 14 | ) 15 | 16 | func TestInprocDryRun(t *testing.T) { 17 | ctx, done := context.WithTimeout(t.Context(), time.Second*30) 18 | defer done() 19 | 20 | t.Parallel() 21 | 22 | mgr, err := manager.New(manager.NewResourceConfig()) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | mgr.SetPipe("foo", make(chan message.Transaction)) 28 | 29 | ip := testInput(t, ` 30 | inproc: foo 31 | `) 32 | 33 | <-time.After(time.Millisecond * 100) 34 | 35 | ip.TriggerStopConsuming() 36 | if err = ip.WaitForClose(ctx); err != nil { 37 | t.Error(err) 38 | } 39 | } 40 | 41 | func TestInprocDryRunNoConn(t *testing.T) { 42 | ctx, done := context.WithTimeout(t.Context(), time.Second*30) 43 | defer done() 44 | 45 | t.Parallel() 46 | 47 | ip := testInput(t, ` 48 | inproc: foo 49 | `) 50 | 51 | <-time.After(time.Millisecond * 100) 52 | 53 | ip.TriggerStopConsuming() 54 | require.NoError(t, ip.WaitForClose(ctx)) 55 | } 56 | -------------------------------------------------------------------------------- /internal/impl/pure/metrics_none.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/public/service" 9 | ) 10 | 11 | func init() { 12 | service.MustRegisterMetricsExporter("none", service.NewConfigSpec(). 13 | Stable(). 14 | Summary(`Disable metrics entirely.`). 15 | Field(service.NewObjectField("").Default(map[string]any{})), 16 | func(conf *service.ParsedConfig, log *service.Logger) (service.MetricsExporter, error) { 17 | return noopMetrics{}, nil 18 | }) 19 | } 20 | 21 | type noopMetrics struct{} 22 | 23 | func (n noopMetrics) NewCounterCtor(name string, labelKeys ...string) service.MetricsExporterCounterCtor { 24 | return func(labelValues ...string) service.MetricsExporterCounter { 25 | return n 26 | } 27 | } 28 | 29 | func (n noopMetrics) NewTimerCtor(name string, labelKeys ...string) service.MetricsExporterTimerCtor { 30 | return func(labelValues ...string) service.MetricsExporterTimer { 31 | return n 32 | } 33 | } 34 | 35 | func (n noopMetrics) NewGaugeCtor(name string, labelKeys ...string) service.MetricsExporterGaugeCtor { 36 | return func(labelValues ...string) service.MetricsExporterGauge { 37 | return n 38 | } 39 | } 40 | 41 | func (n noopMetrics) Close(ctx context.Context) error { 42 | return nil 43 | } 44 | 45 | func (n noopMetrics) Incr(count int64) {} 46 | func (n noopMetrics) IncrFloat64(count float64) {} 47 | func (n noopMetrics) Timing(delta int64) {} 48 | func (n noopMetrics) Set(value int64) {} 49 | func (n noopMetrics) SetFloat64(value float64) {} 50 | -------------------------------------------------------------------------------- /internal/impl/pure/output_broker_greedy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/component" 9 | "github.com/redpanda-data/benthos/v4/internal/component/output" 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | ) 12 | 13 | type greedyOutputBroker struct { 14 | outputs []output.Streamed 15 | } 16 | 17 | func newGreedyOutputBroker(outputs []output.Streamed) (*greedyOutputBroker, error) { 18 | return &greedyOutputBroker{ 19 | outputs: outputs, 20 | }, nil 21 | } 22 | 23 | func (g *greedyOutputBroker) Consume(ts <-chan message.Transaction) error { 24 | for _, out := range g.outputs { 25 | if err := out.Consume(ts); err != nil { 26 | return err 27 | } 28 | } 29 | return nil 30 | } 31 | 32 | func (g *greedyOutputBroker) ConnectionStatus() (s component.ConnectionStatuses) { 33 | for _, out := range g.outputs { 34 | s = append(s, out.ConnectionStatus()...) 35 | } 36 | return 37 | } 38 | 39 | func (g *greedyOutputBroker) TriggerCloseNow() { 40 | for _, out := range g.outputs { 41 | out.TriggerCloseNow() 42 | } 43 | } 44 | 45 | func (g *greedyOutputBroker) WaitForClose(ctx context.Context) error { 46 | for _, out := range g.outputs { 47 | if err := out.WaitForClose(ctx); err != nil { 48 | return err 49 | } 50 | } 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /internal/impl/pure/output_drop.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/component/interop" 9 | "github.com/redpanda-data/benthos/v4/internal/component/output" 10 | "github.com/redpanda-data/benthos/v4/internal/log" 11 | "github.com/redpanda-data/benthos/v4/internal/message" 12 | "github.com/redpanda-data/benthos/v4/public/service" 13 | ) 14 | 15 | func init() { 16 | service.MustRegisterBatchOutput( 17 | "drop", service.NewConfigSpec(). 18 | Stable(). 19 | Categories("Utility"). 20 | Summary(`Drops all messages.`). 21 | Field(service.NewObjectField("").Default(map[string]any{})), 22 | func(conf *service.ParsedConfig, res *service.Resources) (out service.BatchOutput, batchPolicy service.BatchPolicy, maxInFlight int, err error) { 23 | nm := interop.UnwrapManagement(res) 24 | var o output.Streamed 25 | if o, err = output.NewAsyncWriter("drop", 1, newDropWriter(nm.Logger()), nm); err != nil { 26 | return 27 | } 28 | out = interop.NewUnwrapInternalOutput(o) 29 | return 30 | }) 31 | } 32 | 33 | type dropWriter struct { 34 | log log.Modular 35 | } 36 | 37 | func newDropWriter(log log.Modular) *dropWriter { 38 | return &dropWriter{log: log} 39 | } 40 | 41 | func (d *dropWriter) Connect(ctx context.Context) error { 42 | return nil 43 | } 44 | 45 | func (d *dropWriter) WriteBatch(ctx context.Context, msg message.Batch) error { 46 | return nil 47 | } 48 | 49 | func (d *dropWriter) Close(context.Context) error { 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /internal/impl/pure/output_inproc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/redpanda-data/benthos/v4/internal/component" 14 | "github.com/redpanda-data/benthos/v4/internal/component/output" 15 | "github.com/redpanda-data/benthos/v4/internal/manager" 16 | "github.com/redpanda-data/benthos/v4/internal/message" 17 | 18 | _ "github.com/redpanda-data/benthos/v4/public/components/pure" 19 | ) 20 | 21 | func TestInproc(t *testing.T) { 22 | tCtx, done := context.WithTimeout(t.Context(), time.Second*5) 23 | defer done() 24 | 25 | mgr, err := manager.New(manager.NewResourceConfig()) 26 | require.NoError(t, err) 27 | 28 | _, err = mgr.GetPipe("foo") 29 | assert.Equal(t, err, component.ErrPipeNotFound) 30 | 31 | conf := output.NewConfig() 32 | conf.Type = "inproc" 33 | conf.Plugin = "foo" 34 | 35 | ip, err := mgr.NewOutput(conf) 36 | require.NoError(t, err) 37 | 38 | tinchan := make(chan message.Transaction) 39 | require.NoError(t, ip.Consume(tinchan)) 40 | 41 | select { 42 | case tinchan <- message.NewTransaction(nil, nil): 43 | case <-time.After(time.Second): 44 | t.Error("Timed out") 45 | } 46 | 47 | var toutchan <-chan message.Transaction 48 | if toutchan, err = mgr.GetPipe("foo"); err != nil { 49 | t.Error(err) 50 | } 51 | 52 | select { 53 | case <-toutchan: 54 | case <-time.After(time.Second): 55 | t.Error("Timed out") 56 | } 57 | 58 | ip.TriggerCloseNow() 59 | require.NoError(t, ip.WaitForClose(tCtx)) 60 | 61 | select { 62 | case _, open := <-toutchan: 63 | assert.False(t, open) 64 | case <-time.After(time.Second): 65 | t.Error("Timed out") 66 | } 67 | _, err = mgr.GetPipe("foo") 68 | assert.Equal(t, err, component.ErrPipeNotFound) 69 | } 70 | -------------------------------------------------------------------------------- /internal/impl/pure/output_sync_response_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/message" 12 | "github.com/redpanda-data/benthos/v4/internal/transaction" 13 | ) 14 | 15 | func TestSyncResponseWriter(t *testing.T) { 16 | wctx := t.Context() 17 | 18 | impl := transaction.NewResultStore() 19 | w := SyncResponseWriter{} 20 | if err := w.Connect(wctx); err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | ctx := context.WithValue(t.Context(), transaction.ResultStoreKey, impl) 25 | 26 | msg := message.QuickBatch(nil) 27 | p := message.NewPart([]byte("foo")) 28 | p = message.WithContext(ctx, p) 29 | msg = append(msg, p, message.NewPart([]byte("bar"))) 30 | 31 | if err := w.WriteBatch(wctx, msg); err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | impl.Get() 36 | results := impl.Get() 37 | if len(results) != 1 { 38 | t.Fatalf("Wrong count of result batches: %v", len(results)) 39 | } 40 | if results[0].Len() != 2 { 41 | t.Fatalf("Wrong count of messages: %v", results[0].Len()) 42 | } 43 | if exp, act := "foo", string(results[0].Get(0).AsBytes()); exp != act { 44 | t.Errorf("Wrong message contents: %v != %v", act, exp) 45 | } 46 | if exp, act := "bar", string(results[0].Get(1).AsBytes()); exp != act { 47 | t.Errorf("Wrong message contents: %v != %v", act, exp) 48 | } 49 | if store := message.GetContext(results[0].Get(0)).Value(transaction.ResultStoreKey); store != nil { 50 | t.Error("Unexpected nested result store") 51 | } 52 | 53 | require.NoError(t, w.Close(ctx)) 54 | } 55 | -------------------------------------------------------------------------------- /internal/impl/pure/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package pure contains all component implementations that are pure, in that 4 | // they do not interact with external systems. This includes all base component 5 | // types such as brokers and is likely necessary as a base for all builds. 6 | package pure 7 | -------------------------------------------------------------------------------- /internal/impl/pure/processor_crash.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/component/interop" 9 | "github.com/redpanda-data/benthos/v4/internal/log" 10 | "github.com/redpanda-data/benthos/v4/public/service" 11 | ) 12 | 13 | func init() { 14 | spec := service.NewConfigSpec(). 15 | Categories("Utility"). 16 | Beta(). 17 | Summary(`Crashes the process using a fatal log message. The log message can be set using function interpolations described in xref:configuration:interpolation.adoc#bloblang-queries[Bloblang queries] which allows you to log the contents and metadata of messages.`). 18 | Field(service.NewInterpolatedStringField("")) 19 | service.MustRegisterProcessor( 20 | "crash", spec, 21 | func(conf *service.ParsedConfig, res *service.Resources) (service.Processor, error) { 22 | messageStr, err := conf.FieldInterpolatedString() 23 | if err != nil { 24 | return nil, err 25 | } 26 | mgr := interop.UnwrapManagement(res) 27 | return &crashProcessor{mgr.Logger(), messageStr}, nil 28 | }) 29 | } 30 | 31 | type crashProcessor struct { 32 | logger log.Modular 33 | message *service.InterpolatedString 34 | } 35 | 36 | func (l *crashProcessor) Process(ctx context.Context, msg *service.Message) (service.MessageBatch, error) { 37 | m, err := l.message.TryString(msg) 38 | if err != nil { 39 | l.logger.Fatal("failed to interpolate crash message: %v", err) 40 | } else { 41 | l.logger.Fatal("%s", m) 42 | } 43 | return nil, nil 44 | } 45 | 46 | func (l *crashProcessor) Close(ctx context.Context) error { 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /internal/impl/pure/processor_group_by_value_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/component/testutil" 12 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 13 | "github.com/redpanda-data/benthos/v4/internal/message" 14 | 15 | _ "github.com/redpanda-data/benthos/v4/internal/impl/pure" 16 | ) 17 | 18 | func TestGroupByValueBasic(t *testing.T) { 19 | conf, err := testutil.ProcessorFromYAML(` 20 | group_by_value: 21 | value: ${!json("foo")} 22 | `) 23 | require.NoError(t, err) 24 | 25 | proc, err := mock.NewManager().NewProcessor(conf) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | 30 | exp := [][][]byte{ 31 | { 32 | []byte(`{"foo":0,"bar":0}`), 33 | []byte(`{"foo":0,"bar":7}`), 34 | }, 35 | { 36 | []byte(`{"foo":3,"bar":1}`), 37 | }, 38 | { 39 | []byte(`{"bar":2}`), 40 | }, 41 | { 42 | []byte(`{"foo":2,"bar":3}`), 43 | }, 44 | { 45 | []byte(`{"foo":4,"bar":4}`), 46 | }, 47 | { 48 | []byte(`{"foo":1,"bar":5}`), 49 | []byte(`{"foo":1,"bar":6}`), 50 | []byte(`{"foo":1,"bar":8}`), 51 | }, 52 | } 53 | act := [][][]byte{} 54 | 55 | input := message.QuickBatch([][]byte{ 56 | []byte(`{"foo":0,"bar":0}`), 57 | []byte(`{"foo":3,"bar":1}`), 58 | []byte(`{"bar":2}`), 59 | []byte(`{"foo":2,"bar":3}`), 60 | []byte(`{"foo":4,"bar":4}`), 61 | []byte(`{"foo":1,"bar":5}`), 62 | []byte(`{"foo":1,"bar":6}`), 63 | []byte(`{"foo":0,"bar":7}`), 64 | []byte(`{"foo":1,"bar":8}`), 65 | }) 66 | msgs, res := proc.ProcessBatch(t.Context(), input) 67 | if res != nil { 68 | t.Fatal(res) 69 | } 70 | 71 | for _, msg := range msgs { 72 | act = append(act, message.GetAllBytes(msg)) 73 | } 74 | if !reflect.DeepEqual(exp, act) { 75 | t.Errorf("Wrong result: %s != %s", act, exp) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /internal/impl/pure/processor_noop.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/component/interop" 9 | "github.com/redpanda-data/benthos/v4/internal/message" 10 | "github.com/redpanda-data/benthos/v4/public/service" 11 | ) 12 | 13 | func init() { 14 | service.MustRegisterBatchProcessor("noop", service.NewConfigSpec(). 15 | Stable(). 16 | Summary("Noop is a processor that does nothing, the message passes through unchanged. Why? Sometimes doing nothing is the braver option."). 17 | Field(service.NewObjectField("").Default(map[string]any{})), 18 | func(conf *service.ParsedConfig, mgr *service.Resources) (service.BatchProcessor, error) { 19 | p := &noopProcessor{} 20 | return interop.NewUnwrapInternalBatchProcessor(p), nil 21 | }) 22 | } 23 | 24 | type noopProcessor struct{} 25 | 26 | func (c *noopProcessor) ProcessBatch(ctx context.Context, msg message.Batch) ([]message.Batch, error) { 27 | msgs := [1]message.Batch{msg} 28 | return msgs[:], nil 29 | } 30 | 31 | func (c *noopProcessor) Close(context.Context) error { 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /internal/impl/pure/processor_processors_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/component/testutil" 12 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 13 | "github.com/redpanda-data/benthos/v4/internal/message" 14 | 15 | _ "github.com/redpanda-data/benthos/v4/internal/impl/pure" 16 | ) 17 | 18 | func TestProcessors(t *testing.T) { 19 | conf, err := testutil.ProcessorFromYAML(` 20 | processors: 21 | - bloblang: 'root = content().uppercase()' 22 | - bloblang: 'root = content().trim()' 23 | `) 24 | require.NoError(t, err) 25 | 26 | proc, err := mock.NewManager().NewProcessor(conf) 27 | require.NoError(t, err) 28 | 29 | exp := [][][]byte{ 30 | { 31 | []byte(`HELLO FOO WORLD 1`), 32 | []byte(`HELLO WORLD 1`), 33 | []byte(`HELLO BAR WORLD 2`), 34 | }, 35 | } 36 | act := [][][]byte{} 37 | 38 | input := message.QuickBatch([][]byte{ 39 | []byte(` hello foo world 1 `), 40 | []byte(` hello world 1 `), 41 | []byte(` hello bar world 2 `), 42 | }) 43 | msgs, res := proc.ProcessBatch(t.Context(), input) 44 | require.NoError(t, res) 45 | 46 | for _, msg := range msgs { 47 | act = append(act, message.GetAllBytes(msg)) 48 | } 49 | assert.Equal(t, exp, act) 50 | } 51 | -------------------------------------------------------------------------------- /internal/impl/pure/processor_resource_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/component/processor" 9 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | 12 | _ "github.com/redpanda-data/benthos/v4/internal/impl/pure" 13 | ) 14 | 15 | func TestResourceProc(t *testing.T) { 16 | conf := processor.NewConfig() 17 | conf.Type = "bloblang" 18 | conf.Plugin = `root = "foo: " + content()` 19 | 20 | mgr := mock.NewManager() 21 | 22 | resProc, err := mgr.NewProcessor(conf) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | mgr.Processors["foo"] = func(b message.Batch) ([]message.Batch, error) { 28 | msgs, res := resProc.ProcessBatch(t.Context(), b) 29 | if res != nil { 30 | return nil, res 31 | } 32 | return msgs, nil 33 | } 34 | 35 | nConf := processor.NewConfig() 36 | nConf.Type = "resource" 37 | nConf.Plugin = "foo" 38 | 39 | p, err := mgr.NewProcessor(nConf) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | msgs, res := p.ProcessBatch(t.Context(), message.QuickBatch([][]byte{[]byte("bar")})) 45 | if res != nil { 46 | t.Fatal(res) 47 | } 48 | if len(msgs) != 1 { 49 | t.Error("Expected only 1 message") 50 | } 51 | if exp, act := "foo: bar", string(msgs[0].Get(0).AsBytes()); exp != act { 52 | t.Errorf("Wrong result: %v != %v", act, exp) 53 | } 54 | } 55 | 56 | func TestResourceBadName(t *testing.T) { 57 | conf := processor.NewConfig() 58 | conf.Type = "resource" 59 | conf.Plugin = "foo" 60 | 61 | _, err := mock.NewManager().NewProcessor(conf) 62 | if err == nil { 63 | t.Error("expected error from bad resource") 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /internal/impl/pure/processor_sync_response.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/component/interop" 9 | "github.com/redpanda-data/benthos/v4/internal/log" 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | "github.com/redpanda-data/benthos/v4/internal/transaction" 12 | "github.com/redpanda-data/benthos/v4/public/service" 13 | ) 14 | 15 | func init() { 16 | service.MustRegisterBatchProcessor("sync_response", service.NewConfigSpec(). 17 | Categories("Utility"). 18 | Stable(). 19 | Summary("Adds the payload in its current state as a synchronous response to the input source, where it is dealt with according to that specific input type."). 20 | Description(` 21 | For most inputs this mechanism is ignored entirely, in which case the sync response is dropped without penalty. It is therefore safe to use this processor even when combining input types that might not have support for sync responses. An example of an input able to utilize this is the `+"`http_server`"+`. 22 | 23 | For more information please read xref:guides:sync_responses.adoc[synchronous responses].`). 24 | Field(service.NewObjectField("").Default(map[string]any{})), 25 | func(conf *service.ParsedConfig, mgr *service.Resources) (service.BatchProcessor, error) { 26 | p := &syncResponseProc{log: interop.UnwrapManagement(mgr).Logger()} 27 | return interop.NewUnwrapInternalBatchProcessor(p), nil 28 | }) 29 | } 30 | 31 | type syncResponseProc struct { 32 | log log.Modular 33 | } 34 | 35 | func (s *syncResponseProc) ProcessBatch(ctx context.Context, msg message.Batch) ([]message.Batch, error) { 36 | if err := transaction.SetAsResponse(msg); err != nil { 37 | s.log.Debug("Failed to store message as a sync response: %v\n", err) 38 | } 39 | return []message.Batch{msg}, nil 40 | } 41 | 42 | func (s *syncResponseProc) Close(context.Context) error { 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /internal/impl/pure/scanner_chunker_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/component/scanner/testutil" 11 | "github.com/redpanda-data/benthos/v4/public/service" 12 | ) 13 | 14 | func TestLinesChunkerSuite(t *testing.T) { 15 | confSpec := service.NewConfigSpec().Field(service.NewScannerField("test")) 16 | pConf, err := confSpec.ParseYAML(` 17 | test: 18 | chunker: 19 | size: 4 20 | `, nil) 21 | require.NoError(t, err) 22 | 23 | rdr, err := pConf.FieldScanner("test") 24 | require.NoError(t, err) 25 | 26 | testutil.ScannerTestSuite(t, rdr, nil, []byte(`abcdefghijklmnopqrstuvwxyz`), "abcd", "efgh", "ijkl", "mnop", "qrst", "uvwx", "yz") 27 | } 28 | -------------------------------------------------------------------------------- /internal/impl/pure/scanner_decompress_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "encoding/hex" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/component/scanner/testutil" 12 | "github.com/redpanda-data/benthos/v4/public/service" 13 | ) 14 | 15 | func TestDecompressScannerSuite(t *testing.T) { 16 | confSpec := service.NewConfigSpec().Field(service.NewScannerField("test")) 17 | pConf, err := confSpec.ParseYAML(` 18 | test: 19 | decompress: 20 | algorithm: gzip 21 | into: 22 | lines: 23 | custom_delimiter: X 24 | `, nil) 25 | require.NoError(t, err) 26 | 27 | rdr, err := pConf.FieldScanner("test") 28 | require.NoError(t, err) 29 | 30 | inputBytes, err := hex.DecodeString("1f8b080000096e8800ff001e00e1ff68656c6c6f58776f726c64587468697358697358636f6d7072657373656403009104d92d1e000000") 31 | require.NoError(t, err) 32 | 33 | testutil.ScannerTestSuite(t, rdr, nil, inputBytes, "hello", "world", "this", "is", "compressed") 34 | } 35 | -------------------------------------------------------------------------------- /internal/impl/pure/scanner_re_match_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "fmt" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/component/scanner/testutil" 12 | "github.com/redpanda-data/benthos/v4/public/service" 13 | ) 14 | 15 | func TestReMatchScannerSuite(t *testing.T) { 16 | testREPattern := func(pattern, input string, expected ...string) { 17 | confSpec := service.NewConfigSpec().Field(service.NewScannerField("test")) 18 | pConf, err := confSpec.ParseYAML(fmt.Sprintf(` 19 | test: 20 | re_match: 21 | pattern: '%v' 22 | max_buffer_size: 200 23 | `, pattern), nil) 24 | require.NoError(t, err) 25 | 26 | rdr, err := pConf.FieldScanner("test") 27 | require.NoError(t, err) 28 | 29 | testutil.ScannerTestSuite(t, rdr, nil, []byte(input), expected...) 30 | } 31 | 32 | testREPattern("(?m)^", "foo\nbar\nbaz", "foo\n", "bar\n", "baz") 33 | 34 | testREPattern("split", "foo\nbar\nsplit\nbaz\nsplitsplit", "foo\nbar\n", "split\nbaz\n", "split", "split") 35 | 36 | testREPattern("\\n", "split", "split") 37 | testREPattern("split", "split", "split") 38 | 39 | testREPattern("\\n", "foo\nbar\nsplit\nbaz\nsplitsplit", "foo", "\nbar", "\nsplit", "\nbaz", "\nsplitsplit") 40 | 41 | testREPattern("\\n", "foo\nbar\nsplit\nbaz", "foo", "\nbar", "\nsplit", "\nbaz") 42 | 43 | testREPattern("\\n\\d", "20:20:22 ERROR\nCode\n20:20:21 INFO\n20:20:21 INFO\n20:20:22 ERROR\nCode\n", "20:20:22 ERROR\nCode", "\n20:20:21 INFO", "\n20:20:21 INFO", "\n20:20:22 ERROR\nCode\n") 44 | 45 | testREPattern("(?m)^\\d\\d:\\d\\d:\\d\\d", "20:20:22 ERROR\nCode\n20:20:21 INFO\n20:20:21 INFO\n20:20\n20:20:22 ERROR\nCode\n2022", "20:20:22 ERROR\nCode\n", "20:20:21 INFO\n", "20:20:21 INFO\n20:20\n", "20:20:22 ERROR\nCode\n2022") 46 | 47 | testREPattern("split", "") 48 | } 49 | -------------------------------------------------------------------------------- /internal/impl/pure/scanner_switch_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/component/scanner/testutil" 11 | "github.com/redpanda-data/benthos/v4/public/service" 12 | ) 13 | 14 | func TestSwitchScanner(t *testing.T) { 15 | confSpec := service.NewConfigSpec().Field(service.NewScannerField("test")) 16 | pConf, err := confSpec.ParseYAML(` 17 | test: 18 | switch: 19 | - re_match_name: '\.json$' 20 | scanner: { to_the_end: {} } 21 | - re_match_name: '\.csv$' 22 | scanner: { csv: {} } 23 | - re_match_name: '\.chunks$' 24 | scanner: 25 | chunker: 26 | size: 4 27 | - scanner: { to_the_end: {} } 28 | 29 | `, nil) 30 | require.NoError(t, err) 31 | 32 | rdr, err := pConf.FieldScanner("test") 33 | require.NoError(t, err) 34 | 35 | details := service.NewScannerSourceDetails() 36 | details.SetName("a/b/foo.csv") 37 | testutil.ScannerTestSuite(t, rdr, details, []byte(`a,b,c 38 | a1,b1,c1 39 | a2,b2,c2 40 | a3,b3,c3 41 | `), 42 | `{"a":"a1","b":"b1","c":"c1"}`, 43 | `{"a":"a2","b":"b2","c":"c2"}`, 44 | `{"a":"a3","b":"b3","c":"c3"}`, 45 | ) 46 | 47 | details = service.NewScannerSourceDetails() 48 | details.SetName("woof/meow.chunks") 49 | testutil.ScannerTestSuite(t, rdr, details, []byte(`abcdefghijklmnopqrstuvwxyz`), "abcd", "efgh", "ijkl", "mnop", "qrst", "uvwx", "yz") 50 | 51 | details = service.NewScannerSourceDetails() 52 | details.SetName("./meow.json") 53 | testutil.ScannerTestSuite(t, rdr, details, []byte(`{"hello":"world"}`), `{"hello":"world"}`) 54 | } 55 | -------------------------------------------------------------------------------- /internal/impl/pure/scanner_tar_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure_test 4 | 5 | import ( 6 | "archive/tar" 7 | "bytes" 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/redpanda-data/benthos/v4/internal/component/scanner/testutil" 14 | "github.com/redpanda-data/benthos/v4/public/service" 15 | ) 16 | 17 | func TestTarScannerSuite(t *testing.T) { 18 | input := []string{ 19 | "first document", 20 | "second document", 21 | "third document", 22 | } 23 | 24 | var tarBuf bytes.Buffer 25 | tw := tar.NewWriter(&tarBuf) 26 | for i := range input { 27 | hdr := &tar.Header{ 28 | Name: fmt.Sprintf("testfile%v", i), 29 | Mode: 0o600, 30 | Size: int64(len(input[i])), 31 | } 32 | 33 | err := tw.WriteHeader(hdr) 34 | require.NoError(t, err) 35 | 36 | _, err = tw.Write([]byte(input[i])) 37 | require.NoError(t, err) 38 | } 39 | require.NoError(t, tw.Close()) 40 | 41 | confSpec := service.NewConfigSpec().Field(service.NewScannerField("test")) 42 | pConf, err := confSpec.ParseYAML(` 43 | test: 44 | tar: {} 45 | `, nil) 46 | require.NoError(t, err) 47 | 48 | rdr, err := pConf.FieldScanner("test") 49 | require.NoError(t, err) 50 | 51 | testutil.ScannerTestSuite(t, rdr, nil, tarBuf.Bytes(), input...) 52 | } 53 | -------------------------------------------------------------------------------- /internal/impl/pure/tracer_none.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pure 4 | 5 | import ( 6 | "go.opentelemetry.io/otel/trace" 7 | "go.opentelemetry.io/otel/trace/noop" 8 | 9 | "github.com/redpanda-data/benthos/v4/public/service" 10 | ) 11 | 12 | func init() { 13 | service.MustRegisterOtelTracerProvider( 14 | "none", service.NewConfigSpec(). 15 | Stable(). 16 | Summary(`Do not send tracing events anywhere.`). 17 | Field( 18 | service.NewObjectField("").Default(map[string]any{}), 19 | ), 20 | func(conf *service.ParsedConfig) (trace.TracerProvider, error) { 21 | return noop.NewTracerProvider(), nil 22 | }) 23 | } 24 | -------------------------------------------------------------------------------- /internal/log/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package log 4 | 5 | // Modular is a log printer that allows you to branch new modules. 6 | type Modular interface { 7 | WithFields(fields map[string]string) Modular 8 | With(keyValues ...any) Modular 9 | 10 | Fatal(format string, v ...any) 11 | Error(format string, v ...any) 12 | Warn(format string, v ...any) 13 | Info(format string, v ...any) 14 | Debug(format string, v ...any) 15 | Trace(format string, v ...any) 16 | } 17 | -------------------------------------------------------------------------------- /internal/log/tee.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package log 4 | 5 | type teeLogger struct { 6 | a, b Modular 7 | } 8 | 9 | // TeeLogger creates a new log adapter that allows you to branch new modules. 10 | func TeeLogger(a, b Modular) Modular { 11 | return &teeLogger{a: a, b: b} 12 | } 13 | 14 | // WithFields adds extra fields to the log adapter. 15 | func (t *teeLogger) WithFields(fields map[string]string) Modular { 16 | return &teeLogger{ 17 | a: t.a.WithFields(fields), 18 | b: t.b.WithFields(fields), 19 | } 20 | } 21 | 22 | // With returns a Logger that includes the given attributes. Arguments are 23 | // converted to attributes as if by the standard `Logger.Log()`. 24 | func (t *teeLogger) With(keyValues ...any) Modular { 25 | return &teeLogger{ 26 | a: t.a.With(keyValues...), 27 | b: t.b.With(keyValues...), 28 | } 29 | } 30 | 31 | // Fatal logs at error level followed by a call to `os.Exit()`. 32 | func (t *teeLogger) Fatal(format string, v ...any) { 33 | t.a.Fatal(format, v...) 34 | t.b.Fatal(format, v...) 35 | } 36 | 37 | // Error logs at error level. 38 | func (t *teeLogger) Error(format string, v ...any) { 39 | t.a.Error(format, v...) 40 | t.b.Error(format, v...) 41 | } 42 | 43 | // Warn logs at warning level. 44 | func (t *teeLogger) Warn(format string, v ...any) { 45 | t.a.Warn(format, v...) 46 | t.b.Warn(format, v...) 47 | } 48 | 49 | // Info logs at info level. 50 | func (t *teeLogger) Info(format string, v ...any) { 51 | t.a.Info(format, v...) 52 | t.b.Info(format, v...) 53 | } 54 | 55 | // Debug logs at debug level. 56 | func (t *teeLogger) Debug(format string, v ...any) { 57 | t.a.Debug(format, v...) 58 | t.b.Debug(format, v...) 59 | } 60 | 61 | // Trace logs at trace level. 62 | func (t *teeLogger) Trace(format string, v ...any) { 63 | t.a.Trace(format, v...) 64 | t.b.Trace(format, v...) 65 | } 66 | -------------------------------------------------------------------------------- /internal/manager/docs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package manager 4 | 5 | import ( 6 | "errors" 7 | 8 | "github.com/Jeffail/gabs/v2" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/docs" 11 | ) 12 | 13 | func lintResource(ctx docs.LintContext, line, col int, v any) []docs.Lint { 14 | if _, ok := v.(map[string]any); !ok { 15 | return nil 16 | } 17 | gObj := gabs.Wrap(v) 18 | label, _ := gObj.S("label").Data().(string) 19 | if label == "" { 20 | return []docs.Lint{ 21 | docs.NewLintError(line, docs.LintBadLabel, errors.New("the label field for resources must be unique and not empty")), 22 | } 23 | } 24 | return nil 25 | } 26 | 27 | // Spec returns a field spec for the manager configuration. 28 | func Spec() docs.FieldSpecs { 29 | return docs.FieldSpecs{ 30 | docs.FieldInput( 31 | "input_resources", "A list of input resources, each must have a unique label.", 32 | ).Array().LinterFunc(lintResource).HasDefault([]any{}).Advanced(), 33 | 34 | docs.FieldProcessor( 35 | "processor_resources", "A list of processor resources, each must have a unique label.", 36 | ).Array().LinterFunc(lintResource).HasDefault([]any{}).Advanced(), 37 | 38 | docs.FieldOutput( 39 | "output_resources", "A list of output resources, each must have a unique label.", 40 | ).Array().LinterFunc(lintResource).HasDefault([]any{}).Advanced(), 41 | 42 | docs.FieldCache( 43 | "cache_resources", "A list of cache resources, each must have a unique label.", 44 | ).Array().LinterFunc(lintResource).HasDefault([]any{}).Advanced(), 45 | 46 | docs.FieldRateLimit( 47 | "rate_limit_resources", "A list of rate limit resources, each must have a unique label.", 48 | ).Array().LinterFunc(lintResource).HasDefault([]any{}).Advanced(), 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /internal/manager/mock/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock 4 | 5 | import ( 6 | "context" 7 | "time" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/component" 10 | "github.com/redpanda-data/benthos/v4/internal/component/cache" 11 | ) 12 | 13 | // CacheItem represents a cached key/ttl pair. 14 | type CacheItem struct { 15 | Value string 16 | TTL *time.Duration 17 | } 18 | 19 | // Cache provides a mock cache implementation. 20 | type Cache struct { 21 | Values map[string]CacheItem 22 | } 23 | 24 | // Get a mock cache item. 25 | func (c *Cache) Get(ctx context.Context, key string) ([]byte, error) { 26 | i, ok := c.Values[key] 27 | if !ok { 28 | return nil, component.ErrKeyNotFound 29 | } 30 | return []byte(i.Value), nil 31 | } 32 | 33 | // Set a mock cache item. 34 | func (c *Cache) Set(ctx context.Context, key string, value []byte, ttl *time.Duration) error { 35 | c.Values[key] = CacheItem{ 36 | Value: string(value), 37 | TTL: ttl, 38 | } 39 | return nil 40 | } 41 | 42 | // SetMulti sets multiple mock cache items. 43 | func (c *Cache) SetMulti(ctx context.Context, kvs map[string]cache.TTLItem) error { 44 | for k, v := range kvs { 45 | c.Values[k] = CacheItem{ 46 | Value: string(v.Value), 47 | TTL: v.TTL, 48 | } 49 | } 50 | return nil 51 | } 52 | 53 | // Add a mock cache item. 54 | func (c *Cache) Add(ctx context.Context, key string, value []byte, ttl *time.Duration) error { 55 | if _, ok := c.Values[key]; ok { 56 | return component.ErrKeyAlreadyExists 57 | } 58 | c.Values[key] = CacheItem{ 59 | Value: string(value), 60 | TTL: ttl, 61 | } 62 | return nil 63 | } 64 | 65 | // Delete a mock cache item. 66 | func (c *Cache) Delete(ctx context.Context, key string) error { 67 | delete(c.Values, key) 68 | return nil 69 | } 70 | 71 | // Close does nothing. 72 | func (c *Cache) Close(ctx context.Context) error { 73 | return nil 74 | } 75 | -------------------------------------------------------------------------------- /internal/manager/mock/cache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock_test 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/component/cache" 7 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 8 | ) 9 | 10 | var _ cache.V1 = &mock.Cache{} 11 | -------------------------------------------------------------------------------- /internal/manager/mock/input.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock 4 | 5 | import ( 6 | "context" 7 | "sync" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/component" 10 | "github.com/redpanda-data/benthos/v4/internal/message" 11 | ) 12 | 13 | // Input provides a mocked input implementation. 14 | type Input struct { 15 | TChan chan message.Transaction 16 | closed bool 17 | closeOnce sync.Once 18 | } 19 | 20 | // NewInput creates a new mock input that will return transactions containing a 21 | // list of batches, then exit. 22 | func NewInput(batches []message.Batch) *Input { 23 | ts := make(chan message.Transaction, len(batches)) 24 | resChan := make(chan error, len(batches)) 25 | go func() { 26 | defer close(ts) 27 | for _, b := range batches { 28 | ts <- message.NewTransaction(b, resChan) 29 | } 30 | }() 31 | return &Input{TChan: ts} 32 | } 33 | 34 | // ConnectionStatus returns the current connection activity. 35 | func (f *Input) ConnectionStatus() component.ConnectionStatuses { 36 | if f.closed { 37 | return component.ConnectionStatuses{ 38 | component.ConnectionClosed(component.NoopObservability()), 39 | } 40 | } 41 | return component.ConnectionStatuses{ 42 | component.ConnectionActive(component.NoopObservability()), 43 | } 44 | } 45 | 46 | // TransactionChan returns a transaction channel. 47 | func (f *Input) TransactionChan() <-chan message.Transaction { 48 | return f.TChan 49 | } 50 | 51 | // TriggerStopConsuming closes the input transaction channel. 52 | func (f *Input) TriggerStopConsuming() { 53 | f.closeOnce.Do(func() { 54 | close(f.TChan) 55 | f.closed = true 56 | }) 57 | } 58 | 59 | // TriggerCloseNow closes the input transaction channel. 60 | func (f *Input) TriggerCloseNow() { 61 | f.closeOnce.Do(func() { 62 | close(f.TChan) 63 | f.closed = true 64 | }) 65 | } 66 | 67 | // WaitForClose does nothing. 68 | func (f *Input) WaitForClose(ctx context.Context) error { 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /internal/manager/mock/input_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock_test 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/component/input" 7 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 8 | ) 9 | 10 | var _ input.Streamed = &mock.Input{} 11 | -------------------------------------------------------------------------------- /internal/manager/mock/manager_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock_test 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/bundle" 7 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 8 | ) 9 | 10 | var _ bundle.NewManagement = &mock.Manager{} 11 | -------------------------------------------------------------------------------- /internal/manager/mock/output_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock_test 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/component/output" 7 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 8 | ) 9 | 10 | var _ output.Sync = mock.OutputWriter(nil) 11 | -------------------------------------------------------------------------------- /internal/manager/mock/processor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/redpanda-data/benthos/v4/internal/message" 9 | ) 10 | 11 | // Processor provides a mock processor implementation around a closure. 12 | type Processor func(message.Batch) ([]message.Batch, error) 13 | 14 | // ProcessBatch returns the closure result executed on a batch. 15 | func (p Processor) ProcessBatch(ctx context.Context, b message.Batch) ([]message.Batch, error) { 16 | return p(b) 17 | } 18 | 19 | // Close does nothing. 20 | func (p Processor) Close(context.Context) error { 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /internal/manager/mock/processor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock_test 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/component/processor" 7 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 8 | ) 9 | 10 | var _ processor.V1 = mock.Processor(nil) 11 | -------------------------------------------------------------------------------- /internal/manager/mock/ratelimit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock 4 | 5 | import ( 6 | "context" 7 | "time" 8 | ) 9 | 10 | // RateLimit provides a mock rate limit implementation around a closure. 11 | type RateLimit func(context.Context) (time.Duration, error) 12 | 13 | // Access the rate limit. 14 | func (r RateLimit) Access(ctx context.Context) (time.Duration, error) { 15 | return r(ctx) 16 | } 17 | 18 | // Close does nothing. 19 | func (r RateLimit) Close(ctx context.Context) error { 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /internal/manager/mock/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package mock_test 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/component/ratelimit" 7 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 8 | ) 9 | 10 | var _ ratelimit.V1 = mock.RateLimit(nil) 11 | -------------------------------------------------------------------------------- /internal/manager/output_wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package manager 4 | 5 | import ( 6 | "context" 7 | "sync" 8 | 9 | "github.com/Jeffail/shutdown" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/component" 12 | ioutput "github.com/redpanda-data/benthos/v4/internal/component/output" 13 | "github.com/redpanda-data/benthos/v4/internal/message" 14 | ) 15 | 16 | var _ ioutput.Sync = &outputWrapper{} 17 | 18 | type outputWrapper struct { 19 | output ioutput.Streamed 20 | shutSig *shutdown.Signaller 21 | 22 | tranChan chan message.Transaction 23 | tranMut sync.RWMutex 24 | } 25 | 26 | func wrapOutput(o ioutput.Streamed) (*outputWrapper, error) { 27 | tranChan := make(chan message.Transaction) 28 | if err := o.Consume(tranChan); err != nil { 29 | return nil, err 30 | } 31 | return &outputWrapper{ 32 | output: o, 33 | shutSig: shutdown.NewSignaller(), 34 | tranChan: tranChan, 35 | }, nil 36 | } 37 | 38 | func (w *outputWrapper) WriteTransaction(ctx context.Context, t message.Transaction) error { 39 | w.tranMut.RLock() 40 | defer w.tranMut.RUnlock() 41 | select { 42 | case w.tranChan <- t: 43 | case <-w.shutSig.SoftStopChan(): 44 | case <-ctx.Done(): 45 | return component.ErrTimeout 46 | } 47 | return nil 48 | } 49 | 50 | func (w *outputWrapper) ConnectionStatus() component.ConnectionStatuses { 51 | return w.output.ConnectionStatus() 52 | } 53 | 54 | func (w *outputWrapper) TriggerStopConsuming() { 55 | w.shutSig.TriggerSoftStop() 56 | w.tranMut.Lock() 57 | if w.tranChan != nil { 58 | close(w.tranChan) 59 | w.tranChan = nil 60 | } 61 | w.tranMut.Unlock() 62 | } 63 | 64 | func (w *outputWrapper) TriggerCloseNow() { 65 | w.output.TriggerCloseNow() 66 | } 67 | 68 | func (w *outputWrapper) WaitForClose(ctx context.Context) error { 69 | return w.output.WaitForClose(ctx) 70 | } 71 | -------------------------------------------------------------------------------- /internal/manager/output_wrapper_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package manager 4 | 5 | import ( 6 | "context" 7 | "sync" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | 14 | "github.com/redpanda-data/benthos/v4/internal/manager/mock" 15 | "github.com/redpanda-data/benthos/v4/internal/message" 16 | ) 17 | 18 | func TestOutputWrapperShutdown(t *testing.T) { 19 | tCtx, done := context.WithTimeout(t.Context(), time.Second*30) 20 | defer done() 21 | 22 | mOutput := &mock.OutputChanneled{ 23 | TChan: make(<-chan message.Transaction), 24 | } 25 | 26 | mWrapped, err := wrapOutput(mOutput) 27 | require.NoError(t, err) 28 | 29 | wg := sync.WaitGroup{} 30 | wg.Add(1) 31 | go func() { 32 | for ts := range mOutput.TChan { 33 | assert.NoError(t, ts.Ack(tCtx, nil)) 34 | } 35 | wg.Done() 36 | }() 37 | 38 | // Trigger Async Shutdown 39 | go func() { 40 | time.Sleep(time.Millisecond * 50) 41 | mWrapped.TriggerStopConsuming() 42 | }() 43 | 44 | for i := 0; i < 1000; i++ { 45 | require.NoError(t, mWrapped.WriteTransaction(tCtx, message.NewTransactionFunc(message.Batch{ 46 | message.NewPart([]byte("hello world")), 47 | }, func(ctx context.Context, err error) error { 48 | return nil 49 | }))) 50 | } 51 | 52 | wg.Wait() 53 | } 54 | -------------------------------------------------------------------------------- /internal/manager/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package manager implements the types.Manager interface used for creating and 4 | // sharing resources across a Benthos service. 5 | package manager 6 | -------------------------------------------------------------------------------- /internal/message/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package message 4 | 5 | import ( 6 | "errors" 7 | ) 8 | 9 | // Errors returned by the message type. 10 | var ( 11 | ErrMessagePartNotExist = errors.New("target message part does not exist") 12 | ErrBadMessageBytes = errors.New("serialised message bytes were in unexpected format") 13 | ) 14 | -------------------------------------------------------------------------------- /internal/message/sort_group_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package message 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNestedSortGroups(t *testing.T) { 12 | msg := Batch{ 13 | NewPart([]byte("first")), 14 | NewPart([]byte("second")), 15 | } 16 | 17 | group1, msg1 := NewSortGroup(msg) 18 | 19 | assert.Equal(t, -1, group1.GetIndex(msg.Get(0))) 20 | assert.Equal(t, -1, group1.GetIndex(msg.Get(1))) 21 | 22 | assert.Equal(t, 0, group1.GetIndex(msg1.Get(0))) 23 | assert.Equal(t, 1, group1.GetIndex(msg1.Get(1))) 24 | 25 | msg1Reordered := Batch{msg1[1], msg1[0]} 26 | 27 | assert.Equal(t, group1, TopLevelSortGroup(msg1[1])) 28 | 29 | group2, msg2 := NewSortGroup(msg1Reordered) 30 | 31 | assert.Equal(t, -1, group1.GetIndex(msg.Get(0))) 32 | assert.Equal(t, -1, group1.GetIndex(msg.Get(1))) 33 | 34 | assert.Equal(t, 0, group1.GetIndex(msg1.Get(0))) 35 | assert.Equal(t, 1, group1.GetIndex(msg1.Get(1))) 36 | 37 | assert.Equal(t, -1, group2.GetIndex(msg.Get(0))) 38 | assert.Equal(t, -1, group2.GetIndex(msg.Get(1))) 39 | 40 | assert.Equal(t, -1, group2.GetIndex(msg1.Get(0))) 41 | assert.Equal(t, -1, group2.GetIndex(msg1.Get(1))) 42 | 43 | assert.Equal(t, 0, group2.GetIndex(msg2.Get(0))) 44 | assert.Equal(t, 1, group2.GetIndex(msg2.Get(1))) 45 | 46 | assert.Equal(t, 1, group1.GetIndex(msg2.Get(0))) 47 | assert.Equal(t, 0, group1.GetIndex(msg2.Get(1))) 48 | 49 | assert.Equal(t, group1, TopLevelSortGroup(msg1[1])) 50 | assert.Equal(t, group2, TopLevelSortGroup(msg2[1])) 51 | } 52 | -------------------------------------------------------------------------------- /internal/old/util/throttle/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package throttle implements throttle strategies. 4 | package throttle 5 | -------------------------------------------------------------------------------- /internal/periodic/periodic_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package periodic 4 | 5 | import ( 6 | "context" 7 | "sync/atomic" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestCancellation(t *testing.T) { 15 | counter := atomic.Int32{} 16 | p := New(time.Hour, func() { 17 | counter.Add(1) 18 | }) 19 | p.Start() 20 | require.Equal(t, int32(0), counter.Load()) 21 | p.Stop() 22 | require.Equal(t, int32(0), counter.Load()) 23 | } 24 | 25 | func TestWorks(t *testing.T) { 26 | counter := atomic.Int32{} 27 | p := New(time.Millisecond, func() { 28 | counter.Add(1) 29 | }) 30 | p.Start() 31 | require.Eventually(t, func() bool { return counter.Load() > 5 }, time.Second, time.Millisecond) 32 | p.Stop() 33 | snapshot := counter.Load() 34 | time.Sleep(time.Millisecond * 250) 35 | require.Equal(t, snapshot, counter.Load()) 36 | } 37 | 38 | func TestWorksWithContext(t *testing.T) { 39 | active := atomic.Bool{} 40 | p := NewWithContext(time.Millisecond, func(ctx context.Context) { 41 | active.Store(true) 42 | // Block until context is cancelled 43 | <-ctx.Done() 44 | active.Store(false) 45 | }) 46 | p.Start() 47 | require.Eventually(t, func() bool { return active.Load() }, 10*time.Millisecond, time.Millisecond) 48 | p.Stop() 49 | require.False(t, active.Load()) 50 | } 51 | -------------------------------------------------------------------------------- /internal/pipeline/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package pipeline_test 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/bundle" 12 | "github.com/redpanda-data/benthos/v4/internal/docs" 13 | "github.com/redpanda-data/benthos/v4/internal/pipeline" 14 | ) 15 | 16 | func TestConfigParseYAML(t *testing.T) { 17 | tests := []struct { 18 | name string 19 | input string 20 | errContains string 21 | validateFn func(t testing.TB, v pipeline.Config) 22 | }{ 23 | { 24 | name: "basic config", 25 | input: ` 26 | threads: 123 27 | processors: 28 | - label: a 29 | mapping: 'root = "a"' 30 | - label: b 31 | mapping: 'root = "b"' 32 | `, 33 | validateFn: func(t testing.TB, v pipeline.Config) { 34 | assert.Equal(t, 123, v.Threads) 35 | require.Len(t, v.Processors, 2) 36 | assert.Equal(t, "a", v.Processors[0].Label) 37 | assert.Equal(t, "mapping", v.Processors[0].Type) 38 | assert.Equal(t, "b", v.Processors[1].Label) 39 | assert.Equal(t, "mapping", v.Processors[1].Type) 40 | }, 41 | }, 42 | } 43 | 44 | for _, test := range tests { 45 | test := test 46 | t.Run(test.name, func(t *testing.T) { 47 | n, err := docs.UnmarshalYAML([]byte(test.input)) 48 | require.NoError(t, err) 49 | 50 | conf, err := pipeline.FromAny(bundle.GlobalEnvironment, n) 51 | if test.errContains == "" { 52 | require.NoError(t, err) 53 | test.validateFn(t, conf) 54 | } else { 55 | require.Error(t, err) 56 | assert.Contains(t, err.Error(), test.errContains) 57 | } 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /internal/pipeline/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package pipeline contains structures that implement both the Producer and 4 | // Consumer interfaces. They can be used as extra pipeline components for 5 | // various utilities. 6 | package pipeline 7 | -------------------------------------------------------------------------------- /internal/stream/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package stream_test 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/redpanda-data/benthos/v4/internal/component/testutil" 12 | "github.com/redpanda-data/benthos/v4/internal/stream" 13 | ) 14 | 15 | func TestConfigParseYAML(t *testing.T) { 16 | tests := []struct { 17 | name string 18 | input string 19 | errContains string 20 | validateFn func(t testing.TB, v stream.Config) 21 | }{ 22 | { 23 | name: "one of everything", 24 | input: ` 25 | input: 26 | label: a 27 | generate: 28 | count: 1 29 | mapping: 'root.id = "a"' 30 | interval: 1s 31 | 32 | buffer: 33 | memory: 34 | limit: 456 35 | 36 | pipeline: 37 | threads: 123 38 | 39 | output: 40 | label: c 41 | reject: "c rejected" 42 | `, 43 | validateFn: func(t testing.TB, v stream.Config) { 44 | assert.Equal(t, "a", v.Input.Label) 45 | assert.Equal(t, "generate", v.Input.Type) 46 | assert.Equal(t, "memory", v.Buffer.Type) 47 | assert.Equal(t, 123, v.Pipeline.Threads) 48 | assert.Equal(t, "c", v.Output.Label) 49 | assert.Equal(t, "reject", v.Output.Type) 50 | }, 51 | }, 52 | } 53 | 54 | for _, test := range tests { 55 | test := test 56 | t.Run(test.name, func(t *testing.T) { 57 | conf, err := testutil.StreamFromYAML(test.input) 58 | if test.errContains == "" { 59 | require.NoError(t, err) 60 | test.validateFn(t, conf) 61 | } else { 62 | require.Error(t, err) 63 | assert.Contains(t, err.Error(), test.errContains) 64 | } 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/stream/docs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package stream 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/bundle" 7 | "github.com/redpanda-data/benthos/v4/internal/docs" 8 | "github.com/redpanda-data/benthos/v4/internal/pipeline" 9 | ) 10 | 11 | // Spec returns a docs.FieldSpec for a stream configuration. 12 | func Spec() docs.FieldSpecs { 13 | defaultInput := map[string]any{"inproc": ""} 14 | if _, exists := bundle.GlobalEnvironment.GetDocs("stdin", docs.TypeInput); exists { 15 | defaultInput = map[string]any{ 16 | "stdin": map[string]any{}, 17 | } 18 | } 19 | 20 | defaultOutput := map[string]any{"reject": "message rejected by default because an output is not configured"} 21 | if _, exists := bundle.GlobalEnvironment.GetDocs("stdout", docs.TypeOutput); exists { 22 | defaultOutput = map[string]any{ 23 | "stdout": map[string]any{}, 24 | } 25 | } 26 | 27 | return docs.FieldSpecs{ 28 | docs.FieldInput(fieldInput, "An input to source messages from.").HasDefault(defaultInput), 29 | docs.FieldBuffer(fieldBuffer, "An optional buffer to store messages during transit.").HasDefault(map[string]any{ 30 | "none": map[string]any{}, 31 | }), 32 | pipeline.ConfigSpec(), 33 | docs.FieldOutput(fieldOutput, "An output to sink messages to.").HasDefault(defaultOutput), 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/stream/manager/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package manager creates and manages multiple streams, providing an API for 4 | // performing CRUD operations. 5 | package manager 6 | -------------------------------------------------------------------------------- /internal/stream/manager/type_stress_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package manager_test 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "sync" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | 15 | "github.com/redpanda-data/benthos/v4/internal/component/testutil" 16 | bmanager "github.com/redpanda-data/benthos/v4/internal/manager" 17 | "github.com/redpanda-data/benthos/v4/internal/stream/manager" 18 | 19 | // Import pure components for tests. 20 | _ "github.com/redpanda-data/benthos/v4/internal/impl/pure" 21 | ) 22 | 23 | func TestTypeUnderStress(t *testing.T) { 24 | t.Skip("Skipping long running stress test") 25 | 26 | ctx, done := context.WithTimeout(t.Context(), time.Second*30) 27 | defer done() 28 | 29 | res, err := bmanager.New(bmanager.NewResourceConfig()) 30 | require.NoError(t, err) 31 | 32 | mgr := manager.New(res) 33 | 34 | conf, err := testutil.StreamFromYAML(` 35 | input: 36 | generate: 37 | count: 3 38 | interval: 1us 39 | mapping: 'root.id = uuid_v4()' 40 | output: 41 | drop: {} 42 | `) 43 | require.NoError(t, err) 44 | 45 | wg := sync.WaitGroup{} 46 | for j := 0; j < 1000; j++ { 47 | wg.Add(1) 48 | go func(threadID int) { 49 | defer wg.Done() 50 | for i := 0; i < 100; i++ { 51 | streamID := fmt.Sprintf("foo-%v-%v", threadID, i) 52 | require.NoError(t, mgr.Create(streamID, conf)) 53 | 54 | assert.Eventually(t, func() bool { 55 | details, err := mgr.Read(streamID) 56 | return err == nil && !details.IsRunning() 57 | }, time.Second, time.Millisecond*50) 58 | 59 | require.NoError(t, mgr.Delete(ctx, streamID)) 60 | } 61 | }(j) 62 | } 63 | 64 | wg.Wait() 65 | require.NoError(t, mgr.Stop(ctx)) 66 | } 67 | -------------------------------------------------------------------------------- /internal/tls/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package tls provides Benthos configuration fields and wrappers for a 4 | // crypto/tls config. 5 | package tls 6 | -------------------------------------------------------------------------------- /internal/tracing/otel_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package tracing 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "go.opentelemetry.io/otel" 10 | "go.opentelemetry.io/otel/propagation" 11 | "go.opentelemetry.io/otel/trace" 12 | "go.opentelemetry.io/otel/trace/noop" 13 | 14 | "github.com/redpanda-data/benthos/v4/internal/message" 15 | ) 16 | 17 | func TestInitSpansFromParentTextMap(t *testing.T) { 18 | t.Run("it will update the context for each message in the batch", func(t *testing.T) { 19 | textMap := map[string]any{ 20 | "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", 21 | } 22 | 23 | msgOne := message.NewPart([]byte("hello")) 24 | msgTwo := message.NewPart([]byte("world")) 25 | 26 | batch := message.Batch([]*message.Part{msgOne, msgTwo}) 27 | 28 | otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{})) 29 | tp := noop.NewTracerProvider() 30 | 31 | err := InitSpansFromParentTextMap(tp, "test", textMap, batch) 32 | assert.NoError(t, err) 33 | 34 | spanOne := trace.SpanFromContext(batch[0].GetContext()) 35 | assert.Equal(t, "4bf92f3577b34da6a3ce929d0e0e4736", spanOne.SpanContext().TraceID().String()) 36 | assert.Equal(t, "00f067aa0ba902b7", spanOne.SpanContext().SpanID().String()) 37 | 38 | spanTwo := trace.SpanFromContext(batch[1].GetContext()) 39 | assert.Equal(t, "4bf92f3577b34da6a3ce929d0e0e4736", spanTwo.SpanContext().TraceID().String()) 40 | assert.Equal(t, "00f067aa0ba902b7", spanTwo.SpanContext().SpanID().String()) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /internal/tracing/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package tracing implements utility functions for interacting with a global 4 | // tracing system. Currently this system uses the opentelemetry APIs. 5 | package tracing 6 | -------------------------------------------------------------------------------- /internal/tracing/span.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package tracing 4 | 5 | import ( 6 | "context" 7 | 8 | "go.opentelemetry.io/otel" 9 | "go.opentelemetry.io/otel/attribute" 10 | "go.opentelemetry.io/otel/propagation" 11 | "go.opentelemetry.io/otel/trace" 12 | ) 13 | 14 | // Span abstracts the span type of our global tracing system in order to allow 15 | // it to be replaced in future. 16 | type Span struct { 17 | ctx context.Context 18 | w trace.Span 19 | } 20 | 21 | // OtelSpan creates a common span from the open telemetry package. 22 | func OtelSpan(ctx context.Context, s trace.Span) *Span { 23 | if s == nil { 24 | return nil 25 | } 26 | return &Span{ctx: ctx, w: s} 27 | } 28 | 29 | func (s *Span) unwrap() trace.Span { 30 | if s == nil { 31 | return nil 32 | } 33 | return s.w 34 | } 35 | 36 | // LogKV adds log key/value pairs to the span. 37 | func (s *Span) LogKV(name string, kv ...string) { 38 | if s == nil { 39 | return 40 | } 41 | var attrs []attribute.KeyValue 42 | for i := 0; i < len(kv)-1; i += 2 { 43 | attrs = append(attrs, attribute.String(kv[i], kv[i+1])) 44 | } 45 | s.w.AddEvent(name, trace.WithAttributes(attrs...)) 46 | } 47 | 48 | // SetTag sets a given tag to a value. 49 | func (s *Span) SetTag(key, value string) { 50 | if s == nil { 51 | return 52 | } 53 | s.w.SetAttributes(attribute.String(key, value)) 54 | } 55 | 56 | // Finish the span. 57 | func (s *Span) Finish() { 58 | if s == nil { 59 | return 60 | } 61 | s.w.End() 62 | } 63 | 64 | // TextMap attempts to inject a span into a map object in text map format. 65 | func (s *Span) TextMap() (map[string]any, error) { 66 | if s == nil { 67 | return nil, nil 68 | } 69 | c := propagation.MapCarrier{} 70 | otel.GetTextMapPropagator().Inject(s.ctx, c) 71 | 72 | spanMapGeneric := make(map[string]any, len(c)) 73 | for k, v := range c { 74 | spanMapGeneric[k] = v 75 | } 76 | return spanMapGeneric, nil 77 | } 78 | -------------------------------------------------------------------------------- /internal/tracing/v2/otel_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package tracing 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "go.opentelemetry.io/otel" 10 | "go.opentelemetry.io/otel/propagation" 11 | "go.opentelemetry.io/otel/trace" 12 | "go.opentelemetry.io/otel/trace/noop" 13 | 14 | "github.com/redpanda-data/benthos/v4/public/service" 15 | ) 16 | 17 | func TestInitSpansFromParentTextMap(t *testing.T) { 18 | t.Run("it will update the context for each message in the batch", func(t *testing.T) { 19 | textMap := map[string]any{ 20 | "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", 21 | } 22 | 23 | msgOne := service.NewMessage([]byte("hello")) 24 | msgTwo := service.NewMessage([]byte("world")) 25 | 26 | batch := service.MessageBatch{msgOne, msgTwo} 27 | 28 | otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{})) 29 | tp := noop.NewTracerProvider() 30 | 31 | err := InitSpansFromParentTextMap(tp, "test", textMap, batch) 32 | assert.NoError(t, err) 33 | 34 | spanOne := trace.SpanFromContext(batch[0].Context()) 35 | assert.Equal(t, "4bf92f3577b34da6a3ce929d0e0e4736", spanOne.SpanContext().TraceID().String()) 36 | assert.Equal(t, "00f067aa0ba902b7", spanOne.SpanContext().SpanID().String()) 37 | 38 | spanTwo := trace.SpanFromContext(batch[1].Context()) 39 | assert.Equal(t, "4bf92f3577b34da6a3ce929d0e0e4736", spanTwo.SpanContext().TraceID().String()) 40 | assert.Equal(t, "00f067aa0ba902b7", spanTwo.SpanContext().SpanID().String()) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /internal/tracing/v2/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package tracing implements utility functions for interacting with a global 4 | // tracing system. Currently this system uses the opentelemetry APIs. 5 | package tracing 6 | -------------------------------------------------------------------------------- /internal/tracing/v2/span.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package tracing 4 | 5 | import ( 6 | "context" 7 | 8 | "go.opentelemetry.io/otel" 9 | "go.opentelemetry.io/otel/attribute" 10 | "go.opentelemetry.io/otel/propagation" 11 | "go.opentelemetry.io/otel/trace" 12 | ) 13 | 14 | // Span abstracts the span type of our global tracing system in order to allow 15 | // it to be replaced in future. 16 | type Span struct { 17 | ctx context.Context 18 | w trace.Span 19 | } 20 | 21 | // OtelSpan creates a common span from the open telemetry package. 22 | func OtelSpan(ctx context.Context, s trace.Span) *Span { 23 | if s == nil { 24 | return nil 25 | } 26 | return &Span{ctx: ctx, w: s} 27 | } 28 | 29 | func (s *Span) unwrap() trace.Span { 30 | if s == nil { 31 | return nil 32 | } 33 | return s.w 34 | } 35 | 36 | // LogKV adds log key/value pairs to the span. 37 | func (s *Span) LogKV(name string, kv ...string) { 38 | if s == nil { 39 | return 40 | } 41 | var attrs []attribute.KeyValue 42 | for i := 0; i < len(kv)-1; i += 2 { 43 | attrs = append(attrs, attribute.String(kv[i], kv[i+1])) 44 | } 45 | s.w.AddEvent(name, trace.WithAttributes(attrs...)) 46 | } 47 | 48 | // SetTag sets a given tag to a value. 49 | func (s *Span) SetTag(key, value string) { 50 | if s == nil { 51 | return 52 | } 53 | s.w.SetAttributes(attribute.String(key, value)) 54 | } 55 | 56 | // Finish the span. 57 | func (s *Span) Finish() { 58 | if s == nil { 59 | return 60 | } 61 | s.w.End() 62 | } 63 | 64 | // TextMap attempts to inject a span into a map object in text map format. 65 | func (s *Span) TextMap() (map[string]any, error) { 66 | if s == nil { 67 | return nil, nil 68 | } 69 | c := propagation.MapCarrier{} 70 | otel.GetTextMapPropagator().Inject(s.ctx, c) 71 | 72 | spanMapGeneric := make(map[string]any, len(c)) 73 | for k, v := range c { 74 | spanMapGeneric[k] = v 75 | } 76 | return spanMapGeneric, nil 77 | } 78 | -------------------------------------------------------------------------------- /internal/transaction/result_store_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package transaction 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/message" 10 | ) 11 | 12 | func TestResultStore(t *testing.T) { 13 | impl := &resultStoreImpl{} 14 | ctx := context.WithValue(t.Context(), ResultStoreKey, impl) 15 | msg := message.Batch{ 16 | message.WithContext(ctx, message.NewPart([]byte("foo"))), 17 | message.NewPart([]byte("bar")), 18 | } 19 | 20 | impl.Add(msg) 21 | results := impl.Get() 22 | if len(results) != 1 { 23 | t.Fatalf("Wrong count of result batches: %v", len(results)) 24 | } 25 | if results[0].Len() != 2 { 26 | t.Fatalf("Wrong count of messages: %v", results[0].Len()) 27 | } 28 | if exp, act := "foo", string(results[0].Get(0).AsBytes()); exp != act { 29 | t.Errorf("Wrong message contents: %v != %v", act, exp) 30 | } 31 | if exp, act := "bar", string(results[0].Get(1).AsBytes()); exp != act { 32 | t.Errorf("Wrong message contents: %v != %v", act, exp) 33 | } 34 | if store := message.GetContext(results[0].Get(0)).Value(ResultStoreKey); store != nil { 35 | t.Error("Unexpected nested result store") 36 | } 37 | 38 | impl.Clear() 39 | if exp, act := len(impl.Get()), 0; exp != act { 40 | t.Errorf("Unexpected count of stored messages: %v != %v", act, exp) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /internal/value/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package value 4 | 5 | import ( 6 | "bytes" 7 | "fmt" 8 | ) 9 | 10 | // TypeError represents an error where a value of a type was required for a 11 | // function, method or operator but instead a different type was found. 12 | type TypeError struct { 13 | From string 14 | Expected []Type 15 | Actual Type 16 | Value string 17 | } 18 | 19 | // Error implements the standard error interface for TypeError. 20 | func (t *TypeError) Error() string { 21 | var errStr bytes.Buffer 22 | if len(t.Expected) > 0 { 23 | errStr.WriteString("expected ") 24 | for i, exp := range t.Expected { 25 | if i > 0 { 26 | if len(t.Expected) > 2 && i < (len(t.Expected)-1) { 27 | errStr.WriteString(", ") 28 | } else { 29 | errStr.WriteString(" or ") 30 | } 31 | } 32 | errStr.WriteString(string(exp)) 33 | } 34 | errStr.WriteString(" value") 35 | } else { 36 | errStr.WriteString("unexpected value") 37 | } 38 | 39 | fmt.Fprintf(&errStr, ", got %v", t.Actual) 40 | 41 | if t.From != "" { 42 | fmt.Fprintf(&errStr, " from %v", t.From) 43 | } 44 | 45 | if t.Value != "" { 46 | fmt.Fprintf(&errStr, " (%v)", t.Value) 47 | } 48 | 49 | return errStr.String() 50 | } 51 | 52 | // NewTypeError creates a new type error. 53 | func NewTypeError(value any, exp ...Type) *TypeError { 54 | return NewTypeErrorFrom("", value, exp...) 55 | } 56 | 57 | // NewTypeErrorFrom creates a new type error with an annotation of the query 58 | // that provided the wrong type. 59 | func NewTypeErrorFrom(from string, value any, exp ...Type) *TypeError { 60 | valueStr := "" 61 | valueType := ITypeOf(value) 62 | switch valueType { 63 | case TString: 64 | valueStr = fmt.Sprintf(`"%v"`, value) 65 | case TBool, TNumber: 66 | valueStr = fmt.Sprintf("%v", value) 67 | } 68 | return &TypeError{ 69 | From: from, 70 | Expected: exp, 71 | Actual: valueType, 72 | Value: valueStr, 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /internal/value/type_helpers_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package value 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestIToBytes(t *testing.T) { 13 | vb := IToBytes(uint64(12345)) 14 | assert.Equal(t, "12345", string(vb)) 15 | 16 | vb = IToBytes(true) 17 | assert.Equal(t, "true", string(vb)) 18 | 19 | vb = IToBytes(float64(1.2345)) 20 | assert.Equal(t, "1.2345", string(vb)) 21 | 22 | vb = IToBytes(float64(1.234567891234567891234567)) 23 | assert.Equal(t, "1.234567891234568", string(vb)) 24 | 25 | vb = IToBytes(float64(1.23 * 4.567 * 1_000_000_000)) 26 | assert.Equal(t, "5.61741e+09", string(vb)) 27 | } 28 | 29 | func TestIToInt(t *testing.T) { 30 | for _, test := range []struct { 31 | in any 32 | out int64 33 | errContains string 34 | }{ 35 | { 36 | in: 123.0, 37 | out: 123, 38 | }, 39 | { 40 | in: 123.456, 41 | errContains: "contains decimals and therefore cannot be cast as a", 42 | }, 43 | { 44 | in: "123", 45 | out: 123, 46 | }, 47 | { 48 | in: MaxInt, 49 | out: int64(MaxInt), 50 | }, 51 | { 52 | in: MinInt, 53 | out: MinInt, 54 | }, 55 | { 56 | in: float64(MaxInt) + 10000, 57 | errContains: "value is too large to be cast as a", 58 | }, 59 | { 60 | in: float64(MinInt) - 10000, 61 | errContains: "value is too small to be cast as a", 62 | }, 63 | } { 64 | i, err := IToInt(test.in) 65 | if test.errContains != "" { 66 | require.Error(t, err, "value: %v", i) 67 | assert.Contains(t, err.Error(), test.errContains) 68 | } else { 69 | require.NoError(t, err) 70 | assert.Equal(t, test.out, i) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /public/bloblang/environment_unwrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package bloblang 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/bloblang" 7 | ) 8 | 9 | type environmentUnwrapper struct { 10 | child *bloblang.Environment 11 | } 12 | 13 | func (e environmentUnwrapper) Unwrap() *bloblang.Environment { 14 | return e.child 15 | } 16 | 17 | // XUnwrapper is for internal use only, do not use this. 18 | func (e *Environment) XUnwrapper() any { 19 | return environmentUnwrapper{child: e.env} 20 | } 21 | 22 | // XWrapEnvironment is for internal use only, do not use this. 23 | func XWrapEnvironment(v *bloblang.Environment) *Environment { 24 | return &Environment{env: v} 25 | } 26 | -------------------------------------------------------------------------------- /public/bloblang/executor_unwrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package bloblang 4 | 5 | import "github.com/redpanda-data/benthos/v4/internal/bloblang/mapping" 6 | 7 | type executorUnwrapper struct { 8 | child *mapping.Executor 9 | } 10 | 11 | func (e executorUnwrapper) Unwrap() *mapping.Executor { 12 | return e.child 13 | } 14 | 15 | // XUnwrapper is for internal use only, do not use this. 16 | func (e *Executor) XUnwrapper() any { 17 | return executorUnwrapper{child: e.exec} 18 | } 19 | -------------------------------------------------------------------------------- /public/bloblang/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package bloblang provides high level APIs for registering custom Bloblang 4 | // plugins, as well as for parsing and executing Bloblang mappings. 5 | // 6 | // For a video guide on Benthos plugins check out: https://youtu.be/uH6mKw-Ly0g 7 | // And an example repo containing component plugins and tests can be found at: 8 | // https://github.com/benthosdev/benthos-plugin-example 9 | // 10 | // Plugins can either be registered globally, and will be accessible to any 11 | // component parsing Bloblang expressions in the executable, or they can be 12 | // registered as part of an isolated environment. 13 | package bloblang 14 | -------------------------------------------------------------------------------- /public/bloblang/parse_error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package bloblang 4 | 5 | import "github.com/redpanda-data/benthos/v4/internal/bloblang/parser" 6 | 7 | // ParseError is a structured error type for Bloblang parser errors that 8 | // provides access to information such as the line and column where the error 9 | // occurred. 10 | type ParseError struct { 11 | Line int 12 | Column int 13 | 14 | input []rune 15 | iErr *parser.Error 16 | } 17 | 18 | // Error returns a single line error string. 19 | func (p *ParseError) Error() string { 20 | return p.iErr.Error() 21 | } 22 | 23 | // ErrorMultiline returns an error string spanning multiple lines that provides 24 | // a cleaner view of the specific error. 25 | func (p *ParseError) ErrorMultiline() string { 26 | return p.iErr.ErrorAtPositionStructured("", p.input) 27 | } 28 | 29 | func internalToPublicParserError(input []rune, p *parser.Error) *ParseError { 30 | pErr := &ParseError{ 31 | input: input, 32 | iErr: p, 33 | } 34 | pErr.Line, pErr.Column = parser.LineAndColOf(input, p.Input) 35 | return pErr 36 | } 37 | -------------------------------------------------------------------------------- /public/bloblang/parse_error_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package bloblang 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestParseErrors(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | mapping string 16 | expLine int 17 | expCol int 18 | errContains string 19 | }{ 20 | { 21 | name: "Bad assignment error", 22 | mapping: ` 23 | root = 24 | # there wasn't a value! 25 | `, 26 | expLine: 2, 27 | expCol: 8, 28 | errContains: "expected query", 29 | }, 30 | { 31 | name: "Bad function args error", 32 | mapping: ` 33 | root.foo = this.foo 34 | root.bar = this.bar.uppercase().replace("something)`, 35 | expLine: 3, 36 | expCol: 52, 37 | errContains: "expected end quote", 38 | }, 39 | } 40 | 41 | for _, test := range tests { 42 | test := test 43 | t.Run(test.name, func(t *testing.T) { 44 | _, err := Parse(test.mapping) 45 | require.Error(t, err) 46 | 47 | pErr, ok := err.(*ParseError) 48 | require.True(t, ok) 49 | 50 | assert.Equal(t, test.expLine, pErr.Line) 51 | assert.Equal(t, test.expCol, pErr.Column) 52 | assert.Contains(t, pErr.ErrorMultiline(), test.errContains, pErr.ErrorMultiline()) 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /public/components/io/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package io contains component implementations that have a small dependency 4 | // footprint (mostly standard library) and interact with external systems via 5 | // the filesystem and/or network sockets. 6 | // 7 | // EXPERIMENTAL: The specific components excluded by this package may change 8 | // outside of major version releases. This means we may choose to remove certain 9 | // plugins if we determine that their dependencies are likely to interfere with 10 | // the goals of this package. 11 | package io 12 | 13 | import ( 14 | // Import only io packages. 15 | _ "github.com/redpanda-data/benthos/v4/internal/impl/io" 16 | ) 17 | -------------------------------------------------------------------------------- /public/components/pure/extended/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package extended contains component implementations that have a larger 4 | // dependency footprint but do not interact with external systems (so an 5 | // extension of pure components) 6 | // 7 | // EXPERIMENTAL: The specific components excluded by this package may change 8 | // outside of major version releases. This means we may choose to remove certain 9 | // plugins if we determine that their dependencies are likely to interfere with 10 | // the goals of this package. 11 | package extended 12 | 13 | import ( 14 | // Import pure but larger packages. 15 | _ "github.com/redpanda-data/benthos/v4/internal/impl/pure/extended" 16 | ) 17 | -------------------------------------------------------------------------------- /public/components/pure/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package pure imports all component implementations that are pure, in that 4 | // they do not interact with external systems. This includes all base component 5 | // types such as brokers and is likely necessary as a base for all builds. 6 | // 7 | // EXPERIMENTAL: The specific components excluded by this package may change 8 | // outside of major version releases. This means we may choose to remove certain 9 | // plugins if we determine that their dependencies are likely to interfere with 10 | // the goals of this package. 11 | package pure 12 | 13 | import ( 14 | // Import only pure packages. 15 | _ "github.com/redpanda-data/benthos/v4/internal/impl/pure" 16 | ) 17 | -------------------------------------------------------------------------------- /public/service/config_backoff_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "testing" 7 | "time" 8 | 9 | "github.com/cenkalti/backoff/v4" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestConfigBackOff(t *testing.T) { 15 | spec := NewConfigSpec(). 16 | Field(NewBackOffField("a", true, nil)) 17 | 18 | parsedConfig, err := spec.ParseYAML(` 19 | a: 20 | max_interval: 300s 21 | `, nil) 22 | require.NoError(t, err) 23 | 24 | _, err = parsedConfig.FieldBackOff("b") 25 | require.Error(t, err) 26 | 27 | bConf, err := parsedConfig.FieldBackOff("a") 28 | require.NoError(t, err) 29 | 30 | assert.Equal(t, time.Millisecond*500, bConf.InitialInterval) 31 | assert.Equal(t, time.Second*300, bConf.MaxInterval) 32 | assert.Equal(t, time.Minute*1, bConf.MaxElapsedTime) 33 | } 34 | 35 | func TestConfigBackOffCustomDefaults(t *testing.T) { 36 | defaults := backoff.NewExponentialBackOff() 37 | defaults.InitialInterval = time.Minute 38 | defaults.MaxInterval = time.Minute * 5 39 | defaults.MaxElapsedTime = time.Hour * 6 40 | 41 | spec := NewConfigSpec(). 42 | Field(NewBackOffField("a", false, defaults)) 43 | 44 | parsedConfig, err := spec.ParseYAML(` 45 | a: 46 | max_interval: 300s 47 | `, nil) 48 | require.NoError(t, err) 49 | 50 | _, err = parsedConfig.FieldBackOff("b") 51 | require.Error(t, err) 52 | 53 | bConf, err := parsedConfig.FieldBackOff("a") 54 | require.NoError(t, err) 55 | 56 | assert.Equal(t, time.Minute, bConf.InitialInterval) 57 | assert.Equal(t, time.Second*300, bConf.MaxInterval) 58 | assert.Equal(t, time.Hour*6, bConf.MaxElapsedTime) 59 | } 60 | -------------------------------------------------------------------------------- /public/service/config_bloblang.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/docs" 10 | "github.com/redpanda-data/benthos/v4/public/bloblang" 11 | ) 12 | 13 | // NewBloblangField defines a new config field that describes a Bloblang mapping 14 | // string. It is then possible to extract a *bloblang.Executor from the 15 | // resulting parsed config with the method FieldBloblang. 16 | func NewBloblangField(name string) *ConfigField { 17 | tf := docs.FieldBloblang(name, "") 18 | return &ConfigField{field: tf} 19 | } 20 | 21 | // FieldBloblang accesses a field from a parsed config that was defined with 22 | // NewBloblangField and returns either a *bloblang.Executor or an error if the 23 | // mapping was invalid. 24 | func (p *ParsedConfig) FieldBloblang(path ...string) (*bloblang.Executor, error) { 25 | v, exists := p.i.Field(path...) 26 | if !exists { 27 | return nil, fmt.Errorf("field '%v' was not found in the config", strings.Join(path, ".")) 28 | } 29 | 30 | str, ok := v.(string) 31 | if !ok { 32 | return nil, fmt.Errorf("expected field '%v' to be a string, got %T", strings.Join(path, "."), v) 33 | } 34 | 35 | exec, err := bloblang.XWrapEnvironment(p.mgr.BloblEnvironment()).Parse(str) 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to parse bloblang mapping '%v': %v", strings.Join(path, "."), err) 38 | } 39 | return exec, nil 40 | } 41 | -------------------------------------------------------------------------------- /public/service/config_bloblang_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestConfigBloblang(t *testing.T) { 13 | spec := NewConfigSpec(). 14 | Field(NewBloblangField("a")). 15 | Field(NewStringField("b")) 16 | 17 | parsedConfig, err := spec.ParseYAML(` 18 | a: 'root = this.uppercase()' 19 | b: 'root = this.filter(' 20 | `, nil) 21 | require.NoError(t, err) 22 | 23 | _, err = parsedConfig.FieldBloblang("b") 24 | require.Error(t, err) 25 | 26 | _, err = parsedConfig.FieldBloblang("c") 27 | require.Error(t, err) 28 | 29 | exec, err := parsedConfig.FieldBloblang("a") 30 | require.NoError(t, err) 31 | 32 | res, err := exec.Query("hello world") 33 | require.NoError(t, err) 34 | assert.Equal(t, "HELLO WORLD", res) 35 | } 36 | -------------------------------------------------------------------------------- /public/service/config_interpolated_string_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestFieldInterpolatedStringList(t *testing.T) { 12 | conf := ` 13 | listfield: 14 | - hello ${! json("name").uppercase() } 15 | - see you in ${! meta("ttl_days") } days 16 | ` 17 | 18 | spec := NewConfigSpec().Field(NewInterpolatedStringListField("listfield")) 19 | env := NewEnvironment() 20 | parsed, err := spec.ParseYAML(conf, env) 21 | require.NoError(t, err) 22 | 23 | field, err := parsed.FieldInterpolatedStringList("listfield") 24 | require.NoError(t, err) 25 | 26 | msg := NewMessage([]byte(`{"name": "world"}`)) 27 | msg.MetaSet("ttl_days", "3") 28 | 29 | var out []string 30 | for _, f := range field { 31 | str, err := f.TryString(msg) 32 | require.NoError(t, err) 33 | 34 | out = append(out, str) 35 | } 36 | 37 | require.Equal(t, []string{"hello WORLD", "see you in 3 days"}, out) 38 | } 39 | 40 | func TestFieldInterpolatedStringList_InvalidInterpolation(t *testing.T) { 41 | conf := ` 42 | listfield: 43 | - hello ${! json("name")$$uppercas } 44 | - see you in ${! meta("ttl_days") } days 45 | ` 46 | 47 | spec := NewConfigSpec().Field(NewInterpolatedStringListField("listfield")) 48 | env := NewEnvironment() 49 | parsed, err := spec.ParseYAML(conf, env) 50 | require.NoError(t, err) 51 | 52 | _, err = parsed.FieldInterpolatedStringList("listfield") 53 | require.ErrorIs(t, err, errInvalidInterpolation) 54 | } 55 | -------------------------------------------------------------------------------- /public/service/config_max_in_flight.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | // NewOutputMaxInFlightField creates a common field for determining the maximum 6 | // number of in-flight messages an output should allow. This function is a 7 | // short-hand way of creating an integer field with the common name 8 | // max_in_flight, with a typical default of 64. 9 | func NewOutputMaxInFlightField() *ConfigField { 10 | return NewIntField("max_in_flight"). 11 | Description("The maximum number of messages to have in flight at a given time. Increase this to improve throughput."). 12 | Default(64) 13 | } 14 | 15 | // FieldMaxInFlight accesses a field from a parsed config that was defined 16 | // either with NewInputMaxInFlightField or NewOutputMaxInFlightField, and 17 | // returns either an integer or an error if the value was invalid. 18 | func (p *ParsedConfig) FieldMaxInFlight() (int, error) { 19 | return p.FieldInt("max_in_flight") 20 | } 21 | -------------------------------------------------------------------------------- /public/service/config_metadata_filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestMetadataFilterConfig(t *testing.T) { 13 | configSpec := NewConfigSpec().Field(NewMetadataFilterField("foo")) 14 | 15 | configParsed, err := configSpec.ParseYAML(` 16 | foo: 17 | include_prefixes: [ foo_ ] 18 | include_patterns: [ "meo?w" ] 19 | `, nil) 20 | require.NoError(t, err) 21 | 22 | _, err = configParsed.FieldMetadataFilter("bar") 23 | require.Error(t, err) 24 | 25 | f, err := configParsed.FieldMetadataFilter("foo") 26 | require.NoError(t, err) 27 | 28 | msg := NewMessage(nil) 29 | msg.MetaSet("foo_1", "foo value") 30 | msg.MetaSet("bar_1", "bar value") 31 | msg.MetaSet("baz_1", "baz value") 32 | msg.MetaSet("woof_1", "woof value") 33 | msg.MetaSet("meow_1", "meow value") 34 | msg.MetaSet("mew_1", "mew value") 35 | 36 | seen := map[string]string{} 37 | require.NoError(t, f.Walk(msg, func(key, value string) error { 38 | seen[key] = value 39 | return nil 40 | })) 41 | 42 | assert.Equal(t, map[string]string{ 43 | "foo_1": "foo value", 44 | "meow_1": "meow value", 45 | "mew_1": "mew value", 46 | }, seen) 47 | } 48 | -------------------------------------------------------------------------------- /public/service/config_scanner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/bundle" 10 | "github.com/redpanda-data/benthos/v4/internal/component/scanner" 11 | "github.com/redpanda-data/benthos/v4/internal/docs" 12 | ) 13 | 14 | // NewScannerField defines a new scanner field, it is then possible to extract 15 | // an OwnedScannerCreator from the resulting parsed config with the method 16 | // FieldScanner. 17 | func NewScannerField(name string) *ConfigField { 18 | return &ConfigField{ 19 | field: docs.FieldScanner(name, ""), 20 | } 21 | } 22 | 23 | func ownedScannerCreatorFromConfAny(mgr bundle.NewManagement, field any) (*OwnedScannerCreator, error) { 24 | pluginConf, err := scanner.FromAny(mgr.Environment(), field) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | irdr, err := mgr.NewScanner(pluginConf) 30 | if err != nil { 31 | return nil, err 32 | } 33 | return &OwnedScannerCreator{rdr: irdr}, nil 34 | } 35 | 36 | // FieldScanner accesses a field from a parsed config that was defined with 37 | // NewScannerField and returns an OwnedScannerCreator, or an error if the 38 | // configuration was invalid. 39 | func (p *ParsedConfig) FieldScanner(path ...string) (*OwnedScannerCreator, error) { 40 | field, exists := p.i.Field(path...) 41 | if !exists { 42 | return nil, fmt.Errorf("field '%v' was not found in the config", strings.Join(path, ".")) 43 | } 44 | return ownedScannerCreatorFromConfAny(p.mgr.IntoPath(path...), field) 45 | } 46 | -------------------------------------------------------------------------------- /public/service/config_util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/bundle" 7 | "github.com/redpanda-data/benthos/v4/internal/docs" 8 | "github.com/redpanda-data/benthos/v4/internal/value" 9 | ) 10 | 11 | type fieldUnwrapper struct { 12 | child docs.FieldSpec 13 | } 14 | 15 | func (f fieldUnwrapper) Unwrap() docs.FieldSpec { 16 | return f.child 17 | } 18 | 19 | // XUnwrapper is for internal use only, do not use this. 20 | func (c *ConfigField) XUnwrapper() any { 21 | return fieldUnwrapper{child: c.field} 22 | } 23 | 24 | func extractConfig( 25 | nm bundle.NewManagement, 26 | spec *ConfigSpec, 27 | componentName string, 28 | pluginConfig any, 29 | ) (*ParsedConfig, error) { 30 | // All nested fields are under the namespace of the component type, and 31 | // therefore we need to namespace the manager such that metrics and logs 32 | // from nested core component types are corrected labelled. 33 | if nm != nil { 34 | nm = nm.IntoPath(componentName) 35 | } 36 | 37 | if pluginConfig == nil { 38 | if spec.component.Config.Default != nil { 39 | pluginConfig = value.IClone(*spec.component.Config.Default) 40 | } else if len(spec.component.Config.Children) > 0 { 41 | pluginConfig = map[string]any{} 42 | } 43 | } 44 | return spec.configFromAny(nm, pluginConfig) 45 | } 46 | -------------------------------------------------------------------------------- /public/service/environment_schema.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "github.com/redpanda-data/benthos/v4/internal/config/schema" 7 | "github.com/redpanda-data/benthos/v4/internal/cuegen" 8 | ) 9 | 10 | // EnvironmentSchema represents a schema definition for all components 11 | // registered within the environment. 12 | type EnvironmentSchema struct { 13 | s schema.Full 14 | } 15 | 16 | // GenerateSchema creates a new EnvironmentSchema. 17 | func (e *Environment) GenerateSchema(version, dateBuilt string) *EnvironmentSchema { 18 | schema := schema.New(version, dateBuilt, e.internal, e.getBloblangParserEnv()) 19 | return &EnvironmentSchema{s: schema} 20 | } 21 | 22 | // ReduceToStatus removes all components that aren't of the given stability 23 | // status. 24 | func (e *EnvironmentSchema) ReduceToStatus(status string) *EnvironmentSchema { 25 | e.s.ReduceToStatus(status) 26 | return e 27 | } 28 | 29 | // Minimise removes all documentation from the schema definition. 30 | func (e *EnvironmentSchema) Minimise() *EnvironmentSchema { 31 | e.s.Scrub() 32 | return e 33 | } 34 | 35 | // ToCUE attempts to generate a CUE schema. 36 | func (e *EnvironmentSchema) ToCUE() ([]byte, error) { 37 | return cuegen.GenerateSchema(e.s) 38 | } 39 | 40 | // XFlattened returns a generic structure of the schema as a map of component 41 | // types to component names. 42 | // 43 | // Experimental: This method is experimental and therefore could be changed 44 | // outside of major version releases. 45 | func (e *EnvironmentSchema) XFlattened() map[string][]string { 46 | return e.s.Flattened() 47 | } 48 | -------------------------------------------------------------------------------- /public/service/example_rate_limit_plugin_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service_test 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "math/rand" 9 | "time" 10 | 11 | "github.com/redpanda-data/benthos/v4/public/service" 12 | 13 | // Import only pure Benthos components, switch with `components/all` for all 14 | // standard components. 15 | _ "github.com/redpanda-data/benthos/v4/public/components/pure" 16 | ) 17 | 18 | type RandomRateLimit struct { 19 | max time.Duration 20 | } 21 | 22 | func (r *RandomRateLimit) Access(context.Context) (time.Duration, error) { 23 | return time.Duration(rand.Int() % int(r.max)), nil 24 | } 25 | 26 | func (r *RandomRateLimit) Close(ctx context.Context) error { 27 | return nil 28 | } 29 | 30 | // This example demonstrates how to create a rate limit plugin, which is 31 | // configured by providing a struct containing the fields to be parsed from 32 | // within the Benthos configuration. 33 | func Example_rateLimitPlugin() { 34 | configSpec := service.NewConfigSpec(). 35 | Summary("A rate limit that's pretty much just random."). 36 | Description("I guess this isn't really that useful, sorry."). 37 | Field(service.NewStringField("maximum_duration").Default("1s")) 38 | 39 | constructor := func(conf *service.ParsedConfig, mgr *service.Resources) (service.RateLimit, error) { 40 | maxDurStr, err := conf.FieldString("maximum_duration") 41 | if err != nil { 42 | return nil, err 43 | } 44 | maxDuration, err := time.ParseDuration(maxDurStr) 45 | if err != nil { 46 | return nil, fmt.Errorf("invalid max duration: %w", err) 47 | } 48 | return &RandomRateLimit{maxDuration}, nil 49 | } 50 | 51 | err := service.RegisterRateLimit("random", configSpec, constructor) 52 | if err != nil { 53 | panic(err) 54 | } 55 | 56 | // And then execute Benthos with: 57 | // service.RunCLI(context.Background()) 58 | } 59 | -------------------------------------------------------------------------------- /public/service/linttype_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=LintType"; DO NOT EDIT. 2 | 3 | package service 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[LintCustom-0] 12 | _ = x[LintFailedRead-1] 13 | _ = x[LintMissingEnvVar-2] 14 | _ = x[LintInvalidOption-3] 15 | _ = x[LintBadLabel-4] 16 | _ = x[LintMissingLabel-5] 17 | _ = x[LintDuplicateLabel-6] 18 | _ = x[LintBadBloblang-7] 19 | _ = x[LintShouldOmit-8] 20 | _ = x[LintComponentMissing-9] 21 | _ = x[LintComponentNotFound-10] 22 | _ = x[LintUnknown-11] 23 | _ = x[LintMissing-12] 24 | _ = x[LintExpectedArray-13] 25 | _ = x[LintExpectedObject-14] 26 | _ = x[LintExpectedScalar-15] 27 | _ = x[LintDeprecated-16] 28 | } 29 | 30 | const _LintType_name = "LintCustomLintFailedReadLintMissingEnvVarLintInvalidOptionLintBadLabelLintMissingLabelLintDuplicateLabelLintBadBloblangLintShouldOmitLintComponentMissingLintComponentNotFoundLintUnknownLintMissingLintExpectedArrayLintExpectedObjectLintExpectedScalarLintDeprecated" 31 | 32 | var _LintType_index = [...]uint16{0, 10, 24, 41, 58, 70, 86, 104, 119, 133, 153, 174, 185, 196, 213, 231, 249, 263} 33 | 34 | func (i LintType) String() string { 35 | if i < 0 || i >= LintType(len(_LintType_index)-1) { 36 | return "LintType(" + strconv.FormatInt(int64(i), 10) + ")" 37 | } 38 | return _LintType_name[_LintType_index[i]:_LintType_index[i+1]] 39 | } 40 | -------------------------------------------------------------------------------- /public/service/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package service provides a high level API for registering custom plugin 4 | // components and executing either a standard Benthos CLI, or programmatically 5 | // building isolated pipelines with a StreamBuilder API. 6 | // 7 | // For a video guide on Benthos plugins check out: https://youtu.be/uH6mKw-Ly0g 8 | // And an example repo containing component plugins and tests can be found at: 9 | // https://github.com/benthosdev/benthos-plugin-example 10 | // 11 | // In order to add custom Bloblang functions and methods use the 12 | // ./public/bloblang package. 13 | package service 14 | 15 | import ( 16 | "context" 17 | ) 18 | 19 | // Closer is implemented by components that support stopping and cleaning up 20 | // their underlying resources. 21 | type Closer interface { 22 | // Close the component, blocks until either the underlying resources are 23 | // cleaned up or the context is cancelled. Returns an error if the context 24 | // is cancelled. 25 | Close(ctx context.Context) error 26 | } 27 | -------------------------------------------------------------------------------- /public/service/rate_limit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "context" 7 | "time" 8 | 9 | "github.com/redpanda-data/benthos/v4/internal/component/metrics" 10 | "github.com/redpanda-data/benthos/v4/internal/component/ratelimit" 11 | ) 12 | 13 | // RateLimit is an interface implemented by Benthos rate limits. 14 | type RateLimit interface { 15 | // Access the rate limited resource. Returns a duration or an error if the 16 | // rate limit check fails. The returned duration is either zero (meaning the 17 | // resource may be accessed) or a reasonable length of time to wait before 18 | // requesting again. 19 | Access(context.Context) (time.Duration, error) 20 | 21 | Closer 22 | } 23 | 24 | //------------------------------------------------------------------------------ 25 | 26 | func newAirGapRateLimit(c RateLimit, stats metrics.Type) ratelimit.V1 { 27 | return ratelimit.MetricsForRateLimit(c, stats) 28 | } 29 | 30 | //------------------------------------------------------------------------------ 31 | 32 | // Implements RateLimit around a types.RateLimit. 33 | type reverseAirGapRateLimit struct { 34 | r ratelimit.V1 35 | } 36 | 37 | func newReverseAirGapRateLimit(r ratelimit.V1) *reverseAirGapRateLimit { 38 | return &reverseAirGapRateLimit{r} 39 | } 40 | 41 | func (a *reverseAirGapRateLimit) Access(ctx context.Context) (time.Duration, error) { 42 | return a.r.Access(ctx) 43 | } 44 | 45 | func (a *reverseAirGapRateLimit) Close(ctx context.Context) error { 46 | return a.r.Close(ctx) 47 | } 48 | -------------------------------------------------------------------------------- /public/service/servicetest/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package servicetest provides functions and utilities that might be useful for 4 | // testing custom Benthos builds. 5 | package servicetest 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "os" 12 | 13 | "github.com/redpanda-data/benthos/v4/internal/cli" 14 | "github.com/redpanda-data/benthos/v4/internal/cli/common" 15 | ) 16 | 17 | // RunCLIWithArgs executes Benthos as a CLI with an explicit set of arguments. 18 | // This is useful for testing commands without needing to modify os.Args. 19 | // 20 | // This call blocks until either: 21 | // 22 | // 1. The service shuts down gracefully due to the inputs closing 23 | // 2. A termination signal is received 24 | // 3. The provided context has a deadline that is reached, triggering graceful termination 25 | // 4. The provided context is cancelled (WARNING, this prevents graceful termination) 26 | // 27 | // Deprecated: Use the service.CLIOptSetArgs opt func instead. 28 | func RunCLIWithArgs(ctx context.Context, args ...string) { 29 | if err := cli.App(common.NewCLIOpts(cli.Version, cli.DateBuilt)).RunContext(ctx, args); err != nil { 30 | var cerr *common.ErrExitCode 31 | if errors.As(err, &cerr) { 32 | os.Exit(cerr.Code) 33 | } 34 | fmt.Fprintln(os.Stderr, err.Error()) 35 | os.Exit(1) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /public/service/servicetest/service_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package servicetest_test 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | 16 | _ "github.com/redpanda-data/benthos/v4/public/components/io" 17 | _ "github.com/redpanda-data/benthos/v4/public/components/pure" 18 | "github.com/redpanda-data/benthos/v4/public/service/servicetest" 19 | ) 20 | 21 | func TestRunCLIShutdown(t *testing.T) { 22 | tmpDir := t.TempDir() 23 | confPath := filepath.Join(tmpDir, "foo.yaml") 24 | outPath := filepath.Join(tmpDir, "out.txt") 25 | 26 | require.NoError(t, os.WriteFile(confPath, fmt.Appendf(nil, ` 27 | input: 28 | generate: 29 | mapping: 'root.id = "foobar"' 30 | interval: "100ms" 31 | output: 32 | file: 33 | codec: lines 34 | path: %v 35 | `, outPath), 0o644)) 36 | 37 | ctx, cancel := context.WithDeadline(t.Context(), time.Now().Add(time.Second)) 38 | defer cancel() 39 | 40 | servicetest.RunCLIWithArgs(ctx, "benthos", "-c", confPath) 41 | 42 | data, _ := os.ReadFile(outPath) 43 | assert.Contains(t, string(data), "foobar") 44 | } 45 | -------------------------------------------------------------------------------- /public/service/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | package service 4 | 5 | import ( 6 | "io" 7 | "io/fs" 8 | "os" 9 | 10 | "github.com/redpanda-data/benthos/v4/internal/filepath" 11 | "github.com/redpanda-data/benthos/v4/internal/filepath/ifs" 12 | ) 13 | 14 | // ReadFile opens a file from an fs.FS and reads all bytes. When the OpenFile 15 | // method is available this will be used instead of Open with the RDONLY flag. 16 | func ReadFile(f fs.FS, name string) ([]byte, error) { 17 | var i fs.File 18 | var err error 19 | if ef, ok := f.(ifs.FS); ok { 20 | i, err = ef.OpenFile(name, os.O_RDONLY, 0) 21 | } else { 22 | i, err = f.Open(name) 23 | } 24 | if err != nil { 25 | return nil, err 26 | } 27 | return io.ReadAll(i) 28 | } 29 | 30 | // Globs attempts to expand the glob patterns within of a series of paths and 31 | // returns the resulting expanded slice or an error. 32 | func Globs(f fs.FS, paths ...string) ([]string, error) { 33 | return filepath.Globs(f, paths) 34 | } 35 | 36 | // GlobsAndSuperPaths attempts to expand a list of paths, which may include glob 37 | // patterns and super paths (the ... thing) to a list of explicit file paths. 38 | // Extensions must be provided, and limit the file types that are captured with 39 | // a super path. 40 | func GlobsAndSuperPaths(f fs.FS, paths []string, extensions ...string) ([]string, error) { 41 | return filepath.GlobsAndSuperPaths(f, paths) 42 | } 43 | 44 | // OSFS provides an fs.FS implementation that simply calls into the os. 45 | func OSFS() fs.FS { 46 | return ifs.OS() 47 | } 48 | -------------------------------------------------------------------------------- /public/wasm/README.md: -------------------------------------------------------------------------------- 1 | Benthos WASM Plugins 2 | ==================== 3 | 4 | In this directory are libraries and examples tailored to help developers create WASM plugins that can be run by the Benthos [`wasm` processor][processor.wasm]. It's possible to write WASM plugins in any language that compiles to a WASM module. However, given the complexity in passing allocated memory between the module and the host process it's much easier to use the libraries provided here as the basis for your plugin. 5 | 6 | Most of these are adapted from the fantastic range of examples provided by [the Wazero library][wazero_examples]. Our goal is to eventually provide libraries and examples for all popular languages and we'll be tackling them one at a time based on demand. Please be patient but also make [yourself heard][community]. 7 | 8 | [processor.wasm]: https://www.benthos.dev/docs/components/processors/wasm 9 | [wazero_examples]: https://github.com/tetratelabs/wazero/tree/main/examples 10 | [community]: https://www.benthos.dev/community 11 | -------------------------------------------------------------------------------- /public/wasm/examples/rust/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock -------------------------------------------------------------------------------- /public/wasm/examples/rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "louder" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | # cdylib builds a %.wasm file with `cargo build --release --target wasm32-unknown-unknown` 8 | crate-type = ["cdylib"] 9 | name = "louder" 10 | path = "louder.rs" 11 | 12 | [dependencies] 13 | # wee_aloc is a WebAssembly optimized allocator, which is needed to use non-numeric types like strings. 14 | # See https://docs.rs/wee_alloc/latest/wee_alloc/ 15 | wee_alloc = "0.4.5" 16 | 17 | # Below settings dramatically reduce wasm output size 18 | # See https://rustwasm.github.io/book/reference/code-size.html#optimizing-builds-for-code-sizewasm-opt -Oz -o 19 | # See https://doc.rust-lang.org/cargo/reference/profiles.html#codegen-units 20 | [profile.release] 21 | opt-level = "z" 22 | lto = true 23 | codegen-units = 1 24 | -------------------------------------------------------------------------------- /public/wasm/examples/tinygo/README.md: -------------------------------------------------------------------------------- 1 | TinyGo Benthos WASM Module 2 | ========================== 3 | 4 | This example builds a Benthos plugin as a WASM module written in Go and can be compiled using [TinyGo][tinygo] with the following command: 5 | 6 | ```sh 7 | tinygo build -scheduler=none -target=wasi -o uppercase.wasm . 8 | ``` 9 | 10 | You can then run the compiled module using the [`wasm` processor][processor.wasm], configured like so: 11 | 12 | ```yaml 13 | pipeline: 14 | processors: 15 | - wasm: 16 | module_path: ./uppercase.wasm 17 | ``` 18 | 19 | [TinyGo]: https://tinygo.org/ 20 | [processor.wasm]: https://www.benthos.dev/docs/components/processors/wasm 21 | -------------------------------------------------------------------------------- /public/wasm/examples/tinygo/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | //go:build tinygo 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | 10 | "github.com/redpanda-data/benthos/v4/public/wasm/tinygo" 11 | ) 12 | 13 | // main is required for TinyGo to compile to Wasm. 14 | func main() {} 15 | 16 | // _process is a WebAssembly export without arguments that triggers processing 17 | // of a Benthos message. The message data is accessed and mutated by functions 18 | // imported from Benthos and are accessible via the ./public/wasm packages (in 19 | // this case tinygo). 20 | // 21 | //export process 22 | func _process() { 23 | mBytes, err := tinygo.GetMsgAsBytes() 24 | if err != nil { 25 | panic(err) 26 | } 27 | if err := tinygo.SetMsgBytes(bytes.ToUpper(mBytes)); err != nil { 28 | panic(err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /public/wasm/tinygo/package.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Redpanda Data, Inc. 2 | 3 | // Package tinygo provides entry points that allow WASM modules compiled with 4 | // TinyGo to be executed by Benthos using the `wasm` processor. 5 | // 6 | // Check out https://github.com/benthosdev/benthos/tree/main/public/wasm/examples/tinygo 7 | // for an example of how to use this package. 8 | package tinygo 9 | -------------------------------------------------------------------------------- /resources/scripts/add_copyright.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script should be run from the root of the repository. 4 | # 5 | # Scans all files with a .go suffix and filters for files that are missing a 6 | # Copyright notice at the top. Each detected file is then modified to include 7 | # it. 8 | 9 | tmpFile="./copyright_script.tmp" 10 | 11 | for file in $(find . -name \*.go); do 12 | topLine=$(head -n 1 $file) 13 | if [[ $topLine != *"Copyright"* ]]; then 14 | echo -e "// Copyright 2025 Redpanda Data, Inc.\n" > $tmpFile 15 | cat $file >> $tmpFile 16 | cat $tmpFile > $file 17 | fi 18 | done 19 | 20 | rm -f $tmpFile 21 | -------------------------------------------------------------------------------- /resources/scripts/third_party.md.tpl: -------------------------------------------------------------------------------- 1 | # Licenses 2 | 3 | | Software | License | 4 | | :------- | :------ | 5 | {{ range . }}| {{ .Name }} | {{ .LicenseName }} | 6 | {{ end }} 7 | 8 | -------------------------------------------------------------------------------- /resources/scripts/third_party_licenses.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script should be run from the root of the repository. 4 | # 5 | # Creates a summary of all third party dependencies and their licenses. 6 | # 7 | # This script requires `go-licenses` to be installed: 8 | # 9 | # go install github.com/google/go-licenses@latest 10 | 11 | go-licenses report github.com/redpanda-data/benthos/v4/cmd/benthos \ 12 | --template ./resources/scripts/third_party.md.tpl \ 13 | > licenses/third_party.md 14 | 15 | --------------------------------------------------------------------------------