├── examples ├── log-ingestion │ ├── test.log │ ├── data-prepper-config.yaml │ ├── fluent-bit.conf │ ├── log_pipeline.yaml │ └── docker-compose-dataprepper.yaml ├── data-prepper-config.yaml ├── dev │ ├── dns │ │ ├── hosts.prepper │ │ ├── dnsmasq.conf │ │ └── Dockerfile │ ├── data-prepper-emf-monitoring │ │ ├── firelens.Dockerfile │ │ ├── data-prepper-config.yaml │ │ ├── fluent-bit.conf │ │ └── pipelines-raw-trace-stdout.yaml │ ├── README.md │ ├── trace-analytics-sample-app │ │ ├── resources │ │ │ ├── data-prepper-config.yaml │ │ │ └── data-prepper-wait-for-opensearch-and-start.sh │ │ ├── images │ │ │ └── grafana_metrics_sample.png │ │ ├── opentelemetry-collector │ │ │ └── otel-collector-config.yml │ │ └── prometheus │ │ │ └── prometheus.yaml │ └── k8s │ │ └── build_images_for_minikube.sh ├── config │ ├── pipelines-readme.txt │ ├── README.md │ └── example-data-prepper-config.yaml ├── log-ingestion-k8s │ ├── my-app-01-ns.yaml │ ├── fluent-bit-01-ns.yaml │ ├── fluent-bit-02-service-account.yaml │ ├── K8-fluentbit-data-prepper.png │ ├── data-prepper-pipeline-config │ │ └── example_log_pipeline.yaml │ ├── my-app-02-service.yaml │ ├── fluent-bit-03-role-1.22.yaml │ ├── fluent-bit-04-role-binding-1.22.yaml │ └── my-app-03-deployment.yaml ├── aws │ └── README.md ├── demo │ └── test_keystore.p12 ├── trace-analytics-sample-app │ ├── HighLevelDiagram.png │ ├── sample-app │ │ ├── analytics-service │ │ │ ├── settings.gradle │ │ │ ├── src │ │ │ │ └── main │ │ │ │ │ ├── resources │ │ │ │ │ └── application.properties │ │ │ │ │ └── java │ │ │ │ │ └── com │ │ │ │ │ └── example │ │ │ │ │ └── restservice │ │ │ │ │ ├── RestServiceApplication.java │ │ │ │ │ └── Metrics.java │ │ │ ├── gradle │ │ │ │ └── wrapper │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ └── docker-compose.yml │ │ └── requirements.txt │ └── opentelemetry-collector │ │ └── otel-collector-config.yml ├── log-ingestion-ecs-firelens │ └── example_log_pipeline.yaml ├── zipkin-sleuth-webmvc-example │ └── otel-collector-config.yml ├── metrics-ingestion-otel │ └── metric_pipeline.yaml ├── jaeger-hotrod │ └── otel-collector-config.yml ├── certificates │ └── openssl.conf └── adot │ └── aws-ot-collector-config.yml ├── data-prepper-plugins ├── dummy-plugin │ ├── build.gradle │ └── src │ │ └── main │ │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── transforms │ │ ├── rules │ │ └── dummy-plugin-rule.yaml │ │ └── templates │ │ └── dummy-plugin-template.yaml ├── log-generator-source │ ├── README.md │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── source │ │ │ └── loggenerator │ │ │ └── LogTypeGenerator.java │ └── build.gradle ├── kafka-plugins │ └── src │ │ ├── integrationTest │ │ └── resources │ │ │ └── kafka │ │ │ ├── kraft │ │ │ ├── .env │ │ │ └── sasl-scram │ │ │ │ └── .env │ │ │ └── zookeeper │ │ │ ├── .env │ │ │ └── sasl-plaintext │ │ │ ├── .env │ │ │ └── kafka_server_jaas.conf │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── kafka │ │ │ ├── configuration │ │ │ ├── KafkaIsolationLevelConfig.java │ │ │ └── AwsCredentialsConfig.java │ │ │ └── common │ │ │ └── KafkaMdc.java │ │ └── test │ │ └── resources │ │ ├── kafka-pipeline-no-auth-ssl.yaml │ │ ├── kafka-pipeline-no-auth-ssl-none.yaml │ │ ├── kafka-pipeline-bootstrap-servers-glue-default.yaml │ │ ├── sample-pipelines-1.yaml │ │ ├── kafka-pipeline-bootstrap-servers-sasl-iam-default.yaml │ │ ├── valid-data-prepper-config-with-kafka-cluster-extension.yaml │ │ ├── kafka-pipeline-bootstrap-servers-glue-sts-assume-role.yaml │ │ └── kafka-pipeline-bootstrap-servers-glue-override-endpoint.yaml ├── aws-plugin │ ├── src │ │ └── test │ │ │ └── resources │ │ │ ├── test-aws-secret-manager-configuration-default.yaml │ │ │ ├── test-aws-secret-manager-configuration-invalid-refresh-interval.yaml │ │ │ ├── test-aws-secret-manager-configuration-null-refresh-interval.yaml │ │ │ ├── test-aws-secret-manager-configuration-invalid-sts-1.yaml │ │ │ ├── test-aws-secret-manager-configuration-missing-secret-id.yaml │ │ │ ├── test-aws-secret-plugin-config.yaml │ │ │ ├── test-aws-secret-manager-configuration-invalid-sts-2.yaml │ │ │ ├── test-aws-secret-manager-configuration-invalid-sts-3.yaml │ │ │ ├── test-aws-secret-manager-configuration-with-sts.yaml │ │ │ └── test-aws-secret-manager-configuration-with-sts-headers.yaml │ └── README.md ├── common │ └── src │ │ ├── test │ │ └── resources │ │ │ ├── test-file-source-plain.tst │ │ │ ├── test-file-source-json.tst │ │ │ └── test-file-source-invalid-json.tst │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ ├── common │ │ └── sink │ │ │ ├── SinkFlushContext.java │ │ │ ├── LockStrategy.java │ │ │ ├── SinkBufferEntryProvider.java │ │ │ ├── SinkFlushableBuffer.java │ │ │ ├── SinkBufferEntry.java │ │ │ ├── SinkFlushResult.java │ │ │ └── SinkDlqHandler.java │ │ └── plugins │ │ ├── accumulator │ │ ├── BufferFactory.java │ │ └── InMemoryBufferFactory.java │ │ └── certificate │ │ └── CertificateProvider.java ├── grok-processor │ └── src │ │ └── test │ │ └── resources │ │ ├── test_patterns │ │ ├── test_patterns_1.txt │ │ └── test_patterns_2.txt │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── processor │ │ └── grok │ │ ├── single_match_single_pattern.yaml │ │ ├── match_with_keep_empty_captures_true.yaml │ │ ├── match_with_named_captures_syntax.yaml │ │ ├── match_no_captures_with_existing_and_non_existing_key.yaml │ │ ├── match_with_named_captures_only_false.yaml │ │ ├── multiple_match_with_break_on_match_false.yaml │ │ ├── single_match_multiple_pattern_with_break_on_match_false.yaml │ │ ├── single_match_type_conversion.yaml │ │ ├── pattern_definitions.yaml │ │ ├── match_with_no_captures_and_tags.yaml │ │ ├── patterns_dir_with_custom_patterns_files_glob.yaml │ │ └── patterns_dir_with_default_patterns_files_glob.yaml ├── ocsf │ ├── build.gradle │ └── src │ │ └── main │ │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── transforms │ │ └── rules │ │ └── ocsf-v1.1-panw-traffic-rule.yaml ├── rds-source │ └── src │ │ └── main │ │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── transforms │ │ └── rules │ │ └── rds-rule.yaml ├── http-source-common │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── http │ │ └── BaseHttpService.java ├── opensearch │ └── src │ │ ├── test │ │ └── resources │ │ │ ├── test_keystore.jks │ │ │ ├── service-map-1.json │ │ │ └── management-disabled-index-template.json │ │ └── main │ │ ├── java │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ ├── sink │ │ │ └── opensearch │ │ │ │ ├── s3 │ │ │ │ ├── FileType.java │ │ │ │ ├── FileReader.java │ │ │ │ ├── InvalidS3URIException.java │ │ │ │ ├── S3ObjectTooLargeException.java │ │ │ │ └── UnsupportedFileTypeException.java │ │ │ │ ├── bulk │ │ │ │ └── BulkApiWrapper.java │ │ │ │ └── index │ │ │ │ └── IndexManager.java │ │ │ └── source │ │ │ └── opensearch │ │ │ └── worker │ │ │ ├── SearchWorker.java │ │ │ └── client │ │ │ └── exceptions │ │ │ └── IndexNotFoundException.java │ │ └── resources │ │ ├── logs-policy-no-ism-template.json │ │ ├── raw-span-policy-no-ism-template.json │ │ └── metrics-policy-no-ism-template.json ├── mongodb │ └── src │ │ └── main │ │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── transforms │ │ └── rules │ │ ├── mongodb-rule.yaml │ │ └── documentdb-rule.yaml ├── s3-source │ └── src │ │ └── main │ │ ├── resources │ │ └── IntegrationTest.parquet │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── source │ │ └── s3 │ │ ├── exception │ │ ├── S3RetriesExhaustedException.java │ │ └── SqsRetriesExhaustedException.java │ │ └── S3SelectResponseHandlerFactory.java ├── parquet-codecs │ └── src │ │ └── test │ │ └── resources │ │ ├── test-parquet.parquet │ │ └── sample.snappy.parquet ├── buffer-common │ └── build.gradle ├── otel-proto-common │ └── src │ │ └── test │ │ └── resources │ │ ├── test-otel-log.protobuf │ │ └── test-otel-multi-log.protobuf ├── saas-source-plugins │ ├── jira-source │ │ ├── src │ │ │ └── test │ │ │ │ └── resources │ │ │ │ ├── basic-auth-jira-pipeline.yaml │ │ │ │ └── oauth2-auth-jira-pipeline.yaml │ │ └── README.md │ ├── atlassian-commons │ │ └── src │ │ │ └── test │ │ │ └── resources │ │ │ ├── basic-auth-jira-pipeline.yaml │ │ │ └── oauth2-auth-jira-pipeline.yaml │ ├── confluence-source │ │ ├── src │ │ │ └── test │ │ │ │ └── resources │ │ │ │ ├── basic-auth-confluence-pipeline.yaml │ │ │ │ └── oauth2-auth-confluence-pipeline.yaml │ │ └── README.md │ ├── build.gradle │ └── source-crawler │ │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── source │ │ └── source_crawler │ │ └── CrawlerApplicationContextMarker.java ├── parse-json-processor │ └── src │ │ └── test │ │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── processor │ │ └── parse │ │ └── json │ │ ├── default.yaml │ │ └── with-destination.yaml ├── s3-sink │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── sink │ │ │ └── s3 │ │ │ ├── ExtensionProvider.java │ │ │ └── codec │ │ │ └── BufferedCodec.java │ │ ├── test │ │ └── resources │ │ │ └── simplelogger.properties │ │ └── integrationTest │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── sink │ │ └── s3 │ │ └── BufferScenario.java ├── dynamodb-source │ ├── README.md │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── source │ │ └── dynamodb │ │ └── stream │ │ └── ShardNotTrackedException.java ├── kinesis-source │ └── src │ │ └── test │ │ └── resources │ │ ├── simple_pipeline_with_extensions.yaml │ │ ├── pipeline_with_acks_enabled.yaml │ │ ├── pipeline_with_polling_config_enabled.yaml │ │ └── pipeline_with_stream_arn_config.yaml ├── personalize-sink │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── sink │ │ └── personalize │ │ └── configuration │ │ └── PersonalizeAdvancedValidation.java ├── drop-events-processor │ └── src │ │ └── test │ │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── processor │ │ └── drop │ │ └── drop_when_value_is_empty_string.yaml ├── aws-lambda │ └── src │ │ ├── test │ │ └── resources │ │ │ ├── mockito-extensions │ │ │ └── org.mockito.plugins.MockMaker │ │ │ ├── simplelogger.properties │ │ │ ├── lambda-processor-null-key-name.yaml │ │ │ ├── lambda-processor-success-config.yaml │ │ │ ├── lambda-processor-aggregate-mode-config.yaml │ │ │ ├── lambda-processor-large-payload.yaml │ │ │ ├── lambda-processor-success-config-with-keys.yaml │ │ │ ├── lambda-processor-when-condition-config.yaml │ │ │ ├── lambda-processor-payload-limit.yaml │ │ │ ├── lambda-processor-with-retries.yaml │ │ │ ├── lambda-processor-cache-string-key.yaml │ │ │ ├── lambda-processor-circuit-breaker-config.yaml │ │ │ └── lambda-processor-success-config-with-keys-merge-mode.yaml │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── lambda │ │ └── processor │ │ └── exception │ │ └── StrictResponseModeNotRespectedException.java ├── avro-codecs │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── avro │ │ └── SchemaGenerationException.java ├── user-agent-processor │ └── README.md ├── dissect-processor │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── processor │ │ │ └── dissect │ │ │ └── Fields │ │ │ ├── SkipField.java │ │ │ ├── NormalField.java │ │ │ └── IndirectField.java │ └── README.md ├── event-json-codecs │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── codec │ │ └── event_json │ │ └── EventJsonOutputCodecConfig.java ├── armeria-common │ ├── build.gradle │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── exceptions │ │ ├── RequestCancelledException.java │ │ ├── BadRequestException.java │ │ └── BufferWriteException.java ├── cloudwatch-logs │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── sink │ │ └── cloudwatch_logs │ │ ├── exception │ │ └── InvalidBufferTypeException.java │ │ └── buffer │ │ ├── InMemoryBufferFactory.java │ │ └── BufferFactory.java ├── translate-processor │ └── src │ │ ├── test │ │ └── resources │ │ │ └── configs │ │ │ ├── translate_dynamic_key.json │ │ │ ├── translate_nested_path.json │ │ │ ├── translate_static_key.json │ │ │ └── translate_range_mapping.json │ │ └── jmh │ │ └── resources │ │ └── jmh │ │ ├── static_key_test_data.json │ │ ├── static_key_config.json │ │ └── nested_path_config.json ├── aws-plugin-api │ ├── README.md │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── aws │ │ └── api │ │ └── AwsCredentialsConfig.java ├── decompress-processor │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── processor │ │ │ └── decompress │ │ │ ├── encoding │ │ │ ├── DecoderEngineFactory.java │ │ │ └── DecoderEngine.java │ │ │ ├── exceptions │ │ │ └── DecodingException.java │ │ │ └── DecompressionEngineFactory.java │ └── build.gradle ├── trace-peer-forwarder-processor │ └── build.gradle ├── mutate-string-processors │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── processor │ │ └── mutatestring │ │ └── StringProcessorConfig.java ├── geoip-processor │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── geoip │ │ │ ├── extension │ │ │ └── databasedownload │ │ │ │ └── DBSource.java │ │ │ └── exception │ │ │ ├── InvalidIPAddressException.java │ │ │ ├── NoValidDatabaseFoundException.java │ │ │ └── DatabaseReaderInitializationException.java │ │ └── test │ │ └── resources │ │ └── geoip_service_config.yaml ├── newline-codecs │ └── build.gradle ├── in-memory-source-coordination-store │ └── build.gradle ├── cloudwatch-metrics-source │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── source │ │ └── CloudwatchSourceProgressState.java ├── aws-sqs-common │ └── build.gradle ├── obfuscate-processor │ └── build.gradle ├── encryption-plugin │ └── src │ │ └── test │ │ └── resources │ │ ├── test-kms-encryption-engine-config-malformat-sts.yaml │ │ └── test-valid-kms-encryption-engine-config.yaml ├── csv-processor │ └── README.md ├── opensearch-api-source │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── plugins │ │ └── source │ │ └── opensearchapi │ │ └── model │ │ └── BulkAPIRequestParams.java ├── truncate-processor │ └── build.gradle └── aggregate-processor │ └── src │ └── main │ └── java │ └── org │ └── opensearch │ └── dataprepper │ └── plugins │ └── processor │ └── aggregate │ └── DefaultGroupState.java ├── e2e-test ├── log │ └── src │ │ └── integrationTest │ │ └── resources │ │ ├── data_prepper.yml │ │ ├── basic-grok-e2e-pipeline.yml │ │ └── basic-grok-e2e-pipeline-date-pattern-index.yml ├── trace │ └── src │ │ └── integrationTest │ │ └── resources │ │ ├── data_prepper.yml │ │ └── data_prepper_static.yml └── peerforwarder │ └── src │ └── integrationTest │ └── resources │ ├── data_prepper_local_node.yml │ └── data_prepper_static.yml ├── data-prepper-core └── src │ ├── test │ └── resources │ │ ├── invalid_data_prepper_config.yml │ │ ├── invalid_peer_forwarder_with_port_config.yml │ │ ├── invalid_port_data_prepper_config.yml │ │ ├── valid_data_prepper_some_default_config.yml │ │ ├── valid_peer_forwarder_without_ssl_config.yml │ │ ├── invalid_peer_forwarder_with_batch_size_config.yml │ │ ├── invalid_peer_forwarder_with_buffer_size_config.yml │ │ ├── valid_data_prepper_config.yml │ │ ├── invalid_peer_forwarder_with_bad_drain_timeout.yml │ │ ├── invalid_peer_forwarder_with_connection_config.yml │ │ ├── invalid_peer_forwarder_with_discovery_mode_config.yml │ │ ├── org │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── core │ │ │ └── parser │ │ │ └── model │ │ │ ├── heap_without_reset.yaml │ │ │ └── heap_with_reset.yaml │ │ ├── invalid_peer_forwarder_with_negative_drain_timeout.yml │ │ ├── invalid_peer_forwarder_with_ssl_config.yml │ │ ├── invalid_peer_forwarder_with_dns_without_domain_name_config.yml │ │ ├── valid_pipeline.yml │ │ ├── invalid_peer_forwarder_with_thread_count_config.yml │ │ ├── valid_data_prepper_cloudwatch_metrics_config.yml │ │ ├── invalid_peer_forwarder_with_zero_local_write_timeout.yml │ │ ├── valid_data_prepper_multiple_metrics_config.yml │ │ ├── test.p12 │ │ ├── valid_data_prepper_config_with_sink_shutdown_timeout.yml │ │ ├── valid_data_prepper_config_with_tags.yml │ │ ├── invalid_data_prepper_config_with_bad_sink_shutdown_timeout.yml │ │ ├── keystore.jks │ │ ├── valid_data_prepper_config_with_processor_shutdown_timeout.yml │ │ ├── valid_data_prepper_config_with_test_extension.yml │ │ ├── invalid_data_prepper_config_with_bad_processor_shutdown_timeout.yml │ │ ├── invalid_data_prepper_config_with_negative_sink_shutdown_timeout.yml │ │ ├── valid_data_prepper_config_with_pipeline_shutdown.yml │ │ ├── tls │ │ ├── test_keystore.jks │ │ ├── test_keystore.p12 │ │ ├── test_keystore_with_different_passwords.p12 │ │ ├── test_keystore_with_identical_passwords.p12 │ │ └── openssl.conf │ │ ├── valid_data_prepper_config_wth_peer_forwarder_config.yml │ │ ├── invalid_data_prepper_config_with_negative_processor_shutdown_timeout.yml │ │ ├── valid_data_prepper_config_with_heap_circuit_breaker.yml │ │ ├── valid_data_prepper_config_with_metric_filter.yml │ │ ├── single_pipeline_valid_off_heap_buffer.yml │ │ ├── valid_peer_forwarder_config_with_acm_ssl.yml │ │ ├── valid_zero_buffer_single_thread.yml │ │ ├── invalid_zero_buffer_multiple_threads.yml │ │ ├── valid_data_prepper_config_with_iso8601_shutdown_timeouts.yml │ │ ├── invalid_data_prepper_config_with_tags.yml │ │ ├── invalid_peer_forwarder_with_cloud_map_without_namespace_name_config.yml │ │ ├── invalid_peer_forwarder_with_cloud_map_without_service_name_config.yml │ │ ├── invalid_peer_forwarder_with_cloud_map_without_region_config.yml │ │ ├── valid_peer_forwarder_config_with_insecure.yml │ │ ├── single_pipeline_valid_empty_source_plugin_settings.yml │ │ ├── valid_peer_forwarder_config_with_fingerprint.yml │ │ ├── invalid_data_prepper_config_with_metric_filter.yml │ │ ├── compatible_version.yml │ │ ├── invalid_zero_buffer_with_single_thread_processor.yml │ │ ├── invalid_peer_forwarder_with_acm_without_arn_config.yml │ │ ├── cloudwatch_test.properties │ │ ├── invalid_peer_forwarder_with_acm_without_region_config.yml │ │ ├── invalid_zero_buffer_multiple_threads_no_single_thread_processors.yml │ │ ├── multi-pipelines-single-pipeline-extensions │ │ ├── pipeline_configuration2.yml │ │ └── pipeline_configuration_with_test_extension.yml │ │ ├── valid_data_prepper_source_coordination_config.yml │ │ ├── valid_data_prepper_config_with_basic_authentication.yml │ │ ├── valid_peer_forwarder_config_with_drain_timeout.yml │ │ ├── missing_source_multiple_pipeline_configuration.yml │ │ ├── valid_peer_forwarder_config_with_iso8601_drain_timeout.yml │ │ ├── valid_data_prepper_config_with_camel_case_options.yml │ │ ├── valid_peer_forwarder_config_with_unauthenticated.yml │ │ ├── multi-pipelines-distributed-pipeline-extensions │ │ ├── pipeline_configuration_with_test_extension2.yml │ │ └── pipeline_configuration_with_test_extension1.yml │ │ ├── log4j2.properties │ │ ├── missing_name_multiple_pipeline_configuration.yml │ │ ├── valid_peer_forwarder_config_with_mutual_tls.yml │ │ ├── invalid_peer_forwarder_config_with_mutual_tls_not_ssl.yml │ │ ├── missing_pipeline_multiple_pipeline_configuration.yml │ │ └── invalid_peer_forwarder_config_with_many_authentication.yml │ ├── integrationTest │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ ├── configuration │ │ └── data-prepper-config.yaml │ │ └── pipeline │ │ ├── minimal-pipeline.yaml │ │ ├── acknowledgements │ │ ├── simple-test.yaml │ │ └── one-pipeline-ack-expiry-test.yaml │ │ ├── multi-worker.yaml │ │ ├── basic-processor-pipeline.yaml │ │ ├── connected │ │ └── single-connection.yaml │ │ ├── processor-pipeline.yaml │ │ ├── single-thread-processor-pipeline.yaml │ │ ├── multi-processor-pipeline.yaml │ │ ├── pipeline-dlq.yaml │ │ └── processor-swap │ │ └── source │ │ └── processor-swap-pipeline.yaml │ └── main │ ├── resources │ ├── META-INF │ │ └── services │ │ │ ├── org.opensearch.dataprepper.plugin.PluginProvider │ │ │ └── org.opensearch.dataprepper.model.configuration.VersionProvider │ └── cloudwatch.properties │ └── java │ └── org │ └── opensearch │ └── dataprepper │ ├── core │ ├── pipeline │ │ ├── SupportsPipelineRunner.java │ │ ├── PipelineObserver.java │ │ └── ProcessorProvider.java │ ├── parser │ │ └── model │ │ │ └── MetricRegistryType.java │ └── breaker │ │ └── InnerCircuitBreaker.java │ └── DataPrepperShutdownListener.java ├── data-prepper-main └── src │ ├── test │ └── resources │ │ ├── valid_data_prepper_config.yml │ │ ├── single_pipeline_valid_empty_source_plugin_settings.yml │ │ ├── logstash-filter.conf │ │ └── logstash-conf │ │ └── logstash-filter.conf │ └── main │ └── resources │ └── META-INF │ ├── data-prepper.plugins.properties │ └── services │ └── com.linecorp.armeria.client.ClientFactoryProvider ├── data-prepper-api └── src │ ├── test │ ├── resources │ │ ├── single_plugin.yaml │ │ ├── org │ │ │ └── opensearch │ │ │ │ └── dataprepper │ │ │ │ └── model │ │ │ │ └── configuration │ │ │ │ ├── plugin_model_empty.yaml │ │ │ │ ├── plugin_model_not_present.yaml │ │ │ │ ├── plugin_model_null.yaml │ │ │ │ ├── conditional_route_invalid_just_value.yaml │ │ │ │ ├── conditional_route_single.yaml │ │ │ │ ├── conditional_route_invalid_non_string.yaml │ │ │ │ ├── conditional_route_invalid_object.yaml │ │ │ │ ├── conditional_route_list.yaml │ │ │ │ └── sink_plugin.yaml │ │ ├── list_of_plugins.yaml │ │ ├── serialized_with_plugin_settings.yaml │ │ ├── pipelines_data_flow_serialized.yaml │ │ ├── pipeline_with_version.yaml │ │ ├── pipeline_with_short_hand_version.yaml │ │ ├── pipeline_with_extension.yaml │ │ ├── pipeline_with_depreciated_extension.yaml │ │ ├── pipelines_data_flow_route.yaml │ │ ├── pipelines_data_flow_routes.yaml │ │ └── testjson │ │ │ └── log.json │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── model │ │ ├── log │ │ └── .JacksonStandardOTelLogTest.java.swp │ │ └── metric │ │ └── .JacksonStandardSummaryTest.java.swp │ └── main │ └── java │ └── org │ └── opensearch │ └── dataprepper │ ├── model │ ├── io │ │ ├── InputFile.java │ │ └── OutputFile.java │ ├── codec │ │ ├── DecompressionEngine.java │ │ └── HasByteDecoder.java │ ├── encryption │ │ ├── KeyProvider.java │ │ └── EncryptionEnvelope.java │ ├── event │ │ ├── EventFailureMetadata.java │ │ ├── exceptions │ │ │ └── EventKeyNotFoundException.java │ │ └── EventBuilder.java │ ├── types │ │ ├── ByteCountParseException.java │ │ └── ByteCountInvalidInputException.java │ ├── source │ │ └── s3 │ │ │ └── S3ScanEnvironmentVariables.java │ ├── document │ │ └── Document.java │ ├── acknowledgements │ │ └── ProgressCheck.java │ └── metric │ │ └── Gauge.java │ └── typeconverter │ ├── ConverterArguments.java │ └── TypeConverter.java ├── data-prepper-test ├── plugin-test-framework │ └── src │ │ ├── test │ │ ├── resources │ │ │ └── org │ │ │ │ └── opensearch │ │ │ │ └── dataprepper │ │ │ │ └── test │ │ │ │ └── plugins │ │ │ │ ├── junit │ │ │ │ ├── empty.yaml │ │ │ │ ├── no-processor.yaml │ │ │ │ ├── multiple-processors.yaml │ │ │ │ ├── valid.yaml │ │ │ │ └── multiple-pipelines.yaml │ │ │ │ └── test001.yaml │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── test │ │ │ └── TestPluggableInterface.java │ │ └── main │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.opensearch.dataprepper.plugin.PluginProvider ├── test-event │ ├── README.md │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── event │ │ │ └── EventFactoryApplicationContextMarker.java │ └── build.gradle ├── test-common │ ├── build.gradle │ └── src │ │ └── test │ │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── test │ │ └── helper │ │ └── ReflectivelySetFieldTestHelper.java ├── test-data-prepper-version │ ├── build.gradle │ └── src │ │ └── main │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.opensearch.dataprepper.model.configuration.VersionProvider └── build.gradle ├── testing └── aws-testing-cdk │ ├── .npmignore │ ├── .gitignore │ └── jest.config.js ├── docs ├── dev │ └── images │ │ ├── TraceView.png │ │ ├── DashboardView.png │ │ ├── ServiceView.png │ │ ├── HighLevelDesign.jpg │ │ ├── DataPrepperPipeline.png │ │ └── TraceAnalyticsFeature.jpg ├── images │ ├── LogstashGatling.png │ ├── DataPrepperGatling.png │ ├── DataPrepperPipeline.png │ ├── LogAnalyticsComponents.png │ ├── TraceAnalyticsFeature.jpg │ ├── TraceAnalyticsComponents.png │ ├── PerformanceTestEnvironment.png │ ├── PerformanceTestEnvironmentLogstash.png │ └── Log_Ingestion_FluentBit_DataPrepper_OpenSearch.jpg ├── log_analytics.md ├── migrating_from_opendistro.md ├── trace_analytics.md ├── core_apis.md └── peer_forwarder.md ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── release ├── staging-resources-cdk │ ├── .npmignore │ ├── .gitignore │ └── jest.config.js ├── docker │ ├── config │ │ ├── default-keystore.p12 │ │ ├── default-data-prepper-config.yaml │ │ └── public.pem │ └── adoptium.repo ├── archives │ └── linux │ │ └── build.gradle ├── release-notes │ ├── data-prepper.release-notes-2.10.1.md │ ├── data-prepper.release-notes-2.10.3.md │ ├── odfe-data-prepper.release-notes-1.0.0.md │ ├── data-prepper.release-notes-2.10.2.md │ └── data-prepper.release-notes-1.1.1.md └── script │ └── release-notes │ ├── release-notes-template.md │ └── format-release-notes.py ├── data-prepper-logstash-configuration └── src │ ├── main │ └── resources │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ └── logstash │ │ └── mapping │ │ ├── drop.mapping.yaml │ │ ├── mutate.mapping.yaml │ │ ├── csv.mapping.yaml │ │ ├── http.mapping.yaml │ │ ├── elasticsearch.mapping.yaml │ │ ├── amazon_es.mapping.yaml │ │ ├── opensearch.mapping.yaml │ │ ├── date.mapping.yaml │ │ └── kv.mapping.yaml │ └── test │ └── resources │ └── org │ └── opensearch │ └── dataprepper │ └── logstash │ ├── mapping │ ├── sample-with-empty-maps.mapping.yaml │ ├── no_plugin_name.mapping.yaml │ ├── invalid.mapping.yaml │ ├── amazon_es.mapping.yaml │ ├── sample-with-nulls.mapping.yaml │ └── sample.mapping.yaml │ ├── http-to-opensearch.conf │ ├── log-ingest-to-amazon-opensearch.conf │ ├── http-to-opensearch.expected.yaml │ ├── log-ingest-to-logstash-opensearch-insecure.conf │ └── log-ingest-to-opensearch-named-captures.conf ├── data-prepper-pipeline-parser └── src │ ├── test │ └── resources │ │ ├── transformation │ │ ├── rules │ │ │ ├── documentdb1-rule.yaml │ │ │ └── documentdb-rule.yaml │ │ ├── templates │ │ │ └── testSource │ │ │ │ ├── documentdb-simple-template.yaml │ │ │ │ ├── documentdb-function-template.yaml │ │ │ │ └── documentdb-subpipelines-template.yaml │ │ ├── userConfig │ │ │ ├── documentdb-simple-userconfig.yaml │ │ │ └── documentdb1-userconfig.yaml │ │ └── expected │ │ │ └── documentdb1-expected.yaml │ │ ├── org │ │ └── opensearch │ │ │ └── dataprepper │ │ │ └── transforms │ │ │ ├── rules │ │ │ └── test-plugin-rule.yaml │ │ │ └── templates │ │ │ └── test-plugin-template.yaml │ │ ├── incompatible_version.yml │ │ ├── multi-pipelines │ │ ├── valid_multiple_pipeline_configuration2.yml │ │ ├── valid_multiple_pipeline_configuration3.yml │ │ └── valid_multiple_pipeline_configuration1.yml │ │ ├── multi-pipelines-single-pipeline-configurations │ │ ├── pipeline_configuration2.yml │ │ └── pipeline_configuration_with_test_extension.yml │ │ └── multi-pipelines-distributed-pipeline-configurations │ │ ├── pipeline_configuration_with_test_extension2.yml │ │ └── pipeline_configuration_with_test_extension1.yml │ └── main │ ├── resources │ └── META-INF │ │ └── services │ │ └── jakarta.validation.ConstraintValidator │ └── java │ └── org │ └── opensearch │ └── dataprepper │ ├── validation │ └── PluginErrorsHandler.java │ └── pipeline │ └── parser │ └── transformer │ └── PipelineTransformationPathProvider.java ├── .dockerignore ├── gradle.properties ├── .gitattributes ├── performance-test └── src │ └── gatling │ ├── resources │ └── bodies │ │ └── singleLog.json │ └── java │ └── org │ └── opensearch │ └── dataprepper │ └── test │ └── performance │ └── tools │ └── PathTarget.java ├── .github ├── CODEOWNERS ├── codecov.yml └── workflows │ ├── delete_backport_branch.yml │ └── dco.yml ├── data-prepper-plugin-framework └── src │ ├── test │ └── java │ │ └── org │ │ └── opensearch │ │ └── dataprepper │ │ ├── plugin │ │ └── TestPluggableInterface.java │ │ └── plugins │ │ └── test │ │ └── TestComponent.java │ └── main │ └── java │ └── org │ └── opensearch │ └── dataprepper │ └── plugin │ ├── ExtensionsConfiguration.java │ ├── PluginArgumentsContext.java │ └── ExtensionClassProvider.java ├── .ci └── documentation │ └── issue.md ├── data-prepper-event └── src │ └── main │ └── java │ └── org │ └── opensearch │ └── dataprepper │ └── event │ ├── EventConfigurationContainer.java │ └── EventBuilderFactory.java ├── NOTICE ├── .gitignore ├── config └── checkstyle │ └── import-control.xml └── data-prepper-expression └── src └── main └── java └── org └── opensearch └── dataprepper └── expression └── ExceptionOverview.java /examples/log-ingestion/test.log: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /data-prepper-plugins/dummy-plugin/build.gradle: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /data-prepper-plugins/log-generator-source/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /examples/dev/dns/hosts.prepper: -------------------------------------------------------------------------------- 1 | 10.10.1.2 prepper-cluster 2 | -------------------------------------------------------------------------------- /e2e-test/log/src/integrationTest/resources/data_prepper.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /e2e-test/trace/src/integrationTest/resources/data_prepper.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config.yml: -------------------------------------------------------------------------------- 1 | bad_key: badValue 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_port_config.yml: -------------------------------------------------------------------------------- 1 | port: -10 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_port_data_prepper_config.yml: -------------------------------------------------------------------------------- 1 | server_port: -550 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_some_default_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_without_ssl_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_batch_size_config.yml: -------------------------------------------------------------------------------- 1 | batch_size: 0 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_buffer_size_config.yml: -------------------------------------------------------------------------------- 1 | buffer_size: -10 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | -------------------------------------------------------------------------------- /data-prepper-main/src/test/resources/valid_data_prepper_config.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | -------------------------------------------------------------------------------- /examples/config/pipelines-readme.txt: -------------------------------------------------------------------------------- 1 | This directory is a place for pipeline configuration files. 2 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/single_plugin.yaml: -------------------------------------------------------------------------------- 1 | singlePlugin: 2 | pluginName: 3 | key1: value1 -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/kraft/.env: -------------------------------------------------------------------------------- 1 | KAFKA_VERSION=3.5.1 2 | -------------------------------------------------------------------------------- /examples/dev/dns/dnsmasq.conf: -------------------------------------------------------------------------------- 1 | server=127.0.0.11 # Embedded Docker DNS 2 | addn-hosts=/etc/hosts.prepper 3 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_bad_drain_timeout.yml: -------------------------------------------------------------------------------- 1 | drain_timeout: "abc" 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_connection_config.yml: -------------------------------------------------------------------------------- 1 | max_connection_count: 0 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_discovery_mode_config.yml: -------------------------------------------------------------------------------- 1 | discovery_mode: none -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/kraft/sasl-scram/.env: -------------------------------------------------------------------------------- 1 | KAFKA_VERSION=3.5.1 -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/zookeeper/.env: -------------------------------------------------------------------------------- 1 | KAFKA_VERSION=2.8.1 2 | -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/my-app-01-ns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: my-app -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/org/opensearch/dataprepper/core/parser/model/heap_without_reset.yaml: -------------------------------------------------------------------------------- 1 | usage: 24b -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/test/resources/org/opensearch/dataprepper/test/plugins/junit/empty.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/config/README.md: -------------------------------------------------------------------------------- 1 | Directory containing example configuration files to be bundled with release artifacts. 2 | -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/fluent-bit-01-ns.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: logging -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/configuration/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_negative_drain_timeout.yml: -------------------------------------------------------------------------------- 1 | drain_timeout: "PT-6H3M" 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_ssl_config.yml: -------------------------------------------------------------------------------- 1 | ssl_certificate_file: "invalid_path" 2 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-default.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/zookeeper/sasl-plaintext/.env: -------------------------------------------------------------------------------- 1 | KAFKA_VERSION=3.5.1 -------------------------------------------------------------------------------- /examples/aws/README.md: -------------------------------------------------------------------------------- 1 | ## AWS Examples 2 | This directory contains AWS-specific sample content (e.g. running on EC2). 3 | -------------------------------------------------------------------------------- /examples/config/example-data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | 3 | extensions: 4 | geoip_service: 5 | maxmind: 6 | -------------------------------------------------------------------------------- /testing/aws-testing-cdk/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/org/opensearch/dataprepper/core/parser/model/heap_with_reset.yaml: -------------------------------------------------------------------------------- 1 | usage: 24b 2 | reset: 3s -------------------------------------------------------------------------------- /docs/dev/images/TraceView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/dev/images/TraceView.png -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_dns_without_domain_name_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | discovery_mode: dns -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_pipeline.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | source: 3 | random: 4 | sink: 5 | - stdout: -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/resources/test-file-source-plain.tst: -------------------------------------------------------------------------------- 1 | THIS IS A PLAINTEXT LINE 2 | THIS IS ANOTHER PLAINTEXT LINE -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/test_patterns/test_patterns_1.txt: -------------------------------------------------------------------------------- 1 | CUSTOMPHONENUMBERPATTERN \d\d\d-\d\d\d-\d\d\d -------------------------------------------------------------------------------- /data-prepper-plugins/ocsf/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | dependencies { 3 | implementation project(':data-prepper-api') 4 | } 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/dev/images/DashboardView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/dev/images/DashboardView.png -------------------------------------------------------------------------------- /docs/dev/images/ServiceView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/dev/images/ServiceView.png -------------------------------------------------------------------------------- /docs/images/LogstashGatling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/LogstashGatling.png -------------------------------------------------------------------------------- /examples/demo/test_keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/examples/demo/test_keystore.p12 -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /release/staging-resources-cdk/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | customPlugin: {} 3 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_not_present.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | customPlugin: 3 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_null.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | customPlugin: null 3 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_thread_count_config.yml: -------------------------------------------------------------------------------- 1 | client_thread_count: 0 2 | server_thread_count: 0 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metric_registries: [CloudWatch] 3 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/test_patterns/test_patterns_2.txt: -------------------------------------------------------------------------------- 1 | CUSTOMBIRTHDAYPATTERN [A-Z][a-z]+ [\d]+, \d\d\d\d -------------------------------------------------------------------------------- /docs/dev/images/HighLevelDesign.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/dev/images/HighLevelDesign.jpg -------------------------------------------------------------------------------- /docs/images/DataPrepperGatling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/DataPrepperGatling.png -------------------------------------------------------------------------------- /docs/images/DataPrepperPipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/DataPrepperPipeline.png -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_zero_local_write_timeout.yml: -------------------------------------------------------------------------------- 1 | failed_forwarding_requests_local_write_timeout: 0 -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/drop.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: drop_events -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/resources/test-file-source-json.tst: -------------------------------------------------------------------------------- 1 | {"test_key": "test_value"} 2 | {"second_test_key": "second_test_value"} -------------------------------------------------------------------------------- /docs/images/LogAnalyticsComponents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/LogAnalyticsComponents.png -------------------------------------------------------------------------------- /docs/images/TraceAnalyticsFeature.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/TraceAnalyticsFeature.jpg -------------------------------------------------------------------------------- /examples/dev/dns/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | RUN apk --no-cache add dnsmasq-dnssec 3 | EXPOSE 53 53/udp 4 | ENTRYPOINT ["dnsmasq", "-k"] 5 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metric_registries: [Prometheus, CloudWatch] 3 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/resources/test-file-source-invalid-json.tst: -------------------------------------------------------------------------------- 1 | {"test_key: test_value"} 2 | {"second_test_key": "second_test_value" -------------------------------------------------------------------------------- /docs/dev/images/DataPrepperPipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/dev/images/DataPrepperPipeline.png -------------------------------------------------------------------------------- /docs/images/TraceAnalyticsComponents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/TraceAnalyticsComponents.png -------------------------------------------------------------------------------- /examples/dev/data-prepper-emf-monitoring/firelens.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazon/aws-for-fluent-bit:latest 2 | ADD firelens-fluent-bit.conf /fluent-bit.conf -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/conditional_route_invalid_just_value.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | a: b 3 | c: d 4 | -------------------------------------------------------------------------------- /docs/dev/images/TraceAnalyticsFeature.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/dev/images/TraceAnalyticsFeature.jpg -------------------------------------------------------------------------------- /docs/images/PerformanceTestEnvironment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/PerformanceTestEnvironment.png -------------------------------------------------------------------------------- /release/docker/config/default-keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/release/docker/config/default-keystore.p12 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/test.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-core/src/test/resources/test.p12 -------------------------------------------------------------------------------- /examples/dev/data-prepper-emf-monitoring/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metricRegistries: 3 | - "EmbeddedMetricsFormat" 4 | - "Prometheus" 5 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/list_of_plugins.yaml: -------------------------------------------------------------------------------- 1 | listOfPlugins: 2 | - customPluginA: 3 | key1: value1 4 | - customPluginB: 5 | key2: value2 -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/conditional_route_single.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | testRouteName: "/my/property==value" 3 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/serialized_with_plugin_settings.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | customPlugin: 3 | key1: "value1" 4 | key2: "value2" 5 | key3: "value3" 6 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_sink_shutdown_timeout.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | sink_shutdown_timeout: 1s 4 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_tags.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metric_tags: 3 | testKey1: testValue1 4 | testKey2: testValue2 5 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb1-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "documentdb" 2 | apply_when: 3 | - "$..source.documentdb" -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | #ignore gradle specific files and examples from copying to image 2 | build 3 | docs 4 | examples 5 | #research remove this after updating settings.gradle -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config_with_bad_sink_shutdown_timeout.yml: -------------------------------------------------------------------------------- 1 | server_port: 1234 2 | ssl: true 3 | sink_shutdown_timeout: "bad" 4 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-core/src/test/resources/keystore.jks -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-refresh-interval.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret 2 | refresh_interval: PT1m -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-null-refresh-interval.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret 2 | refresh_interval: null -------------------------------------------------------------------------------- /data-prepper-plugins/rds-source/src/main/resources/org/opensearch/dataprepper/transforms/rules/rds-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "rds" 2 | apply_when: 3 | - "$..source.rds" -------------------------------------------------------------------------------- /docs/images/PerformanceTestEnvironmentLogstash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/PerformanceTestEnvironmentLogstash.png -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/fluent-bit-02-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: fluent-bit 5 | namespace: logging 6 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_processor_shutdown_timeout.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | processor_shutdown_timeout: 1s 4 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_test_extension.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | extensions: 3 | test_extension: 4 | test_attribute: test_string -------------------------------------------------------------------------------- /examples/log-ingestion/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | serverPort: 4900 3 | authentication: 4 | http_basic: 5 | username: admin 6 | password: admin 7 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config_with_bad_processor_shutdown_timeout.yml: -------------------------------------------------------------------------------- 1 | server_port: 1234 2 | ssl: true 3 | processor_shutdown_timeout: "bad" 4 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config_with_negative_sink_shutdown_timeout.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | sink_shutdown_timeout: "PT-6H3M" 4 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_pipeline_shutdown.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | pipeline_shutdown: on-all-pipeline-failures 4 | -------------------------------------------------------------------------------- /e2e-test/peerforwarder/src/integrationTest/resources/data_prepper_local_node.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | peer_forwarder: 3 | port: 4994 4 | ssl: false 5 | discovery_mode: local_node -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | version=2.14.0-SNAPSHOT 7 | org.gradle.jvmargs=-Xmx2048M 8 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/test_keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-core/src/test/resources/tls/test_keystore.jks -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/test_keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-core/src/test/resources/tls/test_keystore.p12 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_wth_peer_forwarder_config.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | peer_forwarder: 4 | port: 4994 5 | ssl: false 6 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-1.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret 2 | region: us-east-1 3 | sts_role_arn: invalid_arn -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/K8-fluentbit-data-prepper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/examples/log-ingestion-k8s/K8-fluentbit-data-prepper.png -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/HighLevelDiagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/examples/trace-analytics-sample-app/HighLevelDiagram.png -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | # These are explicitly windows files and should use crlf 5 | *.bat text eol=crlf 6 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/conditional_route_invalid_non_string.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | testRouteName: 3 | this: is 4 | invalid: format 5 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/conditional_route_invalid_object.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | testRouteName: 3 | this: is 4 | invalid: format 5 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config_with_negative_processor_shutdown_timeout.yml: -------------------------------------------------------------------------------- 1 | server_port: 1234 2 | ssl: true 3 | processor_shutdown_timeout: "PT-6H3M" 4 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_heap_circuit_breaker.yml: -------------------------------------------------------------------------------- 1 | server_port: 1234 2 | ssl: true 3 | 4 | circuit_breakers: 5 | heap: 6 | usage: 2.5gb 7 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/org/opensearch/dataprepper/transforms/rules/test-plugin-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "test-plugin" 2 | apply_when: 3 | - "$..source.documentdb" -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-missing-secret-id.yaml: -------------------------------------------------------------------------------- 1 | region: us-east-1 2 | sts_role_arn: arn:aws:iam::123456789012:role/test-role -------------------------------------------------------------------------------- /examples/dev/README.md: -------------------------------------------------------------------------------- 1 | # Dev Examples 2 | This directory contains example projects intended for developing/testing new features. Examples are not guaranteed to be in a working state. 3 | -------------------------------------------------------------------------------- /performance-test/src/gatling/resources/bodies/singleLog.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "log": "127.0.0.1 - frank [17/Dec/2021:10:00:00 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326" 4 | } 5 | ] -------------------------------------------------------------------------------- /testing/aws-testing-cdk/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !.eslintrc.js 3 | !jest.config.js 4 | *.d.ts 5 | node_modules 6 | 7 | # CDK asset staging directory 8 | .cdk.staging 9 | cdk.out 10 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_metric_filter.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metric_tag_filters: 3 | - pattern: "aws.sdk.**" 4 | tags: 5 | tag1: value1 6 | -------------------------------------------------------------------------------- /docs/images/Log_Ingestion_FluentBit_DataPrepper_OpenSearch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/docs/images/Log_Ingestion_FluentBit_DataPrepper_OpenSearch.jpg -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/resources/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | peer_forwarder: 3 | ssl: false 4 | discovery_mode: "dns" 5 | domain_name: "prepper-cluster" 6 | -------------------------------------------------------------------------------- /release/staging-resources-cdk/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !.eslintrc.js 3 | !jest.config.js 4 | *.d.ts 5 | node_modules 6 | 7 | # CDK asset staging directory 8 | .cdk.staging 9 | cdk.out 10 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/single_pipeline_valid_off_heap_buffer.yml: -------------------------------------------------------------------------------- 1 | test-pipeline-1: 2 | source: 3 | stdin: 4 | buffer: 5 | test_off_heap: 6 | sink: 7 | - stdout: 8 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_config_with_acm_ssl.yml: -------------------------------------------------------------------------------- 1 | ssl: true 2 | use_acm_certificate_for_ssl: true 3 | acm_certificate_arn: "acm_arn" 4 | aws_region: "aws_region" 5 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_zero_buffer_single_thread.yml: -------------------------------------------------------------------------------- 1 | simple-pipeline: 2 | workers: 1 3 | source: 4 | random: 5 | buffer: 6 | zero: 7 | sink: 8 | - stdout: 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/http-source-common/src/main/java/org/opensearch/dataprepper/http/BaseHttpService.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.http; 2 | 3 | public interface BaseHttpService { 4 | } 5 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/test/resources/test_keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-plugins/opensearch/src/test/resources/test_keystore.jks -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/mutate.mapping.yaml: -------------------------------------------------------------------------------- 1 | customPluginMapperClass: org.opensearch.dataprepper.logstash.mapping.mutate.MutateMapper -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/rules/documentdb-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "documentdb" 2 | apply_when: 3 | - "$..source.documentdb" 4 | - "$..source.documentdb.s3_bucket" 5 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_zero_buffer_multiple_threads.yml: -------------------------------------------------------------------------------- 1 | simple-sample-pipeline: 2 | workers: 2 3 | source: 4 | random: 5 | buffer: 6 | zero: 7 | sink: 8 | - stdout: 9 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_iso8601_shutdown_timeouts.yml: -------------------------------------------------------------------------------- 1 | server_port: 5678 2 | ssl: false 3 | sink_shutdown_timeout: "PT15M" 4 | processor_shutdown_timeout: "PT45S" 5 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-plugin-config.yaml: -------------------------------------------------------------------------------- 1 | test_config: 2 | secret_id: test-secret 3 | region: us-east-1 4 | sts_role_arn: arn:aws:iam::123456789012:role/test-role -------------------------------------------------------------------------------- /data-prepper-plugins/mongodb/src/main/resources/org/opensearch/dataprepper/transforms/rules/mongodb-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "mongodb" 2 | apply_when: 3 | - "$..source.mongodb" 4 | - "$..source.mongodb.s3_bucket" -------------------------------------------------------------------------------- /data-prepper-plugins/s3-source/src/main/resources/IntegrationTest.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-plugins/s3-source/src/main/resources/IntegrationTest.parquet -------------------------------------------------------------------------------- /data-prepper-test/test-event/README.md: -------------------------------------------------------------------------------- 1 | # Data Prepper Test Event 2 | 3 | This class provides test classes to help with the Data Prepper event model. 4 | 5 | See in particular the `TestEventFactory` class. 6 | -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/images/grafana_metrics_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/examples/dev/trace-analytics-sample-app/images/grafana_metrics_sample.png -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config_with_tags.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metric_tags: 3 | testKey1: testValue1 4 | testKey2: testValue2 5 | testKey3: testValue3 6 | testKey4: testValue4 7 | -------------------------------------------------------------------------------- /data-prepper-plugins/parquet-codecs/src/test/resources/test-parquet.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-plugins/parquet-codecs/src/test/resources/test-parquet.parquet -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/test/resources/org/opensearch/dataprepper/test/plugins/junit/no-processor.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | sink: 6 | - unused: 7 | -------------------------------------------------------------------------------- /release/archives/linux/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | //This file is intentionally left blank to support commands for this project -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/io/InputFile.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.model.io; 2 | 3 | public interface InputFile extends org.apache.parquet.io.InputFile { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/io/OutputFile.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.model.io; 2 | 3 | public interface OutputFile extends org.apache.parquet.io.OutputFile { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-2.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret 2 | region: us-east-1 3 | sts_role_arn: arn:aws:unknown-service::123456789012:role/test-role -------------------------------------------------------------------------------- /data-prepper-plugins/dummy-plugin/src/main/resources/org/opensearch/dataprepper/transforms/rules/dummy-plugin-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "dummy-plugin" 2 | apply_when: 3 | - "$.test-pipeline.processor[?(@.dummy_plugin)]" 4 | -------------------------------------------------------------------------------- /data-prepper-plugins/parquet-codecs/src/test/resources/sample.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-plugins/parquet-codecs/src/test/resources/sample.snappy.parquet -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | rootProject.name = 'analytics-service' -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | server.port=8087 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_cloud_map_without_namespace_name_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | discovery_mode: "aws_cloud_map" 3 | aws_cloud_map_service_name: "servicename" 4 | aws_region: "us-east-1" -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_cloud_map_without_service_name_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | discovery_mode: "aws_cloud_map" 3 | aws_cloud_map_namespace_name: "namespace" 4 | aws_region: "us-east-1" -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-invalid-sts-3.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret 2 | region: us-east-1 3 | sts_role_arn: arn:aws:iam::123456789012:unknown-resource-type/test-role -------------------------------------------------------------------------------- /data-prepper-plugins/buffer-common/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | dependencies { 6 | implementation project(path: ':data-prepper-api') 7 | } 8 | 9 | test { 10 | useJUnitPlatform() 11 | } -------------------------------------------------------------------------------- /data-prepper-plugins/mongodb/src/main/resources/org/opensearch/dataprepper/transforms/rules/documentdb-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "documentdb" 2 | apply_when: 3 | - "$..source.documentdb" 4 | - "$..source.documentdb.s3_bucket" -------------------------------------------------------------------------------- /data-prepper-plugins/otel-proto-common/src/test/resources/test-otel-log.protobuf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-plugins/otel-proto-common/src/test/resources/test-otel-log.protobuf -------------------------------------------------------------------------------- /e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | peer_forwarder: 3 | port: 4994 4 | ssl: false 5 | discovery_mode: static 6 | static_endpoints: ["data-prepper-0", "data-prepper-1"] 7 | -------------------------------------------------------------------------------- /testing/aws-testing-cdk/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | roots: ['/test'], 4 | testMatch: ['**/*.test.ts'], 5 | transform: { 6 | '^.+\\.tsx?$': 'ts-jest' 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/test_keystore_with_different_passwords.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-core/src/test/resources/tls/test_keystore_with_different_passwords.p12 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/test_keystore_with_identical_passwords.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-core/src/test/resources/tls/test_keystore_with_identical_passwords.p12 -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/mapping/sample-with-empty-maps.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: samplePlugin 2 | attributesMapperClass: org.opensearch.dataprepper.Placeholder 3 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/org/opensearch/dataprepper/transforms/templates/test-plugin-template.yaml: -------------------------------------------------------------------------------- 1 | "<>-transformed": 2 | source: "<<$.<>.source>>" 3 | sink: 4 | - noop: 5 | -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml: -------------------------------------------------------------------------------- 1 | hosts: ["https://jira.com/"] 2 | authentication: 3 | basic: 4 | username: "jiraId" 5 | password: "jiraApiKey" 6 | 7 | -------------------------------------------------------------------------------- /release/staging-resources-cdk/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | roots: ['/test'], 4 | testMatch: ['**/*.test.ts'], 5 | transform: { 6 | '^.+\\.tsx?$': 'ts-jest' 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams 2 | * @sb2k16 @chenqi0805 @engechas @san81 @srikanthjg @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_cloud_map_without_region_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | discovery_mode: "aws_cloud_map" 3 | aws_cloud_map_namespace_name: "namespace" 4 | aws_cloud_map_service_name: "servicename" -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/basic-auth-jira-pipeline.yaml: -------------------------------------------------------------------------------- 1 | hosts: ["https://jira.com/"] 2 | authentication: 3 | basic: 4 | username: "jiraId" 5 | password: "jiraApiKey" 6 | 7 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_config_with_insecure.yml: -------------------------------------------------------------------------------- 1 | ssl: true 2 | ssl_certificate_file: src/test/resources/test-crt.crt 3 | ssl_key_file: src/test/resources/test-key.crt 4 | ssl_insecure_disable_verification: true -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/README.md: -------------------------------------------------------------------------------- 1 | # AWS Plugin 2 | 3 | This plugin provides an extension to Data Prepper for AWS support. 4 | 5 | Other plugins should not use this project directly. Instead, they will get it from Data Prepper. 6 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-proto-common/src/test/resources/test-otel-multi-log.protobuf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-plugins/otel-proto-common/src/test/resources/test-otel-multi-log.protobuf -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml: -------------------------------------------------------------------------------- 1 | hosts: ["https://jira.com/"] 2 | authentication: 3 | basic: 4 | username: "jiraId" 5 | password: "jiraApiKey" 6 | 7 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/single_pipeline_valid_empty_source_plugin_settings.yml: -------------------------------------------------------------------------------- 1 | test-pipeline-1: 2 | source: 3 | stdin: 4 | buffer: 5 | bounded_blocking: #to check non object nodes for plugins 6 | sink: 7 | - stdout: -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_config_with_fingerprint.yml: -------------------------------------------------------------------------------- 1 | ssl: true 2 | ssl_certificate_file: src/test/resources/test-crt.crt 3 | ssl_key_file: src/test/resources/test-key.crt 4 | ssl_fingerprint_verification_only: true -------------------------------------------------------------------------------- /data-prepper-main/src/test/resources/single_pipeline_valid_empty_source_plugin_settings.yml: -------------------------------------------------------------------------------- 1 | test-pipeline-1: 2 | source: 3 | stdin: 4 | buffer: 5 | bounded_blocking: #to check non object nodes for plugins 6 | sink: 7 | - stdout: -------------------------------------------------------------------------------- /data-prepper-plugins/parse-json-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/parse/json/default.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - parse_json: 6 | sink: 7 | - unused: 8 | -------------------------------------------------------------------------------- /data-prepper-plugins/s3-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/s3/ExtensionProvider.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.sink.s3; 2 | 3 | public interface ExtensionProvider { 4 | String getExtension(); 5 | } 6 | -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml: -------------------------------------------------------------------------------- 1 | hosts: [ "https://jira.com/" ] 2 | authentication: 3 | oauth2: 4 | client_id: "client_id" 5 | client_secret: "client_secret" 6 | 7 | -------------------------------------------------------------------------------- /examples/log-ingestion-ecs-firelens/example_log_pipeline.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | http: 4 | processor: 5 | - grok: 6 | match: 7 | log: [ "%{COMMONAPACHELOG}" ] 8 | sink: 9 | - stdout: 10 | -------------------------------------------------------------------------------- /release/docker/adoptium.repo: -------------------------------------------------------------------------------- 1 | [adoptium] 2 | name=Adoptium 3 | baseurl=https://packages.adoptium.net/artifactory/rpm/amazonlinux/2/$basearch 4 | enabled=1 5 | gpgcheck=1 6 | gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public 7 | -------------------------------------------------------------------------------- /release/release-notes/data-prepper.release-notes-2.10.1.md: -------------------------------------------------------------------------------- 1 | ## 2024-10-21 Version 2.10.1 2 | 3 | --- 4 | 5 | ### Bug Fixes 6 | * [BUG] Kinesis source is failing on startup ([#5084](https://github.com/opensearch-project/data-prepper/issues/5084)) -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/conditional_route_list.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - testRouteName0: "/my/property==value0" 3 | - testRouteName1: "/my/property==value1" 4 | - testRouteName2: "/my/property==value2" 5 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sink_plugin.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | customSinkPlugin: 3 | routes: 4 | - "routeA" 5 | - "routeB" 6 | tags_target_key: "tags" 7 | key1: "value1" 8 | key2: "value2" 9 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/pipelines_data_flow_serialized.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | testSource: null 4 | processor: 5 | - testProcessor: null 6 | sink: 7 | - testSink: null 8 | workers: 8 9 | delay: 50 10 | -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/oauth2-auth-jira-pipeline.yaml: -------------------------------------------------------------------------------- 1 | hosts: [ "https://jira.com/" ] 2 | authentication: 3 | oauth2: 4 | client_id: "client_id" 5 | client_secret: "client_secret" 6 | 7 | -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml: -------------------------------------------------------------------------------- 1 | hosts: [ "https://jira.com/" ] 2 | authentication: 3 | oauth2: 4 | client_id: "client_id" 5 | client_secret: "client_secret" 6 | 7 | -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/test/resources/org/opensearch/dataprepper/test/plugins/junit/multiple-processors.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - one: 6 | - two: 7 | sink: 8 | - unused: 9 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/pipeline_with_version.yaml: -------------------------------------------------------------------------------- 1 | version: "2.0" 2 | test-pipeline: 3 | source: 4 | testSource: null 5 | processor: 6 | - testProcessor: null 7 | sink: 8 | - testSink: null 9 | workers: 8 10 | delay: 50 11 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/resources/META-INF/services/org.opensearch.dataprepper.plugin.PluginProvider: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | org.opensearch.dataprepper.plugin.ClasspathPluginProvider -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config_with_metric_filter.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metric_tag_filters: 3 | - pattern: "aws.sdk.*" 4 | tags: 5 | tag1: value1 6 | tag2: value2 7 | tag3: value3 8 | tag4: value4 9 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/templates/testSource/documentdb-simple-template.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | source: "<<$..source>>" 3 | sink: 4 | - s3: 5 | bucket_name: "<<$.<>.source.hostname>>" -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-with-sts.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret 2 | region: us-east-1 3 | sts_role_arn: arn:aws:iam::123456789012:role/test-role 4 | sts_header_overrides: 5 | test_key: "test_value" -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /docs/log_analytics.md: -------------------------------------------------------------------------------- 1 | # Log Analytics 2 | 3 | This has been moved to the OpenSearch Data Prepper documentation for 4 | [Log Analytics](https://docs.opensearch.org/docs/latest/data-prepper/common-use-cases/log-analytics/) 5 | in the **Common use cases** section. 6 | -------------------------------------------------------------------------------- /docs/migrating_from_opendistro.md: -------------------------------------------------------------------------------- 1 | # Migrating from Open Distro Data Prepper 2 | 3 | This has been moved to the OpenSearch Data Prepper documentation for 4 | [Migrating from Open Distro](https://docs.opensearch.org/docs/latest/data-prepper/migrate-open-distro/). 5 | -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/data-prepper-pipeline-config/example_log_pipeline.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | http: 4 | processor: 5 | - grok: 6 | match: 7 | log: [ "%{COMMONAPACHELOG}" ] 8 | sink: 9 | - stdout: 10 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/resources/cloudwatch.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | cloudwatch.enabled=true 7 | cloudwatch.namespace=dataprepper 8 | cloudwatch.batchSize=20 9 | cloudwatch.step=PT1M -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/compatible_version.yml: -------------------------------------------------------------------------------- 1 | version: "2" # update with every major version change 2 | test-pipeline-1: 3 | source: 4 | stdin: 5 | buffer: 6 | bounded_blocking: #to check non object nodes for plugins 7 | sink: 8 | - stdout: -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_zero_buffer_with_single_thread_processor.yml: -------------------------------------------------------------------------------- 1 | simple-pipeline: 2 | workers: 1 3 | source: 4 | random: 5 | buffer: 6 | zero: 7 | processor: 8 | - test_processor: 9 | sink: 10 | - stdout: 11 | -------------------------------------------------------------------------------- /data-prepper-main/src/test/resources/logstash-filter.conf: -------------------------------------------------------------------------------- 1 | input { http { } } 2 | 3 | filter { 4 | grok { 5 | match => { "message" => "%{COMBINEDAPACHELOG}" } 6 | } 7 | } 8 | 9 | output { 10 | elasticsearch { hosts => ["localhost:9200"] } 11 | } 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin/src/test/resources/test-aws-secret-manager-configuration-with-sts-headers.yaml: -------------------------------------------------------------------------------- 1 | secret_id: test-secret 2 | region: us-east-1 3 | sts_role_arn: arn:aws:iam::123456789012:role/test-role 4 | sts_header_overrides: 5 | test-header: test-value -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/test/resources/org/opensearch/dataprepper/test/plugins/test001.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - test_plugin: 6 | option_a: some-value-for-a 7 | sink: 8 | - unused: 9 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/java/org/opensearch/dataprepper/model/log/.JacksonStandardOTelLogTest.java.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/log/.JacksonStandardOTelLogTest.java.swp -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/pipeline_with_short_hand_version.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | test-pipeline: 3 | source: 4 | testSource: null 5 | processor: 6 | - testProcessor: null 7 | sink: 8 | - testSink: null 9 | workers: 8 10 | delay: 50 11 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_acm_without_arn_config.yml: -------------------------------------------------------------------------------- 1 | ssl: true 2 | ssl_certificate_file: "src/test/resources/test-crt.crt" 3 | ssl_key_file: "src/test/resources/test-key.key" 4 | use_acm_certificate_for_ssl: true 5 | aws_region: "us-east-1" -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/main/resources/META-INF/services/jakarta.validation.ConstraintValidator: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | org.opensearch.dataprepper.core.validators.NotEmptyValidatorForEventKey -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/incompatible_version.yml: -------------------------------------------------------------------------------- 1 | 2 | version: "3005.0" #far out future version. 3 | test-pipeline-1: 4 | source: 5 | stdin: 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - stdout: -------------------------------------------------------------------------------- /data-prepper-plugins/dynamodb-source/README.md: -------------------------------------------------------------------------------- 1 | # DynamoDB Source 2 | 3 | This source ingests data to Data Prepper from DynamoDB 4 | 5 | See the [`dynamodb` source documentation](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/dynamo-db/) 6 | -------------------------------------------------------------------------------- /docs/trace_analytics.md: -------------------------------------------------------------------------------- 1 | # Trace Analytics 2 | 3 | This has been moved to the OpenSearch Data Prepper documentation for 4 | [Trace Analytics](https://docs.opensearch.org/docs/latest/data-prepper/common-use-cases/trace-analytics/) 5 | in the **Common use cases** section. 6 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/java/org/opensearch/dataprepper/model/metric/.JacksonStandardSummaryTest.java.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/data-prepper/HEAD/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/metric/.JacksonStandardSummaryTest.java.swp -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/pipeline_with_extension.yaml: -------------------------------------------------------------------------------- 1 | extension: 2 | test_extension: 3 | test-pipeline: 4 | source: 5 | testSource: null 6 | processor: 7 | - testProcessor: null 8 | sink: 9 | - testSink: null 10 | workers: 8 11 | delay: 50 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/cloudwatch_test.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | cloudwatch.enabled=true 7 | cloudwatch.namespace=dataprepper-test 8 | cloudwatch.batchSize=20 9 | cloudwatch.step=PT1S -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/mapping/no_plugin_name.mapping.yaml: -------------------------------------------------------------------------------- 1 | mappedAttributeNames: 2 | hosts: hosts 3 | index: index 4 | region: aws_region 5 | additionalAttributes: 6 | aws_sigv4: true 7 | insecure: false -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/integrationTest/resources/kafka/zookeeper/sasl-plaintext/kafka_server_jaas.conf: -------------------------------------------------------------------------------- 1 | KafkaServer { 2 | org.apache.kafka.common.security.plain.PlainLoginModule required 3 | username="admin" 4 | password="admin" 5 | user_admin="admin"; 6 | }; -------------------------------------------------------------------------------- /data-prepper-plugins/kinesis-source/src/test/resources/simple_pipeline_with_extensions.yaml: -------------------------------------------------------------------------------- 1 | extensions: 2 | kinesis: 3 | pipeline_identifier: "sample-kinesis-pipeline-0123" 4 | lease_coordination: 5 | table_name: "kinesis-pipeline-kcl" 6 | region: "us-east-1" -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/main/resources/META-INF/services/org.opensearch.dataprepper.plugin.PluginProvider: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | org.opensearch.dataprepper.plugin.ClasspathPluginProvider -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/minimal-pipeline.yaml: -------------------------------------------------------------------------------- 1 | minimal-pipeline: 2 | delay: 10 3 | source: 4 | in_memory: 5 | testing_key: MinimalPipelineIT 6 | sink: 7 | - in_memory: 8 | testing_key: MinimalPipelineIT -------------------------------------------------------------------------------- /data-prepper-main/src/main/resources/META-INF/data-prepper.plugins.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | org.opensearch.dataprepper.plugin.packages=org.opensearch.dataprepper.plugins,com.amazon.dataprepper.plugins -------------------------------------------------------------------------------- /data-prepper-main/src/test/resources/logstash-conf/logstash-filter.conf: -------------------------------------------------------------------------------- 1 | input { http { } } 2 | 3 | filter { 4 | grok { 5 | match => { "message" => "%{COMBINEDAPACHELOG}" } 6 | } 7 | } 8 | 9 | output { 10 | elasticsearch { hosts => ["localhost:9200"] } 11 | } 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/personalize-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/personalize/configuration/PersonalizeAdvancedValidation.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.sink.personalize.configuration; 2 | 3 | interface PersonalizeAdvancedValidation { 4 | } 5 | -------------------------------------------------------------------------------- /docs/core_apis.md: -------------------------------------------------------------------------------- 1 | # Core Data Prepper APIs 2 | 3 | This has been moved to the OpenSearch Data Prepper documentation for 4 | [Core APIs](https://docs.opensearch.org/docs/latest/data-prepper/managing-data-prepper/core-apis/) 5 | in the **Managing OpenSearch Data Prepper** section. 6 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_with_acm_without_region_config.yml: -------------------------------------------------------------------------------- 1 | ssl: true 2 | ssl_certificate_file: "src/test/resources/test-crt.crt" 3 | ssl_key_file: "src/test/resources/test-key.key" 4 | use_acm_certificate_for_ssl: true 5 | acm_certificate_arn: "sample_arn" -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/mapping/invalid.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: 2 | mappedAttributeNames: 3 | hosts: hosts 4 | cacert: cert 5 | user: username 6 | password: password 7 | index: index 8 | proxy: proxy -------------------------------------------------------------------------------- /data-prepper-plugins/parse-json-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/parse/json/with-destination.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - parse_json: 6 | destination: parsed_json 7 | sink: 8 | - unused: 9 | -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/my-app-02-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: my-nginx 5 | namespace: my-app 6 | labels: 7 | run: my-nginx 8 | spec: 9 | ports: 10 | - port: 80 11 | protocol: TCP 12 | selector: 13 | run: my-nginx -------------------------------------------------------------------------------- /docs/peer_forwarder.md: -------------------------------------------------------------------------------- 1 | ## Peer Forwarder 2 | 3 | This has been moved to the OpenSearch Data Prepper documentation for 4 | [Peer Forwarder](https://docs.opensearch.org/docs/latest/data-prepper/managing-data-prepper/peer-forwarder/) 5 | in the **Managing OpenSearch Data Prepper** section. 6 | -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/fluent-bit-03-role-1.22.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: fluent-bit-read 5 | rules: 6 | - apiGroups: [""] 7 | resources: 8 | - namespaces 9 | - pods 10 | verbs: ["get", "list", "watch"] 11 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/mapping/amazon_es.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: opensearch 2 | mappedAttributeNames: 3 | hosts: hosts 4 | index: index 5 | region: aws_region 6 | additionalAttributes: 7 | aws_sigv4: true 8 | insecure: false -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/pipeline_with_depreciated_extension.yaml: -------------------------------------------------------------------------------- 1 | pipeline_configurations: 2 | test_extension: 3 | test-pipeline: 4 | source: 5 | testSource: null 6 | processor: 7 | - testProcessor: null 8 | sink: 9 | - testSink: null 10 | workers: 8 11 | delay: 50 -------------------------------------------------------------------------------- /data-prepper-plugins/drop-events-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/drop/drop_when_value_is_empty_string.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - drop_events: 6 | drop_when: /my_key == "" 7 | sink: 8 | - unused: 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/configuration/KafkaIsolationLevelConfig.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.kafka.configuration; 2 | 3 | public interface KafkaIsolationLevelConfig { 4 | IsolationLevel getIsolationLevel(); 5 | } 6 | -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/jira-source/README.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | ### Counter 4 | 5 | - `issuesRequested`: measures total number of issue Requests sent. 6 | 7 | ### Timer 8 | 9 | - `requestProcessDuration`: measures latency of requests processed by the jira source plugin. 10 | -------------------------------------------------------------------------------- /release/release-notes/data-prepper.release-notes-2.10.3.md: -------------------------------------------------------------------------------- 1 | ## 2025-04-16 Version 2.10.3 2 | 3 | --- 4 | 5 | ### Security 6 | * Updates Parquet to 1.15.1. Fixes [CVE-2025-30065](https://github.com/advisories/GHSA-2c59-37c4-qrx5). ([#5597](https://github.com/opensearch-project/data-prepper/pull/5597)) 7 | -------------------------------------------------------------------------------- /data-prepper-plugins/dummy-plugin/src/main/resources/org/opensearch/dataprepper/transforms/templates/dummy-plugin-template.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | file: 4 | path: "/tmp/input-file.log" 5 | processor: 6 | - string_converter: 7 | upper_case: true 8 | sink: 9 | - noop: -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/confluence-source/README.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | ### Counter 4 | 5 | - `issuesRequested`: measures total number of issue Requests sent. 6 | 7 | ### Timer 8 | 9 | - `requestProcessDuration`: measures latency of requests processed by the jira source plugin. 10 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_zero_buffer_multiple_threads_no_single_thread_processors.yml: -------------------------------------------------------------------------------- 1 | simple-pipeline: 2 | workers: 2 3 | source: 4 | random: 5 | buffer: 6 | zero: 7 | processor: 8 | - string_converter: 9 | upper_case: true 10 | sink: 11 | - stdout: 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | # To enable mocking of final classes with vanilla Mockito 2 | # https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods 3 | mock-maker-inline 4 | -------------------------------------------------------------------------------- /data-prepper-plugins/ocsf/src/main/resources/org/opensearch/dataprepper/transforms/rules/ocsf-v1.1-panw-traffic-rule.yaml: -------------------------------------------------------------------------------- 1 | plugin_name: "ocsf-v1.1-panw-traffic" 2 | apply_when: 3 | - "$..processor[?(@.ocsf.type == 'palo_alto_networks_traffic_logs')]" 4 | - "$..processor[?(@.ocsf.version == '1.1')]" 5 | 6 | 7 | -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | 6 | subprojects { 7 | apply plugin: 'data-prepper.publish' 8 | } 9 | 10 | dependencies { 11 | subprojects.forEach { api project(':data-prepper-plugins::saas-source-plugins:' + it.name) } 12 | } -------------------------------------------------------------------------------- /data-prepper-test/test-common/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | dependencies { 7 | implementation testLibs.hamcrest 8 | testRuntimeOnly testLibs.junit.engine 9 | } 10 | 11 | test { 12 | useJUnitPlatform() 13 | } -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/org/opensearch/dataprepper/core/pipeline/SupportsPipelineRunner.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.core.pipeline; 2 | 3 | public interface SupportsPipelineRunner { 4 | PipelineRunner getPipelineRunner(); 5 | 6 | void setPipelineRunner(PipelineRunner pipelineRunner); 7 | } 8 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/mapping/sample-with-nulls.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: samplePlugin 2 | attributesMapperClass: org.opensearch.dataprepper.Placeholder 3 | mappedAttributeNames: null 4 | additionalAttributes: null 5 | nestedSyntaxAttributeNames: null 6 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/multi-pipelines/valid_multiple_pipeline_configuration2.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-2: 3 | source: 4 | pipeline: 5 | name: "test-pipeline-1" 6 | sink: 7 | - pipeline: 8 | name: "test-pipeline-3" 9 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/multi-pipelines/valid_multiple_pipeline_configuration3.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-3: 3 | source: 4 | pipeline: 5 | name: "test-pipeline-2" 6 | sink: 7 | - file: 8 | path: "/tmp/todelete.txt" 9 | -------------------------------------------------------------------------------- /data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/TestPluggableInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugin; 7 | 8 | public interface TestPluggableInterface { 9 | } 10 | -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: false 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "70...100" 8 | status: 9 | project: 10 | default: 11 | target: 70% # the required coverage value 12 | threshold: 1% # the leniency in hitting the target 13 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/pipelines_data_flow_route.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | testSource: null 4 | processor: 5 | - testPrepper: null 6 | routes: 7 | - my-route: "/a==b" 8 | sink: 9 | - testSink: 10 | routes: 11 | - "my-route" 12 | workers: 8 13 | delay: 50 14 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/pipelines_data_flow_routes.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | testSource: null 4 | processor: 5 | - testPrepper: null 6 | routes: 7 | - my-route: "/a==b" 8 | sink: 9 | - testSink: 10 | routes: 11 | - "my-route" 12 | workers: 8 13 | delay: 50 14 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration2.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-2: 3 | source: 4 | pipeline: 5 | name: "test-pipeline-1" 6 | sink: 7 | - pipeline: 8 | name: "test-pipeline-3" 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/avro-codecs/src/main/java/org/opensearch/dataprepper/avro/SchemaGenerationException.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.avro; 2 | 3 | public class SchemaGenerationException extends RuntimeException { 4 | SchemaGenerationException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkFlushContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.common.sink; 7 | 8 | public interface SinkFlushContext { 9 | } 10 | 11 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_source_coordination_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | source_coordination: 3 | partition_prefix: "some-prefix" 4 | store: 5 | dynamodb: 6 | table_name: "Test" 7 | region: "us-east-1" 8 | sts_role_arn: "my_arn" 9 | metric_registries: [Prometheus, CloudWatch] -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/validation/PluginErrorsHandler.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.validation; 2 | 3 | import java.util.Collection; 4 | 5 | public interface PluginErrorsHandler { 6 | 7 | public void handleErrors(final Collection pluginErrors); 8 | } 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/single_match_single_pattern.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{COMMONAPACHELOG}"] 8 | 9 | sink: 10 | - unused: 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/user-agent-processor/README.md: -------------------------------------------------------------------------------- 1 | # User Agent Processor 2 | This processor parses User-Agent (UA) string in an event and add the parsing result to the event. 3 | 4 | See the [`user_agent` processor documentation](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/user-agent/). 5 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/multi-pipelines-single-pipeline-configurations/pipeline_configuration2.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-2: 3 | source: 4 | pipeline: 5 | name: "test-pipeline-1" 6 | sink: 7 | - pipeline: 8 | name: "test-pipeline-3" 9 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/ConverterArguments.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.typeconverter; 2 | 3 | /** 4 | * Interface for arguments passed to the {@link TypeConverter} 5 | * 6 | * @since 1.2 7 | */ 8 | public interface ConverterArguments { 9 | int getScale(); 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExtensionsConfiguration.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugin; 2 | 3 | import org.opensearch.dataprepper.model.configuration.PipelineExtensions; 4 | 5 | public interface ExtensionsConfiguration { 6 | PipelineExtensions getPipelineExtensions(); 7 | } 8 | -------------------------------------------------------------------------------- /data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugins/test/TestComponent.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.test; 2 | 3 | import javax.inject.Named; 4 | 5 | @Named 6 | public class TestComponent { 7 | public String getIdentifier() { 8 | return "test-component"; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/s3/FileType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.opensearch.s3; 7 | 8 | public enum FileType { 9 | JSON 10 | } 11 | -------------------------------------------------------------------------------- /.ci/documentation/issue.md: -------------------------------------------------------------------------------- 1 | **Is your feature request related to a problem?** 2 | 3 | A new feature has been added. 4 | 5 | **What solution would you like?** 6 | 7 | Document the usage of the new feature. 8 | 9 | **What alternatives have you considered?** 10 | 11 | N/A 12 | 13 | **Do you have any additional context?** 14 | 15 | See issue -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_basic_authentication.yml: -------------------------------------------------------------------------------- 1 | server_port: 1234 2 | ssl: true 3 | key_store_file_path: "src/test/resources/test.p12" 4 | key_store_password: "store" 5 | private_key_password: "key" 6 | authentication: 7 | http_basic: 8 | username: testuser 9 | password: testpass 10 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/userConfig/documentdb-simple-userconfig.yaml: -------------------------------------------------------------------------------- 1 | dodb-pipeline: 2 | source: 3 | documentdb: 4 | hostname: "database.example.com" 5 | sink: 6 | - s3: 7 | bucket_name: "test_bucket" 8 | aws: 9 | region: "us-test-1" 10 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/LockStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.common.sink; 7 | 8 | public interface LockStrategy { 9 | void lock(); 10 | void unlock(); 11 | } -------------------------------------------------------------------------------- /data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/Fields/SkipField.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.processor.dissect.Fields; 2 | 3 | public class SkipField extends Field { 4 | 5 | public SkipField(String key) { 6 | this.setKey(key); 7 | } 8 | 9 | } 10 | -------------------------------------------------------------------------------- /release/docker/config/default-data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: true 2 | keyStoreFilePath: "/usr/share/data-prepper/keystore.p12" 3 | keyStorePassword: "password" 4 | privateKeyPassword: "password" 5 | serverPort: 4900 6 | metricRegistries: [Prometheus] 7 | authentication: 8 | http_basic: 9 | username: admin 10 | password: admin 11 | -------------------------------------------------------------------------------- /data-prepper-main/src/main/resources/META-INF/services/com.linecorp.armeria.client.ClientFactoryProvider: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | com.linecorp.armeria.internal.client.grpc.protocol.UnaryGrpcClientFactoryProvider 7 | com.linecorp.armeria.internal.client.grpc.GrpcClientFactoryProvider -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/userConfig/documentdb1-userconfig.yaml: -------------------------------------------------------------------------------- 1 | simple-pipeline: 2 | source: 3 | documentdb: 4 | hostname: "database.example.com" 5 | port: "27017" 6 | sink: 7 | - opensearch: 8 | hosts: ["https://search-service.example.com"] 9 | index: "my_index" 10 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/accumulator/BufferFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.accumulator; 7 | 8 | public interface BufferFactory { 9 | Buffer getBuffer(); 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/dissect-processor/README.md: -------------------------------------------------------------------------------- 1 | # Dissect Processor 2 | 3 | The dissect processor extracts values from an event and maps them to individual fields based on user-defined dissect patterns. 4 | 5 | See the [`dissect` processor documentation](https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/dissect/). 6 | -------------------------------------------------------------------------------- /data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/Fields/NormalField.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.processor.dissect.Fields; 2 | 3 | public class NormalField extends Field { 4 | 5 | public NormalField(String key) { 6 | this.setKey(key); 7 | } 8 | 9 | } 10 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/codec/DecompressionEngine.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.model.codec; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | 6 | public interface DecompressionEngine { 7 | InputStream createInputStream(final InputStream responseInputStream) throws IOException; 8 | } 9 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/org/opensearch/dataprepper/DataPrepperShutdownListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper; 7 | 8 | @FunctionalInterface 9 | public interface DataPrepperShutdownListener { 10 | void handleShutdown(); 11 | } 12 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/http-to-opensearch.conf: -------------------------------------------------------------------------------- 1 | input { 2 | http { 3 | } 4 | } 5 | output { 6 | elasticsearch { 7 | hosts => ["https://localhost:19000"] 8 | user => myuser 9 | password => mypassword 10 | index => "simple-pipeline" 11 | } 12 | } -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/test/resources/org/opensearch/dataprepper/test/plugins/junit/valid.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - some_test_plugin: 6 | option_a: some-value-for-a 7 | option_b: some-value-for-b 8 | option_c: some-value-for-c 9 | sink: 10 | - unused: 11 | -------------------------------------------------------------------------------- /release/script/release-notes/release-notes-template.md: -------------------------------------------------------------------------------- 1 | ## YYYY-MM-DD Version M.m.p 2 | 3 | --- 4 | 5 | ### Breaking Changes 6 | 7 | 8 | ### Features 9 | 10 | 11 | ### Enhancements 12 | 13 | 14 | ### Bug Fixes 15 | 16 | 17 | ### Security 18 | 19 | 20 | ### Maintenance 21 | 22 | 23 | # TODO: MOVE CONTENT FROM BELOW TO THE DESIRED SECTION 24 | -------------------------------------------------------------------------------- /data-prepper-event/src/main/java/org/opensearch/dataprepper/event/EventConfigurationContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.event; 7 | 8 | public interface EventConfigurationContainer { 9 | EventConfiguration getEventConfiguration(); 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/simplelogger.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | org.slf4j.simpleLogger.showDateTime=true 7 | org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd' 'HH:mm:ss.SSS 8 | org.slf4j.simpleLogger.log.org.opensearch.dataprepper.plugins.lambda.sink=trace 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/Fields/IndirectField.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.processor.dissect.Fields; 2 | 3 | public class IndirectField extends Field { 4 | 5 | 6 | public IndirectField(String key) { 7 | this.setKey(key); 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/event-json-codecs/src/main/java/org/opensearch/dataprepper/plugins/codec/event_json/EventJsonOutputCodecConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | package org.opensearch.dataprepper.plugins.codec.event_json; 6 | 7 | public class EventJsonOutputCodecConfig { 8 | } 9 | 10 | -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/test/resources/org/opensearch/dataprepper/test/plugins/junit/multiple-pipelines.yaml: -------------------------------------------------------------------------------- 1 | test-pipeline-1: 2 | source: 3 | unused: 4 | processor: 5 | - any: 6 | sink: 7 | - unused: 8 | test-pipeline-2: 9 | source: 10 | unused: 11 | processor: 12 | - any: 13 | sink: 14 | - unused: 15 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/codec/HasByteDecoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.codec; 7 | 8 | public interface HasByteDecoder { 9 | default ByteDecoder getDecoder() { 10 | return null; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/armeria-common/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright OpenSearch Contributors 4 | * SPDX-License-Identifier: Apache-2.0 5 | */ 6 | 7 | dependencies { 8 | implementation project(':data-prepper-api') 9 | implementation libs.armeria.core 10 | implementation libs.armeria.grpc 11 | testImplementation libs.armeria.junit 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_keep_empty_captures_true.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{COMMONAPACHELOG}"] 8 | keep_empty_captures: true 9 | 10 | sink: 11 | - unused: 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/SearchWorker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.source.opensearch.worker; 7 | 8 | public interface SearchWorker extends Runnable { 9 | } 10 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/encryption/KeyProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.encryption; 7 | 8 | @FunctionalInterface 9 | public interface KeyProvider { 10 | byte[] decryptKey(byte[] encryptedKey); 11 | } 12 | -------------------------------------------------------------------------------- /examples/log-ingestion/fluent-bit.conf: -------------------------------------------------------------------------------- 1 | [INPUT] 2 | name tail 3 | refresh_interval 5 4 | path /var/log/test.log 5 | read_from_head true 6 | 7 | [OUTPUT] 8 | Name http 9 | Match * 10 | Host data-prepper 11 | Port 2021 12 | URI /log/ingest 13 | Format json 14 | HTTP_User admin 15 | HTTP_Passwd admin 16 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventFailureMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.event; 7 | 8 | public interface EventFailureMetadata { 9 | EventFailureMetadata with(String key, Object value); 10 | } 11 | 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_named_captures_syntax.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{GREEDYDATA:greedy_data} (?\\d\\d\\d-\\d\\d\\d-\\d\\d\\d)"] 8 | 9 | sink: 10 | - unused: 11 | -------------------------------------------------------------------------------- /examples/dev/data-prepper-emf-monitoring/fluent-bit.conf: -------------------------------------------------------------------------------- 1 | # TCP input used for EMF payloads 2 | [INPUT] 3 | Name tcp 4 | Listen 0.0.0.0 5 | Port 25888 6 | Chunk_Size 32 7 | Buffer_Size 64 8 | Format none 9 | Tag emf-test 10 | 11 | [OUTPUT] 12 | Name stdout 13 | Match emf-* 14 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/acknowledgements/simple-test.yaml: -------------------------------------------------------------------------------- 1 | simple-pipeline-test: 2 | delay: 10 3 | source: 4 | in_memory: 5 | testing_key: PipelinesWithAcksIT 6 | acknowledgments: true 7 | sink: 8 | - in_memory: 9 | testing_key: PipelinesWithAcksIT 10 | acknowledgments: true 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_no_captures_with_existing_and_non_existing_key.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{SYSLOGBASE}"] 8 | bad_key: ["%{SYSLOGBASE}"] 9 | 10 | sink: 11 | - unused: 12 | -------------------------------------------------------------------------------- /examples/zipkin-sleuth-webmvc-example/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | zipkin: 3 | 4 | exporters: 5 | otlp/2: 6 | endpoint: data-prepper:21890 7 | tls: 8 | insecure: true 9 | insecure_skip_verify: true 10 | logging: 11 | 12 | service: 13 | pipelines: 14 | traces: 15 | receivers: [zipkin] 16 | exporters: [logging, otlp/2] 17 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/mapping/sample.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: samplePlugin 2 | attributesMapperClass: org.opensearch.dataprepper.Placeholder 3 | mappedAttributeNames: 4 | valueA: keyA 5 | valueB: keyB 6 | additionalAttributes: 7 | addA: true 8 | addB: staticValueB 9 | nestedSyntaxAttributeNames: 10 | - valueA -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/fluent-bit-04-role-binding-1.22.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: fluent-bit-read 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: fluent-bit-read 9 | subjects: 10 | - kind: ServiceAccount 11 | name: fluent-bit 12 | namespace: logging 13 | 14 | -------------------------------------------------------------------------------- /examples/metrics-ingestion-otel/metric_pipeline.yaml: -------------------------------------------------------------------------------- 1 | metric-pipeline: 2 | source: 3 | otel_metrics_source: 4 | ssl: false 5 | processor: 6 | - otel_metrics: 7 | sink: 8 | - opensearch: 9 | hosts: [ "https://opensearch:9200" ] 10 | insecure: true 11 | username: admin 12 | password: Developer@123 13 | index: otel_metrics 14 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_config_with_drain_timeout.yml: -------------------------------------------------------------------------------- 1 | port: 21895 2 | request_timeout: 1000 3 | server_thread_count: 100 4 | max_connection_count: 100 5 | max_pending_requests: 512 6 | ssl: false 7 | use_acm_certificate_for_ssl: false 8 | discovery_mode: static 9 | client_thread_count: 100 10 | batch_size: 100 11 | buffer_size: 100 12 | drain_timeout: 60s -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_named_captures_only_false.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{GREEDYDATA} %{IPORHOST:host} %{NUMBER}"] 8 | named_captures_only: false 9 | 10 | sink: 11 | - unused: 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/common/KafkaMdc.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.kafka.common; 7 | 8 | public class KafkaMdc { 9 | public static final String MDC_KAFKA_PLUGIN_KEY = "kafkaPluginType"; 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/exception/S3RetriesExhaustedException.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.source.s3.exception; 2 | 3 | public class S3RetriesExhaustedException extends RuntimeException { 4 | 5 | public S3RetriesExhaustedException(final String errorMessage) { 6 | super(errorMessage); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/CrawlerApplicationContextMarker.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.source.source_crawler; 2 | 3 | /** 4 | * Market interface to indicate the base package to scan for dependency injection 5 | */ 6 | public interface CrawlerApplicationContextMarker { 7 | } 8 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/TypeConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.typeconverter; 7 | 8 | public interface TypeConverter { 9 | T convert(Object source); 10 | T convert(Object source, ConverterArguments arguments); 11 | } 12 | -------------------------------------------------------------------------------- /data-prepper-event/src/main/java/org/opensearch/dataprepper/event/EventBuilderFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.event; 7 | 8 | abstract class EventBuilderFactory { 9 | abstract Class getEventClass(); 10 | 11 | abstract DefaultBaseEventBuilder createNew(); 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/exception/InvalidBufferTypeException.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.exception; 2 | 3 | public class InvalidBufferTypeException extends RuntimeException { 4 | public InvalidBufferTypeException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /examples/dev/k8s/build_images_for_minikube.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright OpenSearch Contributors 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | 7 | eval $(minikube docker-env) 8 | docker build ../../trace-analytics-sample-app/sample-app -t example-k8s/sample-app 9 | docker build ../../.. -f ../../../examples/dev/trace-analytics-sample-app/Dockerfile -t example-k8s/data-prepper 10 | -------------------------------------------------------------------------------- /release/script/release-notes/format-release-notes.py: -------------------------------------------------------------------------------- 1 | #!python3 2 | 3 | # 4 | # Copyright OpenSearch Contributors 5 | # SPDX-License-Identifier: Apache-2.0 6 | # 7 | 8 | import json 9 | import sys 10 | 11 | json_input = sys.stdin.read() 12 | 13 | issues = json.loads(json_input) 14 | 15 | for issue in issues: 16 | 17 | print(f'* {issue["title"]} ([#{issue["number"]}]({issue["url"]}))') 18 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/missing_source_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | stdin: 5 | buffer: 6 | bounded_blocking: #to check non object nodes for plugins 7 | sink: 8 | - pipeline: 9 | name: "test-pipeline-2" 10 | test-pipeline-2: 11 | sink: 12 | - file: 13 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_config_with_iso8601_drain_timeout.yml: -------------------------------------------------------------------------------- 1 | port: 21895 2 | request_timeout: 1000 3 | server_thread_count: 100 4 | max_connection_count: 100 5 | max_pending_requests: 512 6 | ssl: false 7 | use_acm_certificate_for_ssl: false 8 | discovery_mode: static 9 | client_thread_count: 100 10 | batch_size: 100 11 | buffer_size: 100 12 | drain_timeout: "PT15S" -------------------------------------------------------------------------------- /examples/jaeger-hotrod/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | jaeger: 3 | protocols: 4 | grpc: 5 | 6 | exporters: 7 | otlp/2: 8 | endpoint: data-prepper:21890 9 | tls: 10 | insecure: true 11 | insecure_skip_verify: true 12 | logging: 13 | 14 | service: 15 | pipelines: 16 | traces: 17 | receivers: [jaeger] 18 | exporters: [logging, otlp/2] 19 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/multi-pipelines/valid_multiple_pipeline_configuration1.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | path: "/tmp/file-source.tmp" 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - pipeline: 10 | name: "test-pipeline-2" 11 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/templates/testSource/documentdb-function-template.yaml: -------------------------------------------------------------------------------- 1 | "<>-transformed": 2 | source: "<<$.<>.source>>" 3 | sink: 4 | - opensearch: 5 | hosts: "<<$.<>.source.documentdb.hostname>>" 6 | depth: "<>.source.documentdb.s3_prefix>>" -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_camel_case_options.yml: -------------------------------------------------------------------------------- 1 | serverPort: 1234 2 | ssl: true 3 | keyStoreFilePath: "src/test/resources/test.p12" 4 | keyStorePassword: "store" 5 | privateKeyPassword: "key" 6 | authentication: 7 | http_basic: 8 | username: testuser 9 | password: testpass 10 | metricTags: 11 | testKey1: testValue1 12 | metricRegistries: [CloudWatch] 13 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_config_with_unauthenticated.yml: -------------------------------------------------------------------------------- 1 | port: 21895 2 | request_timeout: 1000 3 | server_thread_count: 100 4 | max_connection_count: 100 5 | max_pending_requests: 512 6 | ssl: false 7 | use_acm_certificate_for_ssl: false 8 | discovery_mode: static 9 | client_thread_count: 100 10 | batch_size: 100 11 | buffer_size: 100 12 | authentication: 13 | unauthenticated: -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/csv.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: csv 2 | attributesMapperClass: org.opensearch.dataprepper.logstash.mapping.CsvLogstashPluginAttributesMapper 3 | mappedAttributeNames: 4 | columns: column_names 5 | quote_char: quote_character 6 | separator: delimiter 7 | source: source 8 | additionalAttributes: 9 | delete_header: true -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/test/resources/service-map-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "hashId": "aQ/2NNEmtuwsGAOR5ntCNwk=", 3 | "serviceName": "Payment", 4 | "kind": "Client", 5 | "target": 6 | { 7 | "domain": "Purchase", 8 | "resource": "Buy" 9 | }, 10 | "destination": 11 | { 12 | "domain": "Purchase", 13 | "resource": "Buy" 14 | }, 15 | "traceGroupName": "MakePayement.auto" 16 | } -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/kafka-pipeline-no-auth-ssl.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline : 2 | source: 3 | kafka: 4 | bootstrap_servers: 5 | - "localhost:9092" 6 | encryption: 7 | type: "SSL" 8 | certificate: "CERTIFICATE_DATA" 9 | topics: 10 | - name: "quickstart-events" 11 | group_id: "groupdID1" 12 | sink: 13 | stdout: -------------------------------------------------------------------------------- /data-prepper-plugins/kinesis-source/src/test/resources/pipeline_with_acks_enabled.yaml: -------------------------------------------------------------------------------- 1 | source: 2 | kinesis: 3 | streams: 4 | - stream_name: "stream-1" 5 | - stream_name: "stream-2" 6 | - stream_name: "stream-3" 7 | codec: 8 | ndjson: 9 | aws: 10 | sts_role_arn: "arn:aws:iam::123456789012:role/OSI-PipelineRole" 11 | region: "us-east-1" 12 | acknowledgments: true -------------------------------------------------------------------------------- /data-prepper-plugins/translate-processor/src/test/resources/configs/translate_dynamic_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": [ 3 | { 4 | "source": "user.${type}.id", 5 | "targets": [ 6 | { 7 | "target": "targetField", 8 | "map": { 9 | "admin": "mappedValue1", 10 | "user": "mappedValue2" 11 | } 12 | } 13 | ] 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /data-prepper-test/test-data-prepper-version/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | 10 | dependencies { 11 | implementation project(':data-prepper-api') 12 | } 13 | -------------------------------------------------------------------------------- /examples/log-ingestion/log_pipeline.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline: 2 | source: 3 | http: 4 | ssl: false 5 | processor: 6 | - grok: 7 | match: 8 | log: [ "%{COMMONAPACHELOG}" ] 9 | sink: 10 | - opensearch: 11 | hosts: [ "https://opensearch:9200" ] 12 | insecure: true 13 | username: admin 14 | password: Developer@123 15 | index: apache_logs 16 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/acknowledgements/one-pipeline-ack-expiry-test.yaml: -------------------------------------------------------------------------------- 1 | pipeline-expiry-test: 2 | delay: 2 3 | source: 4 | in_memory: 5 | testing_key: PipelinesWithAcksIT 6 | acknowledgments: true 7 | sink: 8 | - stdout: 9 | - in_memory: 10 | testing_key: PipelinesWithAcksIT 11 | acknowledgments: false 12 | 13 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/model/MetricRegistryType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.core.parser.model; 7 | 8 | public enum MetricRegistryType { 9 | // TODO: capitalize enum values 10 | Prometheus, 11 | CloudWatch, 12 | EmbeddedMetricsFormat; 13 | } -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin-api/README.md: -------------------------------------------------------------------------------- 1 | # AWS Plugin API 2 | 3 | This provides an API for plugins which wish to use the AWS Plugin for Data Prepper. 4 | 5 | ## Using 6 | 7 | Add the following project: 8 | 9 | ``` 10 | implementation project(':data-prepper-plugins:aws-plugin-api') 11 | ``` 12 | 13 | This project does not yet deploy to Maven, so you cannot use it for external plugin development at the time. 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/kafka-pipeline-no-auth-ssl-none.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline : 2 | source: 3 | kafka: 4 | bootstrap_servers: 5 | - "localhost:9092" 6 | encryption: 7 | type: "NONE" 8 | certificate: "CERTIFICATE_DATA" 9 | topics: 10 | - name: "quickstart-events" 11 | group_id: "groupdID1" 12 | sink: 13 | stdout: -------------------------------------------------------------------------------- /data-prepper-plugins/translate-processor/src/test/resources/configs/translate_nested_path.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": [ 3 | { 4 | "source": "collection/sourceField", 5 | "targets": [ 6 | { 7 | "target": "targetField", 8 | "map": { 9 | "key1": "mappedValue1", 10 | "key2": "mappedValue2" 11 | } 12 | } 13 | ] 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | OpenSearch (https://opensearch.org/) 2 | Copyright OpenSearch Contributors 3 | 4 | This product includes software developed by 5 | Elasticsearch (http://www.elastic.co). 6 | Copyright 2009-2018 Elasticsearch 7 | 8 | This product includes software developed by The Apache Software 9 | Foundation (http://www.apache.org/). 10 | 11 | This product includes software developed by 12 | Joda.org (http://www.joda.org/). 13 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/http.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: http 2 | mappedAttributeNames: 3 | port: port 4 | max_pending_requests: max_pending_requests 5 | threads: thread_count 6 | ssl: ssl 7 | ssl_key: ssl_key_file 8 | ssl_certificate: ssl_certificate_file 9 | additionalAttributes: 10 | max_connection_count: 500 11 | request_timeout: 10_000 -------------------------------------------------------------------------------- /data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/encoding/DecoderEngineFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.processor.decompress.encoding; 7 | 8 | public interface DecoderEngineFactory { 9 | DecoderEngine getDecoderEngine(); 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/bulk/BulkApiWrapper.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.sink.opensearch.bulk; 2 | 3 | import org.opensearch.client.opensearch.core.BulkRequest; 4 | import org.opensearch.client.opensearch.core.BulkResponse; 5 | 6 | public interface BulkApiWrapper { 7 | BulkResponse bulk(BulkRequest request) throws Exception; 8 | } 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/s3/FileReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.opensearch.s3; 7 | 8 | import java.io.InputStream; 9 | 10 | public interface FileReader { 11 | InputStream readFile(final String fineName); 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/trace-peer-forwarder-processor/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | plugins { 7 | id 'java' 8 | } 9 | 10 | dependencies { 11 | implementation project(':data-prepper-api') 12 | implementation 'com.fasterxml.jackson.core:jackson-core' 13 | implementation 'com.fasterxml.jackson.core:jackson-databind' 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/org/opensearch/dataprepper/core/pipeline/PipelineObserver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.core.pipeline; 7 | 8 | /** 9 | * Allows an observer to observe major changes in pipelines. 10 | */ 11 | public interface PipelineObserver { 12 | void shutdown(Pipeline pipeline); 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/exception/StrictResponseModeNotRespectedException.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.lambda.processor.exception; 2 | 3 | public class StrictResponseModeNotRespectedException extends RuntimeException { 4 | public StrictResponseModeNotRespectedException(final String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/multiple_match_with_break_on_match_false.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{COMMONAPACHELOG}"] 8 | extra_field: ["%{GREEDYDATA} %{IPORHOST:host}"] 9 | break_on_match: false 10 | 11 | sink: 12 | - unused: 13 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/types/ByteCountParseException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.types; 7 | 8 | public class ByteCountParseException extends RuntimeException { 9 | public ByteCountParseException(final String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/resources/testjson/log.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId": "1234", 3 | "spanId": "4321", 4 | "flags": 1, 5 | "time": "2022-01-02T00:00:00Z", 6 | "severityNumber": 2, 7 | "droppedAttributesCount": 4, 8 | "serviceName": "service", 9 | "body": { 10 | "log": "message" 11 | }, 12 | "observedTimestamp": "2022-01-01T00:00:00Z", 13 | "schemaUrl": "schema", 14 | "key1": %s, 15 | "key2": "%s" 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/resources/META-INF/services/org.opensearch.dataprepper.model.configuration.VersionProvider: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | # The OpenSearch Contributors require contributions made to 6 | # this file be licensed under the Apache-2.0 license or a 7 | # compatible open source license. 8 | # 9 | 10 | org.opensearch.dataprepper.core.CoreVersionProvider -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension2.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | pipeline_extensions: 3 | test_extension_2: 4 | test_attribute: test_string 5 | test-pipeline-2: 6 | source: 7 | pipeline: 8 | name: "test-pipeline-1" 9 | sink: 10 | - pipeline: 11 | name: "test-pipeline-3" 12 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/templates/testSource/documentdb-subpipelines-template.yaml: -------------------------------------------------------------------------------- 1 | "<>-transformed": 2 | source: "<<$.<>.source>>" 3 | processor: "<<$.<>.processor>>" 4 | sink: 5 | - s3: 6 | aws: "<<$.<>.sink[?(@.opensearch)].opensearch.aws>>" 7 | bucket: "<<$.<>.source.documentdb.collections[0].s3_bucket>>" -------------------------------------------------------------------------------- /data-prepper-plugins/s3-sink/src/test/resources/simplelogger.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | org.slf4j.simpleLogger.showDateTime=true 7 | org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd' 'HH:mm:ss.SSS 8 | org.slf4j.simpleLogger.log.org.opensearch.dataprepper.plugins.sink.s3=debug 9 | org.slf4j.simpleLogger.log.org.opensearch.dataprepper.plugins.codec.parquet=debug -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/source/s3/S3ScanEnvironmentVariables.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.model.source.s3; 2 | 3 | import org.opensearch.dataprepper.model.annotations.SkipTestCoverageGenerated; 4 | 5 | @SkipTestCoverageGenerated 6 | public class S3ScanEnvironmentVariables { 7 | 8 | public static final String STOP_S3_SCAN_PROCESSING_PROPERTY = "STOP_S3_SCAN_PROCESSING"; 9 | } 10 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | appender.console.type = Console 7 | appender.console.name = STDOUT 8 | appender.console.layout.type = PatternLayout 9 | appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n 10 | 11 | 12 | rootLogger.level = debug 13 | rootLogger.appenderRef.stdout.ref = STDOUT 14 | 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/kafka-pipeline-bootstrap-servers-glue-default.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline : 2 | source: 3 | kafka: 4 | bootstrap_servers: 5 | - "localhost:9092" 6 | encryption: 7 | type: "SSL" 8 | schema: 9 | type: aws_glue 10 | topics: 11 | - name: "quickstart-events" 12 | group_id: "groupdID1" 13 | sink: 14 | stdout: -------------------------------------------------------------------------------- /examples/certificates/openssl.conf: -------------------------------------------------------------------------------- 1 | [req] 2 | distinguished_name = req_distinguished_name 3 | prompt = no 4 | x509_extensions = v3_ext 5 | 6 | [req_distinguished_name] 7 | L = test 8 | O = Example Com Inc. 9 | OU = Example Com Inc. Root CA 10 | CN = data-prepper.example.com 11 | 12 | [v3_ext] 13 | subjectAltName = @alt_names 14 | 15 | [alt_names] 16 | DNS.1 = *.data-prepper.example.com 17 | DNS.2 = localhost 18 | IP.1 = 127.0.0.1 19 | -------------------------------------------------------------------------------- /release/release-notes/odfe-data-prepper.release-notes-1.0.0.md: -------------------------------------------------------------------------------- 1 | # 2021-05-11 Version 1.0.0 2 | 3 | ## Highlights 4 | * Now builds using [version 1.0+](https://github.com/open-telemetry/opentelemetry-specification/pull/1372) of the OpenTelemetry tracing specification 5 | * Additional TraceGroup fields are emitted for enhanced searching and filtering 6 | 7 | ## Compatibility 8 | * No compatibility issues with previous versions of Data Prepper. 9 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-amazon-opensearch.conf: -------------------------------------------------------------------------------- 1 | input { 2 | http { 3 | } 4 | } 5 | filter { 6 | grok { 7 | match => {"log" => "%{COMBINEDAPACHELOG}"} 8 | } 9 | } 10 | output { 11 | amazon_es { 12 | hosts => ["fakedomain.us-east-1.es.amazonaws.com"] 13 | region => "us-east-1" 14 | index => "my-index" 15 | } 16 | } -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-null-key-name.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | batch: 5 | key_name: 6 | threshold: 7 | event_count: 100 8 | maximum_size: 1mb 9 | event_collect_timeout: 335 10 | aws: 11 | region: "us-east-1" 12 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 13 | 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/configuration/AwsCredentialsConfig.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.kafka.configuration; 2 | 3 | import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; 4 | 5 | public interface AwsCredentialsConfig { 6 | String getRegion(); 7 | String getStsRoleArn(); 8 | 9 | AwsCredentialsOptions toCredentialsOptions(); 10 | } 11 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | opensearch: 4 | image: opensearchproject/opensearch:1.1.0 5 | ports: 6 | - "9200:9200" 7 | - "9300:9300" 8 | environment: 9 | - discovery.type=single-node 10 | analytics-service: 11 | build: 12 | context: . 13 | dockerfile: Dockerfile 14 | ports: 15 | - "8087:8087" 16 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/org/opensearch/dataprepper/core/pipeline/ProcessorProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.core.pipeline; 7 | 8 | import org.opensearch.dataprepper.model.processor.Processor; 9 | 10 | import java.util.List; 11 | 12 | public interface ProcessorProvider { 13 | List getProcessors(); 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/single_match_multiple_pattern_with_break_on_match_false.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: 8 | - "%{COMMONAPACHELOG}" 9 | - "%{IPORHOST:custom_client_field}" 10 | break_on_match: false 11 | 12 | sink: 13 | - unused: 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/log-generator-source/src/main/java/org/opensearch/dataprepper/plugins/source/loggenerator/LogTypeGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.source.loggenerator; 7 | 8 | import org.opensearch.dataprepper.model.event.Event; 9 | 10 | public interface LogTypeGenerator { 11 | Event generateEvent(); 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/StringProcessorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.processor.mutatestring; 7 | 8 | import java.util.List; 9 | 10 | public interface StringProcessorConfig { 11 | List getIterativeConfig(); 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/translate-processor/src/test/resources/configs/translate_static_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": [ 3 | { 4 | "source": "sourceField", 5 | "targets": [ 6 | { 7 | "target": "targetField", 8 | "map": { 9 | "key1": "mappedValue1", 10 | "key2": "mappedValue2", 11 | "key3": "mappedValue3" 12 | } 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /examples/dev/data-prepper-emf-monitoring/pipelines-raw-trace-stdout.yaml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | source: 3 | otel_trace_source: 4 | ssl: false 5 | unframed_requests: true 6 | sink: 7 | - pipeline: 8 | name: "raw-pipeline" 9 | raw-pipeline: 10 | source: 11 | pipeline: 12 | name: "entry-pipeline" 13 | processor: 14 | - otel_traces: 15 | # trace_flush_interval: 6 16 | sink: 17 | - stdout: 18 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/requirements.txt: -------------------------------------------------------------------------------- 1 | dash==2.15.0 2 | mysql-connector==2.2.9 3 | opentelemetry-exporter-otlp==1.25.0 4 | opentelemetry-instrumentation-flask==0.46b0 5 | opentelemetry-instrumentation-mysql==0.46b0 6 | opentelemetry-instrumentation-requests==0.46b0 7 | opentelemetry-sdk==1.25.0 8 | protobuf==4.25.8 9 | requests==2.32.4 10 | setuptools==78.1.1 11 | urllib3==2.6.0 12 | werkzeug==3.0.6 13 | zipp==3.19.1 14 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/types/ByteCountInvalidInputException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.types; 7 | 8 | public class ByteCountInvalidInputException extends RuntimeException { 9 | public ByteCountInvalidInputException(final String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/openssl.conf: -------------------------------------------------------------------------------- 1 | [req] 2 | distinguished_name = req_distinguished_name 3 | prompt = no 4 | x509_extensions = v3_ext 5 | 6 | [req_distinguished_name] 7 | L = test 8 | O = Example Com Inc. 9 | OU = Example Com Inc. Root CA 10 | CN = data-prepper.example.com 11 | 12 | [v3_ext] 13 | subjectAltName = @alt_names 14 | 15 | [alt_names] 16 | DNS.1 = data-prepper.example.com 17 | DNS.2 = localhost 18 | IP.1 = 127.0.0.1 19 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/multi-pipelines-distributed-pipeline-configurations/pipeline_configuration_with_test_extension2.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | pipeline_configurations: 3 | test_extension_2: 4 | test_attribute: test_string 5 | test-pipeline-2: 6 | source: 7 | pipeline: 8 | name: "test-pipeline-1" 9 | sink: 10 | - pipeline: 11 | name: "test-pipeline-3" 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-success-config.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | batch: 5 | key_name: "osi_key" 6 | threshold: 7 | event_count: 100 8 | maximum_size: 1mb 9 | event_collect_timeout: 335 10 | aws: 11 | region: "us-east-1" 12 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 13 | 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/certificate/CertificateProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.certificate; 7 | 8 | import org.opensearch.dataprepper.plugins.certificate.model.Certificate; 9 | 10 | public interface CertificateProvider { 11 | Certificate getCertificate(); 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/translate-processor/src/test/resources/configs/translate_range_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": [ 3 | { 4 | "source": "sourceField", 5 | "targets": [ 6 | { 7 | "target": "targetField", 8 | "map": { 9 | "1-10": "mappedValue1", 10 | "11-20": "mappedValue2", 11 | "21-30": "mappedValue3" 12 | } 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/elasticsearch.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: opensearch 2 | attributesMapperClass: org.opensearch.dataprepper.logstash.mapping.OpenSearchPluginAttributesMapper 3 | mappedAttributeNames: 4 | hosts: hosts 5 | cacert: cert 6 | user: username 7 | password: password 8 | index: index 9 | proxy: proxy 10 | defaultSettings: 11 | index: "logstash-%{uuuu.MM.dd}" -------------------------------------------------------------------------------- /data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/exceptions/RequestCancelledException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.exceptions; 7 | 8 | public class RequestCancelledException extends RuntimeException { 9 | public RequestCancelledException(final String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-plugin-api/src/main/java/org/opensearch/dataprepper/aws/api/AwsCredentialsConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright OpenSearch Contributors 4 | * * SPDX-License-Identifier: Apache-2.0 5 | * 6 | */ 7 | 8 | package org.opensearch.dataprepper.aws.api; 9 | 10 | public interface AwsCredentialsConfig { 11 | String getRegion(); 12 | String getStsRoleArn(); 13 | 14 | AwsCredentialsOptions toCredentialsOptions(); 15 | } 16 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/single_match_type_conversion.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: 8 | - "\"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response:int} (?:%{NUMBER:bytes:float}|-)" 9 | 10 | sink: 11 | - unused: 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManager.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.sink.opensearch.index; 2 | 3 | import java.io.IOException; 4 | 5 | public interface IndexManager{ 6 | 7 | void setupIndex() throws IOException; 8 | String getIndexName(final String indexAlias) throws IOException; 9 | Boolean isIndexAlias(final String indexAlias) throws IOException; 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/translate-processor/src/jmh/resources/jmh/static_key_test_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "testRecords": [ 3 | {"sourceField": "key0"}, 4 | {"sourceField": "key1"}, 5 | {"sourceField": "key2"}, 6 | {"sourceField": "key3"}, 7 | {"sourceField": "key4"}, 8 | {"sourceField": "key0"}, 9 | {"sourceField": "key1"}, 10 | {"sourceField": "key2"}, 11 | {"sourceField": "key3"}, 12 | {"sourceField": "key4"} 13 | ] 14 | } -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/opentelemetry-collector/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:55680 6 | 7 | exporters: 8 | otlp/2: 9 | endpoint: data-prepper:21890 10 | insecure: false 11 | insecure_skip_verify: true 12 | logging: 13 | 14 | service: 15 | pipelines: 16 | traces: 17 | receivers: [otlp] 18 | exporters: [logging, otlp/2] 19 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/exceptions/EventKeyNotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.event.exceptions; 7 | 8 | public class EventKeyNotFoundException extends RuntimeException { 9 | public EventKeyNotFoundException(final String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/multi-worker.yaml: -------------------------------------------------------------------------------- 1 | minimal-pipeline: 2 | delay: 10 3 | workers: 4 4 | source: 5 | in_memory: 6 | testing_key: MultiWorkerPipelineIT 7 | 8 | buffer: 9 | bounded_blocking: 10 | buffer_size: 1000 11 | batch_size: 10 12 | 13 | processor: 14 | - thread_info: 15 | 16 | sink: 17 | - in_memory: 18 | testing_key: MultiWorkerPipelineIT -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/missing_name_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | someProperty: "someValue" 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - pipeline: 10 | test-pipeline-2: 11 | source: 12 | pipeline: 13 | name: "test-pipeline-1" 14 | sink: 15 | - file: 16 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/amazon_es.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: opensearch 2 | attributesMapperClass: org.opensearch.dataprepper.logstash.mapping.OpenSearchPluginAttributesMapper 3 | mappedAttributeNames: 4 | hosts: hosts 5 | index: index 6 | region: aws_region 7 | additionalAttributes: 8 | aws_sigv4: true 9 | insecure: false 10 | defaultSettings: 11 | index: "logstash-%{yyyy.MM.dd}" -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: opensearch 2 | attributesMapperClass: org.opensearch.dataprepper.logstash.mapping.OpenSearchPluginAttributesMapper 3 | mappedAttributeNames: 4 | hosts: hosts 5 | user: username 6 | password: password 7 | index: index 8 | ssl_certificate_verification: "!insecure" 9 | defaultSettings: 10 | index: "logstash-%{uuuu.MM.dd}" -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-aggregate-mode-config.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: false 3 | tags_on_failure: [ "lambda_failure" ] 4 | batch: 5 | key_name: "osi_key" 6 | threshold: 7 | event_count: 100 8 | maximum_size: 1mb 9 | event_collect_timeout: 335 10 | aws: 11 | region: "us-east-1" 12 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 13 | 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/accumulator/InMemoryBufferFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.accumulator; 7 | 8 | public class InMemoryBufferFactory implements BufferFactory { 9 | @Override 10 | public Buffer getBuffer() { 11 | return new InMemoryBuffer(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/extension/databasedownload/DBSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.geoip.extension.databasedownload; 7 | 8 | public interface DBSource { 9 | String MAXMIND_DATABASE_EXTENSION = ".mmdb"; 10 | void initiateDownload() throws Exception; 11 | } 12 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/pattern_definitions.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{GREEDYDATA:greedy_data} %{CUSTOMPHONENUMBERPATTERN:my_number}"] 8 | pattern_definitions: 9 | CUSTOMPHONENUMBERPATTERN: "\\d\\d\\d-\\d\\d\\d-\\d\\d\\d" 10 | 11 | sink: 12 | - unused: 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/sample-pipelines-1.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline: 2 | source: 3 | kafka: 4 | bootstrap_servers: 5 | - 127.0.0.1:9093 6 | topics: 7 | - name: my-topic-2 8 | group_id: my-test-group 9 | - name: my-topic-1 10 | group_id: my-test-group 11 | schema: 12 | registry_url: http://localhost:8081/ 13 | version: 1 14 | sink: 15 | - stdout: 16 | -------------------------------------------------------------------------------- /data-prepper-plugins/newline-codecs/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | dependencies { 6 | implementation project(':data-prepper-api') 7 | implementation 'com.fasterxml.jackson.core:jackson-annotations' 8 | implementation libs.parquet.common 9 | testImplementation project(':data-prepper-plugins:common') 10 | testImplementation project(':data-prepper-test:test-event') 11 | } 12 | 13 | test { 14 | useJUnitPlatform() 15 | } -------------------------------------------------------------------------------- /data-prepper-plugins/s3-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/s3/codec/BufferedCodec.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.sink.s3.codec; 2 | 3 | import org.opensearch.dataprepper.model.codec.OutputCodec; 4 | 5 | import java.util.Optional; 6 | 7 | /** 8 | * Represents a {@link OutputCodec} which supplies its own buffer. 9 | */ 10 | public interface BufferedCodec extends OutputCodec { 11 | Optional getSize(); 12 | } 13 | -------------------------------------------------------------------------------- /performance-test/src/gatling/java/org/opensearch/dataprepper/test/performance/tools/PathTarget.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.test.performance.tools; 2 | 3 | public class PathTarget { 4 | private static final String PATH_PROPERTY_NAME = "path"; 5 | private static final String DEFAULT_PATH = "/log/ingest"; 6 | 7 | public static String getPath() { 8 | return System.getProperty(PATH_PROPERTY_NAME, DEFAULT_PATH); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/basic-processor-pipeline.yaml: -------------------------------------------------------------------------------- 1 | basic-processor-pipeline: 2 | workers: 4 3 | source: 4 | in_memory: 5 | testing_key: ProcessorValidationIT 6 | buffer: 7 | bounded_blocking: 8 | buffer_size: 1000 9 | batch_size: 10 10 | processor: 11 | - basic_events_tracking_test: 12 | sink: 13 | - in_memory: 14 | testing_key: ProcessorValidationIT 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/exceptions/BadRequestException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.exceptions; 7 | 8 | public class BadRequestException extends RuntimeException { 9 | public BadRequestException(final String message, final Throwable cause) { 10 | super(message, cause); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-large-payload.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | tags_on_failure: [ "lambda_failure" ] 3 | batch: 4 | key_name: "osi_key" 5 | threshold: 6 | event_count: 100 7 | maximum_size: 5mb 8 | event_collect_timeout: 100 9 | client: 10 | max_retries: 50 11 | max_concurrency: 5 12 | aws: 13 | region: "us-east-1" 14 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" -------------------------------------------------------------------------------- /data-prepper-plugins/decompress-processor/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | dependencies { 7 | implementation 'commons-io:commons-io:2.15.1' 8 | implementation project(':data-prepper-api') 9 | implementation project(':data-prepper-plugins:common') 10 | implementation 'com.fasterxml.jackson.core:jackson-databind' 11 | implementation 'io.micrometer:micrometer-core' 12 | } -------------------------------------------------------------------------------- /data-prepper-plugins/in-memory-source-coordination-store/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | dependencies { 6 | implementation project(':data-prepper-api') 7 | } 8 | 9 | test { 10 | useJUnitPlatform() 11 | } 12 | 13 | jacocoTestCoverageVerification { 14 | dependsOn jacocoTestReport 15 | violationRules { 16 | rule { 17 | limit { 18 | minimum = 1.0 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /data-prepper-test/test-data-prepper-version/src/main/resources/META-INF/services/org.opensearch.dataprepper.model.configuration.VersionProvider: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright OpenSearch Contributors 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | # The OpenSearch Contributors require contributions made to 6 | # this file be licensed under the Apache-2.0 license or a 7 | # compatible open source license. 8 | # 9 | 10 | org.opensearch.dataprepper.test.version.TestVersionProvider -------------------------------------------------------------------------------- /e2e-test/peerforwarder/src/integrationTest/resources/data_prepper_static.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | peer_forwarder: 3 | port: 4994 4 | ssl: true 5 | ssl_certificate_file: "/usr/share/data-prepper/config/default_certificate.pem" 6 | ssl_key_file: "/usr/share/data-prepper/config/default_private_key.pem" 7 | discovery_mode: static 8 | static_endpoints: ["node0.data-prepper.example.com", "node1.data-prepper.example.com"] 9 | authentication: 10 | mutual_tls: 11 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/opentelemetry-collector/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:55680 6 | 7 | exporters: 8 | otlp/2: 9 | endpoint: data-prepper:21890 10 | tls: 11 | insecure: true 12 | insecure_skip_verify: true 13 | logging: 14 | 15 | service: 16 | pipelines: 17 | traces: 18 | receivers: [otlp] 19 | exporters: [logging, otlp/2] 20 | -------------------------------------------------------------------------------- /.github/workflows/delete_backport_branch.yml: -------------------------------------------------------------------------------- 1 | name: Delete merged branch of the backport PRs 2 | on: 3 | pull_request: 4 | types: 5 | - closed 6 | 7 | jobs: 8 | delete-branch: 9 | runs-on: ubuntu-latest 10 | if: startsWith(github.event.pull_request.head.ref,'backport/') 11 | steps: 12 | - name: Delete merged branch 13 | uses: SvanBoxel/delete-merged-branch@main 14 | env: 15 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Gradle directories 2 | build 3 | .gradle 4 | gradle/tools 5 | 6 | # Ignore config file generated by test 7 | data-prepper-main/src/test/resources/logstash-filter.yaml 8 | data-prepper-main/src/test/resources/logstash-conf/logstash-filter.yaml 9 | 10 | # Python virtual environments 11 | .venv 12 | 13 | # output folder created when we run test cases 14 | **/out/ 15 | 16 | # Development tools 17 | .DS_Store 18 | .idea 19 | *.iml 20 | .kiro 21 | .vscode 22 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/document/Document.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.document; 7 | 8 | import org.opensearch.dataprepper.model.event.Event; 9 | 10 | /** 11 | * A document event in Data Prepper represents a document type. 12 | * @since 2.1 13 | */ 14 | public interface Document extends Event { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/connected/single-connection.yaml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | delay: 5 3 | source: 4 | in_memory: 5 | testing_key: Connected_SingleIT 6 | sink: 7 | - pipeline: 8 | name: exit-pipeline 9 | 10 | exit-pipeline: 11 | delay: 5 12 | source: 13 | pipeline: 14 | name: entry-pipeline 15 | sink: 16 | - in_memory: 17 | testing_key: Connected_SingleIT 18 | -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/http-to-opensearch.expected.yaml: -------------------------------------------------------------------------------- 1 | logstash-converted-pipeline: 2 | source: 3 | http: 4 | max_connection_count: 500 5 | request_timeout: 10000 6 | processor: [] 7 | sink: 8 | - opensearch: 9 | hosts: 10 | - "https://localhost:19000" 11 | username: "myuser" 12 | password: "mypassword" 13 | index: "simple-pipeline" 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/exceptions/BufferWriteException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.exceptions; 7 | 8 | public class BufferWriteException extends RuntimeException { 9 | public BufferWriteException(final String message, final Throwable cause) { 10 | super(message, cause); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkBufferEntryProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.common.sink; 7 | 8 | import org.opensearch.dataprepper.model.event.Event; 9 | 10 | public interface SinkBufferEntryProvider { 11 | SinkBufferEntry getSinkBufferEntry(final Event event) throws Exception; 12 | } 13 | 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_no_captures_and_tags.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: ["%{GREEDYDATA:greedy_data} (?\\d\\d\\d-\\d\\d\\d-\\d\\d\\d)"] 8 | tags_on_match_failure: 9 | - some_test_tag_1 10 | - some_test_tag_2 11 | 12 | sink: 13 | - unused: 14 | -------------------------------------------------------------------------------- /examples/log-ingestion-k8s/my-app-03-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: my-nginx 5 | namespace: my-app 6 | spec: 7 | selector: 8 | matchLabels: 9 | run: my-nginx 10 | replicas: 2 11 | template: 12 | metadata: 13 | labels: 14 | run: my-nginx 15 | spec: 16 | containers: 17 | - name: my-nginx 18 | image: nginx 19 | ports: 20 | - containerPort: 80 -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/kafka-pipeline-bootstrap-servers-sasl-iam-default.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline : 2 | source: 3 | kafka: 4 | bootstrap_servers: 5 | - "localhost:9092" 6 | encryption: 7 | type: "SSL" 8 | authentication: 9 | sasl: 10 | aws_msk_iam: default 11 | topics: 12 | - name: "quickstart-events" 13 | group_id: "groupdID1" 14 | sink: 15 | stdout: -------------------------------------------------------------------------------- /data-prepper-test/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | * 9 | */ 10 | 11 | subprojects { 12 | apply plugin: 'java-library' 13 | apply plugin: 'data-prepper.publish' 14 | 15 | group = 'org.opensearch.dataprepper.test' 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-test/test-common/src/test/java/org/opensearch/dataprepper/test/helper/ReflectivelySetFieldTestHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.test.helper; 7 | 8 | public class ReflectivelySetFieldTestHelper { 9 | private String internalField = "first value"; 10 | 11 | public String getInternalField() { 12 | return internalField; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginArgumentsContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugin; 7 | 8 | interface PluginArgumentsContext { 9 | Object[] createArguments(final Class[] parameterTypes, final Object ... args); 10 | 11 | default Object getArgument(int index) { 12 | return null; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/stream/ShardNotTrackedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.source.dynamodb.stream; 7 | 8 | public class ShardNotTrackedException extends RuntimeException { 9 | public ShardNotTrackedException(String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/valid-data-prepper-config-with-kafka-cluster-extension.yaml: -------------------------------------------------------------------------------- 1 | extensions: 2 | kafka_cluster_config: 3 | bootstrap_servers: 4 | - localhost:9092 5 | encryption: 6 | type: none 7 | aws: 8 | region: "us-east-1" 9 | msk: 10 | # Provide the MSK ARN. 11 | broker_connection_type: public 12 | arn: "test-arn" 13 | authentication: 14 | sasl: 15 | aws_msk_iam: default -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/s3/InvalidS3URIException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.opensearch.s3; 7 | 8 | public class InvalidS3URIException extends RuntimeException { 9 | 10 | public InvalidS3URIException(final String errorMessage) { 11 | super(errorMessage); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3SelectResponseHandlerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.source.s3; 7 | 8 | public class S3SelectResponseHandlerFactory { 9 | 10 | public S3SelectResponseHandler provideS3SelectResponseHandler() { 11 | return new S3SelectResponseHandler(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-test/test-event/src/main/java/org/opensearch/dataprepper/event/EventFactoryApplicationContextMarker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.event; 7 | 8 | /** 9 | * Intended for Data Prepper core tests. Using this, tests that need to scan 10 | * the Application Context can scan this class. 11 | */ 12 | public interface EventFactoryApplicationContextMarker { 13 | } 14 | -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/resources/data-prepper-wait-for-opensearch-and-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Copyright OpenSearch Contributors 5 | # SPDX-License-Identifier: Apache-2.0 6 | # 7 | 8 | until [[ $(curl --write-out %{http_code} --output /dev/null --silent --head --fail https://node-0.example.com:9200 -u admin:myStrongPassword123! --insecure) == 200 ]]; do 9 | echo "Waiting for OpenSearch to be ready" 10 | sleep 1 11 | done 12 | 13 | ./bin/data-prepper 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-success-config-with-keys.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | keys: [ "key1" ] 5 | batch: 6 | key_name: "osi_key" 7 | threshold: 8 | event_count: 100 9 | maximum_size: 1mb 10 | event_collect_timeout: 335 11 | aws: 12 | region: "us-east-1" 13 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 14 | 15 | 16 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkFlushableBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.common.sink; 7 | 8 | import org.opensearch.dataprepper.model.event.Event; 9 | 10 | import java.util.List; 11 | 12 | public interface SinkFlushableBuffer { 13 | SinkFlushResult flush(); 14 | 15 | List getEvents(); 16 | } 17 | 18 | -------------------------------------------------------------------------------- /data-prepper-plugins/geoip-processor/src/test/resources/geoip_service_config.yaml: -------------------------------------------------------------------------------- 1 | extensions: 2 | geoip_service: 3 | maxmind: 4 | databases: 5 | city: "city_path" 6 | country: "country_path" 7 | database_refresh_interval: "P10D" 8 | cache_count: 2048 9 | insecure: true 10 | database_destination: "/tst/resources" 11 | aws: 12 | region: "us-east-1" 13 | sts_role_arn: "arn:aws:iam::123456789:role/data-prepper-execution-role" -------------------------------------------------------------------------------- /data-prepper-plugins/translate-processor/src/jmh/resources/jmh/static_key_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": [ 3 | { 4 | "source": "sourceField", 5 | "targets": [ 6 | { 7 | "target": "targetField", 8 | "map": { 9 | "key0": "value0", 10 | "key1": "value1", 11 | "key2": "value2", 12 | "key3": "value3", 13 | "key4": "value4" 14 | } 15 | } 16 | ] 17 | } 18 | ] 19 | } -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.event; 7 | 8 | public interface EventBuilder extends BaseEventBuilder { 9 | /** 10 | * Returns a newly created {@link JacksonEvent}. 11 | * 12 | * @return an event 13 | * @since 2.2 14 | */ 15 | Event build(); 16 | } 17 | 18 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-when-condition-config.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | lambda_when: "some-condition" 5 | batch: 6 | key_name: "osi_key" 7 | threshold: 8 | event_count: 100 9 | maximum_size: 1mb 10 | event_collect_timeout: 335 11 | aws: 12 | region: "us-east-1" 13 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 14 | 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkBufferEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.common.sink; 7 | 8 | import org.opensearch.dataprepper.model.event.Event; 9 | 10 | public interface SinkBufferEntry { 11 | public long getEstimatedSize(); 12 | public Event getEvent(); 13 | public boolean exceedsMaxEventSizeThreshold(); 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/exceptions/DecodingException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.processor.decompress.exceptions; 7 | 8 | public class DecodingException extends RuntimeException { 9 | public DecodingException(final String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/exception/InvalidIPAddressException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.geoip.exception; 7 | 8 | public class InvalidIPAddressException extends EnrichFailedException { 9 | public InvalidIPAddressException(final String exceptionMsg) { 10 | super(exceptionMsg); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/s3-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/s3/BufferScenario.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.s3; 7 | 8 | import org.opensearch.dataprepper.plugins.sink.s3.accumulator.BufferTypeOptions; 9 | 10 | public interface BufferScenario { 11 | BufferTypeOptions getBufferType(); 12 | int getMaximumNumberOfEvents(); 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-test/test-event/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | dependencies { 7 | implementation project(':data-prepper-api') 8 | implementation project(':data-prepper-event') 9 | implementation(libs.spring.context) { 10 | exclude group: 'commons-logging', module: 'commons-logging' 11 | } 12 | } 13 | 14 | jacocoTestCoverageVerification { 15 | violationRules.setFailOnViolation(false) 16 | } -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/java/com/example/restservice/RestServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.example.restservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class RestServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(RestServiceApplication.class, args); 11 | } 12 | } -------------------------------------------------------------------------------- /data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExtensionClassProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugin; 7 | 8 | import org.opensearch.dataprepper.model.plugin.ExtensionPlugin; 9 | 10 | import java.util.Collection; 11 | 12 | interface ExtensionClassProvider { 13 | Collection> loadExtensionPluginClasses(); 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-payload-limit.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | batch: 5 | key_name: "osi_key" 6 | threshold: 7 | event_count: 100 8 | maximum_size: 100b 9 | event_collect_timeout: 335 10 | client: 11 | max_retries: 50 12 | max_concurrency: 5 13 | aws: 14 | region: "us-east-1" 15 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-with-retries.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | batch: 5 | key_name: "osi_key" 6 | threshold: 7 | event_count: 100 8 | maximum_size: 1mb 9 | event_collect_timeout: 335 10 | client: 11 | max_retries: 50 12 | max_concurrency: 5 13 | aws: 14 | region: "us-east-1" 15 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" -------------------------------------------------------------------------------- /data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/buffer/InMemoryBufferFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.buffer; 7 | 8 | public class InMemoryBufferFactory implements BufferFactory{ 9 | @Override 10 | public Buffer getBuffer() { 11 | return new InMemoryBuffer(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/cloudwatch-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/CloudwatchSourceProgressState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.source; 7 | 8 | import com.fasterxml.jackson.annotation.JsonCreator; 9 | 10 | public class CloudwatchSourceProgressState { 11 | 12 | 13 | @JsonCreator 14 | CloudwatchSourceProgressState() { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkFlushResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.common.sink; 7 | 8 | import org.opensearch.dataprepper.model.event.Event; 9 | 10 | import java.util.List; 11 | 12 | public interface SinkFlushResult { 13 | List getEvents(); 14 | Throwable getException(); 15 | int getStatusCode(); 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/s3/S3ObjectTooLargeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.opensearch.s3; 7 | 8 | public class S3ObjectTooLargeException extends RuntimeException { 9 | 10 | public S3ObjectTooLargeException(final String errorMessage) { 11 | super(errorMessage); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/IndexNotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.source.opensearch.worker.client.exceptions; 7 | 8 | public class IndexNotFoundException extends RuntimeException { 9 | public IndexNotFoundException(final String message) { super (message); } 10 | } 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/translate-processor/src/jmh/resources/jmh/nested_path_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": [ 3 | { 4 | "source": "parent/items/sourceField", 5 | "targets": [ 6 | { 7 | "target": "targetField", 8 | "map": { 9 | "key0": "value0", 10 | "key1": "value1", 11 | "key2": "value2", 12 | "key3": "value3", 13 | "key4": "value4" 14 | } 15 | } 16 | ] 17 | } 18 | ] 19 | } -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/encryption/EncryptionEnvelope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.encryption; 7 | 8 | public interface EncryptionEnvelope { 9 | /** 10 | * The encrypted data. 11 | */ 12 | byte[] getEncryptedData(); 13 | 14 | /** 15 | * The encrypted data key. 16 | */ 17 | String getEncryptedDataKey(); 18 | } 19 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/processor-pipeline.yaml: -------------------------------------------------------------------------------- 1 | processor-pipeline: 2 | delay: 10 3 | source: 4 | in_memory: 5 | testing_key: ProcessorPipelineIT 6 | 7 | processor: 8 | - simple_test: 9 | key1: /test1 10 | value_prefix1: knownPrefix1 11 | - simple_copy_test: 12 | source: /test1 13 | target: /test1_copy 14 | 15 | sink: 16 | - in_memory: 17 | testing_key: ProcessorPipelineIT 18 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/single-thread-processor-pipeline.yaml: -------------------------------------------------------------------------------- 1 | single-thread-processor-pipeline: 2 | delay: 10 3 | workers: 4 4 | source: 5 | in_memory: 6 | testing_key: ProcessorValidationIT 7 | buffer: 8 | bounded_blocking: 9 | buffer_size: 1000 10 | batch_size: 10 11 | processor: 12 | - single_thread_events_tracking_test: 13 | sink: 14 | - in_memory: 15 | testing_key: ProcessorValidationIT 16 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/multi-pipelines-single-pipeline-extensions/pipeline_configuration_with_test_extension.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | pipeline_extensions: 3 | test_extension: 4 | test_attribute: test_string 5 | test-pipeline-1: 6 | source: 7 | file: 8 | path: "/tmp/file-source.tmp" 9 | buffer: 10 | bounded_blocking: #to check non object nodes for plugins 11 | sink: 12 | - pipeline: 13 | name: "test-pipeline-2" 14 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_peer_forwarder_config_with_mutual_tls.yml: -------------------------------------------------------------------------------- 1 | port: 21895 2 | request_timeout: 1000 3 | server_thread_count: 100 4 | max_connection_count: 100 5 | max_pending_requests: 512 6 | ssl: true 7 | ssl_certificate_file: src/test/resources/test-crt.crt 8 | ssl_key_file: src/test/resources/test-key.crt 9 | use_acm_certificate_for_ssl: false 10 | discovery_mode: static 11 | client_thread_count: 100 12 | batch_size: 100 13 | buffer_size: 100 14 | authentication: 15 | mutual_tls: -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/date.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: date 2 | attributesMapperClass: org.opensearch.dataprepper.logstash.mapping.DateLogstashPluginAttributesMapper 3 | mappedAttributeNames: 4 | match: match 5 | target: destination 6 | timezone: source_timezone 7 | locale: locale 8 | additionalAttributes: 9 | from_time_received: false 10 | destination_timezone: "UTC" 11 | nestedSyntaxAttributeNames: 12 | - target 13 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/transformer/PipelineTransformationPathProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | package org.opensearch.dataprepper.pipeline.parser.transformer; 6 | 7 | public interface PipelineTransformationPathProvider { 8 | 9 | String getTransformationTemplateDirectoryLocation(); 10 | 11 | String getTransformationRulesDirectoryLocation(); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/s3/UnsupportedFileTypeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.opensearch.s3; 7 | 8 | public class UnsupportedFileTypeException extends RuntimeException { 9 | 10 | public UnsupportedFileTypeException(final String errorMessage) { 11 | super(errorMessage); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/test/resources/management-disabled-index-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "settings": { 4 | "number_of_shards" : 1, 5 | "number_of_replicas" : 0 6 | }, 7 | "mappings": { 8 | "date_detection": false, 9 | "properties": { 10 | "name": { 11 | "ignore_above": 1024, 12 | "type": "keyword" 13 | }, 14 | "someId": { 15 | "ignore_above": 1024, 16 | "type": "keyword" 17 | } 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /examples/adot/aws-ot-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:55680 6 | 7 | exporters: 8 | otlp/2: 9 | endpoint: data-prepper:21890 10 | tls: 11 | insecure: true 12 | insecure_skip_verify: true 13 | logging: 14 | 15 | service: 16 | pipelines: 17 | traces: 18 | receivers: [otlp] 19 | exporters: [otlp/2, logging] 20 | metrics: 21 | receivers: [ otlp ] 22 | exporters: [ logging ] 23 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/multi-processor-pipeline.yaml: -------------------------------------------------------------------------------- 1 | multi-processor-pipeline: 2 | workers: 4 3 | source: 4 | in_memory: 5 | testing_key: ProcessorValidationIT 6 | buffer: 7 | bounded_blocking: 8 | buffer_size: 1000 9 | batch_size: 10 10 | processor: 11 | - single_thread_events_tracking_test: 12 | - basic_events_tracking_test: 13 | sink: 14 | - in_memory: 15 | testing_key: ProcessorValidationIT 16 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/multi-pipelines-distributed-pipeline-extensions/pipeline_configuration_with_test_extension1.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | pipeline_extensions: 3 | test_extension_1: 4 | test_attribute: test_string 5 | test-pipeline-1: 6 | source: 7 | file: 8 | path: "/tmp/file-source.tmp" 9 | buffer: 10 | bounded_blocking: #to check non object nodes for plugins 11 | sink: 12 | - pipeline: 13 | name: "test-pipeline-2" 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-sqs-common/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | dependencies { 5 | implementation project(':data-prepper-api') 6 | implementation project(':data-prepper-plugins:aws-plugin-api') 7 | implementation project(':data-prepper-plugins:buffer-common') 8 | implementation libs.armeria.core 9 | implementation 'software.amazon.awssdk:sts' 10 | implementation 'software.amazon.awssdk:sqs' 11 | implementation 'software.amazon.awssdk:arns:' 12 | 13 | } -------------------------------------------------------------------------------- /data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressionEngineFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.processor.decompress; 7 | 8 | import org.opensearch.dataprepper.model.codec.DecompressionEngine; 9 | 10 | public interface DecompressionEngineFactory { 11 | public DecompressionEngine getDecompressionEngine(); 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/exception/NoValidDatabaseFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.geoip.exception; 7 | 8 | public class NoValidDatabaseFoundException extends EngineFailureException { 9 | public NoValidDatabaseFoundException(final String exceptionMsg) { 10 | super(exceptionMsg); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/log-generator-source/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | plugins { 7 | id 'java' 8 | } 9 | 10 | dependencies { 11 | implementation project(':data-prepper-api') 12 | implementation project(':data-prepper-plugins:blocking-buffer') 13 | implementation 'com.fasterxml.jackson.core:jackson-databind' 14 | testImplementation libs.commons.lang3 15 | } 16 | 17 | test { 18 | useJUnitPlatform() 19 | } -------------------------------------------------------------------------------- /data-prepper-plugins/obfuscate-processor/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation project(path: ':data-prepper-api') 3 | implementation project(':data-prepper-plugins:common') 4 | implementation 'com.fasterxml.jackson.core:jackson-core' 5 | implementation 'com.fasterxml.jackson.core:jackson-databind' 6 | testImplementation project(':data-prepper-test:test-common') 7 | testImplementation project(':data-prepper-test:test-event') 8 | } 9 | 10 | test { 11 | useJUnitPlatform() 12 | } -------------------------------------------------------------------------------- /data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/exception/SqsRetriesExhaustedException.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.source.s3.exception; 2 | 3 | /** 4 | * This exception is thrown when SQS retries are exhausted 5 | * 6 | * @since 2.1 7 | */ 8 | public class SqsRetriesExhaustedException extends RuntimeException { 9 | 10 | public SqsRetriesExhaustedException(final String errorMessage) { 11 | super(errorMessage); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_config_with_mutual_tls_not_ssl.yml: -------------------------------------------------------------------------------- 1 | port: 21895 2 | request_timeout: 1000 3 | server_thread_count: 100 4 | max_connection_count: 100 5 | max_pending_requests: 512 6 | ssl: false 7 | ssl_certificate_file: src/test/resources/test-crt.crt 8 | ssl_key_file: src/test/resources/test-key.crt 9 | use_acm_certificate_for_ssl: false 10 | discovery_mode: static 11 | client_thread_count: 100 12 | batch_size: 100 13 | buffer_size: 100 14 | authentication: 15 | mutual_tls: -------------------------------------------------------------------------------- /data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/buffer/BufferFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.buffer; 7 | 8 | /** 9 | * BufferFactory will act as a means for decoupling the rest of 10 | * the code from the type of buffer being used. 11 | */ 12 | public interface BufferFactory { 13 | Buffer getBuffer(); 14 | } -------------------------------------------------------------------------------- /data-prepper-plugins/encryption-plugin/src/test/resources/test-kms-encryption-engine-config-malformat-sts.yaml: -------------------------------------------------------------------------------- 1 | kms: 2 | encryption_key: AQIDAHjFkMiA+2vRaQi3zhsZlkWK0PIkoEtyUW5mjJOPpgk7wQG01/MpWJcuSIRtCsU/9XIUAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM+bWmsE5QSh1qFsPFAgEQgDuUEZ8iMZmqZyD9Y3yIqROvURbXGvhMGkIuibiSFZ3INpx5hUhbCGLXg++HC6mPWg4aMI2C+fNYzaOKog 3 | key_id: arn:aws:kms:us-east-1:0123456789012:key/8376d0d2-e25f-4b61-99ed-53e6f76d65d5 4 | region: us-east-1 5 | sts_role_arn: invalid_arn -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/patterns_dir_with_custom_patterns_files_glob.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: 8 | - "My phone number is %{CUSTOMPHONENUMBERPATTERN:my_number}" 9 | patterns_directories: 10 | - ./src/test/resources/test_patterns 11 | patterns_files_glob: "*1.txt" 12 | 13 | sink: 14 | - unused: 15 | -------------------------------------------------------------------------------- /release/docker/config/public.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtdT8h+hpRUg0alln+XHS 3 | rYLZW+/8Xgd7P6fuOCF74g6mHr9KWJJ6SMPBjIhl3lzLJwDeD7rHHd6wm4WKsH00 4 | amUK8iWXxKP2ug0N78TNqGkKJi3Z1d8LaEaLJYtRuycucNR1UZLvzzKzG2TV4ayY 5 | JQG9MjJCVa0RrJjZoxfiayC1yUsNnqtw5Ezb46Hspoalje6/kYKZ6AN3y6BUC9Za 6 | +GK9YtRvaitfV9o6X3CDOp6ut/xL/kApvMK9HbiS1/10Kqu736ERdvYGCvRQeEom 7 | 3ODLNpe1E+s9Bs+lTJydaYBTdJEf2UeQe6XsdVaQoE80bXMk9BQF64ECvX2pKRX+ 8 | fQIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /.github/workflows/dco.yml: -------------------------------------------------------------------------------- 1 | name: Developer Certificate of Origin Check 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | check: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - name: Get PR Commits 11 | id: 'get-pr-commits' 12 | uses: tim-actions/get-pr-commits@v1.1.0 13 | with: 14 | token: ${{ secrets.GITHUB_TOKEN }} 15 | - name: DCO Check 16 | uses: tim-actions/dco@v1.1.0 17 | with: 18 | commits: ${{ steps.get-pr-commits.outputs.commits }} 19 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/acknowledgements/ProgressCheck.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.acknowledgements; 7 | 8 | public interface ProgressCheck { 9 | /** 10 | * Returns the pending ratio 11 | * 12 | * @return returns the ratio of pending to the total acknowledgements 13 | * @since 2.6 14 | */ 15 | Double getRatio(); 16 | } 17 | 18 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/transformation/expected/documentdb1-expected.yaml: -------------------------------------------------------------------------------- 1 | simple-pipeline-transformed: 2 | source: 3 | documentdb: 4 | hostname: "database.example.com" 5 | port: "27017" 6 | sink: 7 | - opensearch: 8 | hosts: "database.example.com" 9 | port: "27017" 10 | index: "my_index" 11 | aws: 12 | sts_role_arn: "arn123" 13 | region: "us-test-1" 14 | dlq: 15 | s3: 16 | bucket: "test-bucket" -------------------------------------------------------------------------------- /data-prepper-plugins/csv-processor/README.md: -------------------------------------------------------------------------------- 1 | # CSV Processor 2 | This is a processor that takes in an Event and parses its CSV data into columns. 3 | 4 | https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/csv 5 | 6 | ## Developer Guide 7 | This plugin is compatible with Java 8 and up. See 8 | - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) 9 | - [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/monitoring.md) 10 | -------------------------------------------------------------------------------- /data-prepper-plugins/kinesis-source/src/test/resources/pipeline_with_polling_config_enabled.yaml: -------------------------------------------------------------------------------- 1 | source: 2 | kinesis: 3 | streams: 4 | - stream_name: "stream-1" 5 | - stream_name: "stream-2" 6 | - stream_name: "stream-3" 7 | codec: 8 | ndjson: 9 | aws: 10 | sts_role_arn: "arn:aws:iam::123456789012:role/OSI-PipelineRole" 11 | region: "us-east-1" 12 | consumer_strategy: "polling" 13 | polling: 14 | max_polling_records: 10 15 | idle_time_between_reads: 10s -------------------------------------------------------------------------------- /data-prepper-plugins/kinesis-source/src/test/resources/pipeline_with_stream_arn_config.yaml: -------------------------------------------------------------------------------- 1 | source: 2 | kinesis: 3 | streams: 4 | - stream_arn: "arn:aws:kinesis:us-east-1:123456789012:stream/stream-1" 5 | - stream_arn: "arn:aws:kinesis:us-east-1:123456789012:stream/stream-2" 6 | - stream_arn: "arn:aws:kinesis:us-east-1:123456789012:stream/stream-3" 7 | codec: 8 | ndjson: 9 | aws: 10 | sts_role_arn: "arn:aws:iam::123456789012:role/OSI-PipelineRole" 11 | region: "us-east-1" -------------------------------------------------------------------------------- /e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline.yml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | workers: 2 3 | source: 4 | http: 5 | path: "/${pipelineName}/logs" 6 | processor: 7 | - grok: 8 | match: 9 | log: [ "%{COMMONAPACHELOG}" ] 10 | sink: 11 | - opensearch: 12 | hosts: [ "https://node-0.example.com:9200" ] 13 | username: "admin" 14 | password: "admin" 15 | insecure: true 16 | index: "test-grok-index" 17 | flush_timeout: 5000 18 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/multi-pipelines-single-pipeline-configurations/pipeline_configuration_with_test_extension.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | pipeline_configurations: 3 | test_extension: 4 | test_attribute: test_string 5 | test-pipeline-1: 6 | source: 7 | file: 8 | path: "/tmp/file-source.tmp" 9 | buffer: 10 | bounded_blocking: #to check non object nodes for plugins 11 | sink: 12 | - pipeline: 13 | name: "test-pipeline-2" 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch-api-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearchapi/model/BulkAPIRequestParams.java: -------------------------------------------------------------------------------- 1 | package org.opensearch.dataprepper.plugins.source.opensearchapi.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Getter; 6 | 7 | @AllArgsConstructor 8 | @Getter 9 | @Builder 10 | public class BulkAPIRequestParams { 11 | private final String index; 12 | private final String pipeline; 13 | private final String routing; 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/resources/logs-policy-no-ism-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "policy": { 3 | "description": "Managing logs for log analytics", 4 | "default_state": "current_write_index", 5 | "states": [ 6 | { 7 | "name": "current_write_index", 8 | "actions": [ 9 | { 10 | "rollover": { 11 | "min_size": "50gb", 12 | "min_index_age": "24h" 13 | } 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /data-prepper-plugins/truncate-processor/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | plugins { 7 | id 'java' 8 | } 9 | 10 | dependencies { 11 | implementation project(':data-prepper-api') 12 | implementation project(':data-prepper-test:test-common') 13 | implementation project(':data-prepper-plugins:common') 14 | implementation 'com.fasterxml.jackson.core:jackson-databind' 15 | testImplementation libs.commons.lang3 16 | } 17 | 18 | -------------------------------------------------------------------------------- /data-prepper-pipeline-parser/src/test/resources/multi-pipelines-distributed-pipeline-configurations/pipeline_configuration_with_test_extension1.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | pipeline_configurations: 3 | test_extension_1: 4 | test_attribute: test_string 5 | test-pipeline-1: 6 | source: 7 | file: 8 | path: "/tmp/file-source.tmp" 9 | buffer: 10 | bounded_blocking: #to check non object nodes for plugins 11 | sink: 12 | - pipeline: 13 | name: "test-pipeline-2" 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/DefaultGroupState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.processor.aggregate; 7 | 8 | import java.util.HashMap; 9 | 10 | /** 11 | * The primary implementation of {@link GroupState} 12 | * @since 1.3 13 | */ 14 | class DefaultGroupState extends HashMap implements GroupState { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/patterns_dir_with_default_patterns_files_glob.yaml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | unused: 4 | processor: 5 | - grok: 6 | match: 7 | message: 8 | - "My birthday is %{CUSTOMBIRTHDAYPATTERN:my_birthday} and my phone number is %{CUSTOMPHONENUMBERPATTERN:my_number}" 9 | patterns_directories: 10 | - ./src/test/resources/test_patterns 11 | 12 | sink: 13 | - unused: 14 | -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/resources/raw-span-policy-no-ism-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "policy": { 3 | "description": "Managing raw spans for trace analytics", 4 | "default_state": "current_write_index", 5 | "states": [ 6 | { 7 | "name": "current_write_index", 8 | "actions": [ 9 | { 10 | "rollover": { 11 | "min_size": "50gb", 12 | "min_index_age": "24h" 13 | } 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /config/checkstyle/import-control.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/missing_pipeline_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | someProperty: "someValue" 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - pipeline: 10 | name: "test-pipeline-4" 11 | test-pipeline-2: 12 | source: 13 | pipeline: 14 | name: "test-pipeline-1" 15 | sink: 16 | - pipeline: 17 | name: "test-pipeline-3" -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/kv.mapping.yaml: -------------------------------------------------------------------------------- 1 | pluginName: key_value 2 | mappedAttributeNames: 3 | source: source 4 | target: destination 5 | field_split: field_split_characters 6 | field_split_pattern: field_delimiter_regex 7 | value_split: value_split_characters 8 | value_split_pattern: key_value_delimiter_regex 9 | remove_char_key: delete_key_regex 10 | remove_char_value: delete_value_regex 11 | nestedSyntaxAttributeNames: 12 | - source 13 | - target -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch-insecure.conf: -------------------------------------------------------------------------------- 1 | input { 2 | http { 3 | } 4 | } 5 | filter { 6 | grok { 7 | match => {"log" => "%{COMBINEDAPACHELOG}"} 8 | } 9 | } 10 | output { 11 | opensearch { 12 | hosts => ["fakedomain.us-east-1.es.amazonaws.com"] 13 | user => myuser 14 | password => mypassword 15 | index => "my-index" 16 | ssl_certificate_verification => false 17 | } 18 | } -------------------------------------------------------------------------------- /data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-opensearch-named-captures.conf: -------------------------------------------------------------------------------- 1 | input { 2 | http { 3 | } 4 | } 5 | filter { 6 | grok { 7 | match => {"log" => "%{COMBINEDAPACHELOG}"} 8 | match => {"message" => "(?[A-Za-z0-9_.-]+java)"} 9 | } 10 | } 11 | output { 12 | elasticsearch { 13 | hosts => ["https://localhost:9200"] 14 | user => admin 15 | password => admin 16 | index => "log-ingest" 17 | } 18 | } -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-cache-string-key.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | keys: [ "key1"] 5 | cache: 6 | ttl: 1000 7 | max_size: 1048576 8 | batch: 9 | key_name: "osi_key" 10 | threshold: 11 | event_count: 100 12 | maximum_size: 1mb 13 | event_collect_timeout: 335 14 | aws: 15 | region: "us-east-1" 16 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 17 | 18 | 19 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-circuit-breaker-config.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | batch: 5 | key_name: "osi_key" 6 | threshold: 7 | event_count: 100 8 | maximum_size: 1mb 9 | event_collect_timeout: 335 10 | circuit_breaker_retries: 15 11 | circuit_breaker_wait_interval: 1000 12 | aws: 13 | region: "us-east-1" 14 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 15 | 16 | -------------------------------------------------------------------------------- /data-prepper-plugins/aws-lambda/src/test/resources/lambda-processor-success-config-with-keys-merge-mode.yaml: -------------------------------------------------------------------------------- 1 | function_name: "lambdaProcessorTest" 2 | response_events_match: true 3 | tags_on_failure: [ "lambda_failure" ] 4 | keys: [ "key1" ] 5 | response_mode: "merge" 6 | batch: 7 | key_name: "osi_key" 8 | threshold: 9 | event_count: 100 10 | maximum_size: 1mb 11 | event_collect_timeout: 335 12 | aws: 13 | region: "us-east-1" 14 | sts_role_arn: "arn:aws:iam::1234567890:role/sample-pipeine-role" 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /data-prepper-plugins/encryption-plugin/src/test/resources/test-valid-kms-encryption-engine-config.yaml: -------------------------------------------------------------------------------- 1 | kms: 2 | encryption_key: AQIDAHjFkMiA+2vRaQi3zhsZlkWK0PIkoEtyUW5mjJOPpgk7wQG01/MpWJcuSIRtCsU/9XIUAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM+bWmsE5QSh1qFsPFAgEQgDuUEZ8iMZmqZyD9Y3yIqROvURbXGvhMGkIuibiSFZ3INpx5hUhbCGLXg++HC6mPWg4aMI2C+fNYzaOKog 3 | key_id: arn:aws:kms:us-east-1:0123456789012:key/8376d0d2-e25f-4b61-99ed-53e6f76d65d5 4 | region: us-east-1 5 | sts_role_arn: arn:aws:iam::123456789012:role/test-role -------------------------------------------------------------------------------- /data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/exception/DatabaseReaderInitializationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.geoip.exception; 7 | 8 | public class DatabaseReaderInitializationException extends EngineFailureException { 9 | public DatabaseReaderInitializationException(final String exceptionMsg) { 10 | super(exceptionMsg); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/kafka-pipeline-bootstrap-servers-glue-sts-assume-role.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline : 2 | source: 3 | kafka: 4 | bootstrap_servers: 5 | - "localhost:9092" 6 | encryption: 7 | type: "SSL" 8 | aws: 9 | region: us-east-2 10 | sts_role_arn: sts_role_arn 11 | schema: 12 | type: aws_glue 13 | topics: 14 | - name: "quickstart-events" 15 | group_id: "groupdID1" 16 | sink: 17 | stdout: -------------------------------------------------------------------------------- /data-prepper-plugins/opensearch/src/main/resources/metrics-policy-no-ism-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "policy": { 3 | "description": "Managing metrics for metric analytics", 4 | "default_state": "current_write_index", 5 | "states": [ 6 | { 7 | "name": "current_write_index", 8 | "actions": [ 9 | { 10 | "rollover": { 11 | "min_size": "50gb", 12 | "min_index_age": "24h" 13 | } 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-date-pattern-index.yml: -------------------------------------------------------------------------------- 1 | grok-pipeline: 2 | source: 3 | http: 4 | path: "/${pipelineName}/logs" 5 | processor: 6 | - grok: 7 | match: 8 | log: [ "%{COMMONAPACHELOG}" ] 9 | sink: 10 | - opensearch: 11 | hosts: [ "https://node-0.example.com:9200" ] 12 | username: "admin" 13 | password: "admin" 14 | insecure: true 15 | index: "test-grok-index-%{yyyy.MM.dd}" 16 | flush_timeout: 5000 17 | -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/prometheus/prometheus.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | 5 | scrape_configs: 6 | - job_name: "prometheus" 7 | static_configs: 8 | - targets: ["localhost:9090"] 9 | 10 | - job_name: "data-prepper" 11 | metrics_path: "/metrics/prometheus" 12 | static_configs: 13 | - targets: ["data-prepper:4900"] 14 | 15 | - job_name: "sys" 16 | metrics_path: "/metrics/sys" 17 | static_configs: 18 | - targets: ["data-prepper:4900"] -------------------------------------------------------------------------------- /examples/log-ingestion/docker-compose-dataprepper.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | data-prepper: 4 | image: opensearchproject/data-prepper:2 5 | container_name: data-prepper 6 | volumes: 7 | - ./log_pipeline.yaml:/usr/share/data-prepper/pipelines/log_pipeline.yaml 8 | - ./data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml 9 | ports: 10 | - 2021:2021 11 | networks: 12 | - opensearch-net 13 | 14 | networks: 15 | opensearch-net: 16 | driver: bridge 17 | -------------------------------------------------------------------------------- /release/release-notes/data-prepper.release-notes-2.10.2.md: -------------------------------------------------------------------------------- 1 | ## 2024-12-09 Version 2.10.2 2 | 3 | --- 4 | 5 | ### Bug Fixes 6 | * FIX: missed exception in plugin error ([#5105](https://github.com/opensearch-project/data-prepper/pull/5105)) 7 | 8 | 9 | ### Security 10 | * Fix otel_logs_source server configuration for `getHttpAuthenticationService`. Fixes [CVE-2024-55886](https://github.com/opensearch-project/data-prepper/security/advisories/GHSA-725p-63vv-v948). ([#5215](https://github.com/opensearch-project/data-prepper/pull/5215)) 11 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/Gauge.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.model.metric; 7 | 8 | /** 9 | * A gauge event in Data Prepper represents a metric event. 10 | * @since 1.4 11 | */ 12 | public interface Gauge extends Metric { 13 | 14 | /** 15 | * Gets the value for a gauge 16 | * @return the value 17 | * @since 1.4 18 | */ 19 | Double getValue(); 20 | } 21 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/org/opensearch/dataprepper/core/breaker/InnerCircuitBreaker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.core.breaker; 7 | 8 | import org.opensearch.dataprepper.model.breaker.CircuitBreaker; 9 | 10 | /** 11 | * Interface to signal that this {@link CircuitBreaker} to prevent 12 | * access beyond the {@link CircuitBreakerManager}. 13 | */ 14 | interface InnerCircuitBreaker extends CircuitBreaker { 15 | } 16 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_peer_forwarder_config_with_many_authentication.yml: -------------------------------------------------------------------------------- 1 | port: 21895 2 | request_timeout: 1000 3 | server_thread_count: 100 4 | max_connection_count: 100 5 | max_pending_requests: 512 6 | ssl: true 7 | ssl_certificate_file: src/test/resources/test-crt.crt 8 | ssl_key_file: src/test/resources/test-key.crt 9 | use_acm_certificate_for_ssl: false 10 | discovery_mode: static 11 | client_thread_count: 100 12 | batch_size: 100 13 | buffer_size: 100 14 | authentication: 15 | mutual_tls: 16 | unauthenticated: -------------------------------------------------------------------------------- /data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ExceptionOverview.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.expression; 7 | 8 | final class ExceptionOverview extends RuntimeException { 9 | 10 | ExceptionOverview(final String message) { 11 | super(message); 12 | } 13 | 14 | @Override 15 | public synchronized Throwable fillInStackTrace() { 16 | return this; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkDlqHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.common.sink; 7 | 8 | import org.opensearch.dataprepper.model.event.Event; 9 | 10 | import java.util.List; 11 | 12 | public interface SinkDlqHandler { 13 | void flushDlqList(); 14 | void addFailedEventsToDlq(final List events, final Throwable ex, final int statusCode); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-test/plugin-test-framework/src/test/java/org/opensearch/dataprepper/plugins/test/TestPluggableInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | */ 9 | 10 | package org.opensearch.dataprepper.plugins.test; 11 | 12 | public interface TestPluggableInterface { 13 | String getOptionAValue(); 14 | } 15 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/java/com/example/restservice/Metrics.java: -------------------------------------------------------------------------------- 1 | package com.example.restservice; 2 | 3 | public class Metrics { 4 | 5 | private final long id; 6 | private final String service; 7 | 8 | public Metrics(long id, String service) { 9 | this.id = id; 10 | this.service = service; 11 | } 12 | 13 | public long getId() { 14 | return id; 15 | } 16 | 17 | public String getService() { 18 | return service; 19 | } 20 | } -------------------------------------------------------------------------------- /data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/encoding/DecoderEngine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package org.opensearch.dataprepper.plugins.processor.decompress.encoding; 7 | 8 | import org.opensearch.dataprepper.plugins.processor.decompress.exceptions.DecodingException; 9 | 10 | public interface DecoderEngine { 11 | byte[] decode(final String encodedValue) throws DecodingException; 12 | } 13 | -------------------------------------------------------------------------------- /data-prepper-plugins/kafka-plugins/src/test/resources/kafka-pipeline-bootstrap-servers-glue-override-endpoint.yaml: -------------------------------------------------------------------------------- 1 | log-pipeline : 2 | source: 3 | kafka: 4 | bootstrap_servers: 5 | - "localhost:9092" 6 | encryption: 7 | type: "SSL" 8 | schema: 9 | type: aws_glue 10 | registry_url: http://fake-glue-registry 11 | override_endpoint: true 12 | topics: 13 | - name: "quickstart-events" 14 | group_id: "groupdID1" 15 | sink: 16 | stdout: 17 | -------------------------------------------------------------------------------- /release/release-notes/data-prepper.release-notes-1.1.1.md: -------------------------------------------------------------------------------- 1 | ## 2021-12-10 Version 1.1.1 2 | 3 | --- 4 | 5 | ### Security 6 | * Fixes [CVE-2021-44228](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-44228) by using Apache Log4J 2.15.0. [#718](https://github.com/opensearch-project/data-prepper/pull/718), [#723](https://github.com/opensearch-project/data-prepper/pull/723) 7 | * Run yum update on Docker images to get all security patches at the time of the Data Prepper Docker build. [#717](https://github.com/opensearch-project/data-prepper/pull/717) 8 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/pipeline-dlq.yaml: -------------------------------------------------------------------------------- 1 | test-dlq-pipeline: 2 | delay: 10 3 | source: 4 | in_memory: 5 | testing_key: PipelineDLQIT 6 | 7 | processor: 8 | - simple_test: 9 | key1: /test1 10 | value_prefix1: knownPrefix1 11 | throw_exception: true 12 | 13 | sink: 14 | - in_memory: 15 | testing_key: PipelineDLQNone 16 | 17 | dlq_pipeline: 18 | sink: 19 | - in_memory: 20 | testing_key: PipelineDLQIT 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/processor-swap/source/processor-swap-pipeline.yaml: -------------------------------------------------------------------------------- 1 | processor-pipeline: 2 | delay: 10 3 | source: 4 | in_memory: 5 | testing_key: ProcessorSwapPipelineIT 6 | 7 | processor: 8 | - simple_test: 9 | key1: /test1 10 | value_prefix1: knownPrefix1 11 | - simple_copy_test: 12 | source: /test1 13 | target: /test1_copy_original 14 | 15 | sink: 16 | - in_memory: 17 | testing_key: ProcessorSwapPipelineIT 18 | --------------------------------------------------------------------------------