├── research ├── build.gradle └── zipkin-elastic-to-otel │ ├── README.md │ └── build.gradle ├── data-prepper-benchmarks ├── build.gradle ├── mapdb-benchmarks │ ├── build.gradle │ └── README.md └── service-map-stateful-benchmarks │ ├── build.gradle │ ├── READEME.md │ └── results │ └── summary.md ├── gradle.properties ├── examples ├── data-prepper-config.yaml ├── config │ ├── example-data-prepper-config.yaml │ ├── README.md │ └── example-pipelines.yaml ├── dev │ ├── dns │ │ ├── hosts.prepper │ │ ├── dnsmasq.conf │ │ └── Dockerfile │ ├── trace-analytics-sample-app │ │ ├── resources │ │ │ ├── data-prepper-config.yaml │ │ │ ├── data-prepper-wait-for-odfe-and-start.sh │ │ │ └── pipelines.yaml │ │ ├── images │ │ │ └── grafana_metrics_sample.png │ │ ├── opentelemetry-collector │ │ │ └── otel-collector-config.yml │ │ ├── Dockerfile │ │ ├── prometheus │ │ │ └── prometheus.yaml │ │ └── README.md │ ├── README.md │ └── k8s │ │ ├── build_images_for_minikube.sh │ │ └── README.md ├── trace-analytics-sample-app │ ├── sample-app │ │ ├── analytics-service │ │ │ ├── settings.gradle │ │ │ ├── src │ │ │ │ └── main │ │ │ │ │ ├── resources │ │ │ │ │ └── application.properties │ │ │ │ │ └── java │ │ │ │ │ └── com │ │ │ │ │ └── example │ │ │ │ │ └── restservice │ │ │ │ │ ├── RestServiceApplication.java │ │ │ │ │ ├── Metrics.java │ │ │ │ │ ├── MetricsController.java │ │ │ │ │ ├── Logging.java │ │ │ │ │ └── LoggingController.java │ │ │ ├── gradle │ │ │ │ └── wrapper │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── docker-compose.yml │ │ │ ├── build.gradle │ │ │ ├── README.md │ │ │ └── Dockerfile │ │ ├── requirements.txt │ │ ├── Error.py │ │ └── Dockerfile │ ├── HighLevelDiagram.png │ ├── resources │ │ └── data-prepper-wait-for-odfe-and-start.sh │ └── opentelemetry-collector │ │ └── otel-collector-config.yml ├── aws │ ├── README.md │ └── jaeger-hotrod-on-ec2 │ │ └── README.md ├── demo │ ├── test_keystore.p12 │ ├── demo-data-prepper.crt │ ├── root-ca.pem │ └── demo-data-prepper.key ├── kibana-trace-analytics │ └── Dockerfile ├── zipkin-sleuth-webmvc-example │ ├── otel-collector-config.yml │ └── README.md ├── jaeger-hotrod │ ├── otel-collector-config.yml │ └── README.md ├── adot │ └── aws-ot-collector-config.yml ├── trace_analytics_no_ssl.yml └── trace_analytics.yml ├── NOTICE ├── data-prepper-core └── src │ ├── integrationTest │ └── resources │ │ ├── data_prepper.yml │ │ ├── service-map-e2e-pipeline.yml │ │ ├── raw-span-e2e-pipeline.yml │ │ └── raw-span-e2e-pipeline-latest-release.yml │ ├── test │ ├── resources │ │ ├── invalid_data_prepper_config.yml │ │ ├── valid_data_prepper_some_default_config.yml │ │ ├── invalid_port_data_prepper_config.yml │ │ ├── valid_data_prepper_config.yml │ │ ├── valid_data_prepper_config_default_log4j.yml │ │ ├── valid_data_prepper_cloudwatch_metrics_config.yml │ │ ├── valid_data_prepper_multiple_metrics_config.yml │ │ ├── no_pipelines_to_execute.yml │ │ ├── test.p12 │ │ ├── keystore.jks │ │ ├── cloudwatch_test.properties │ │ ├── tls │ │ │ ├── test_keystore.jks │ │ │ ├── test_keystore.p12 │ │ │ └── test_keystore_with_passwords.p12 │ │ ├── invalid_data_prepper_config_with_bad_keystore_password.yml │ │ ├── valid_data_prepper_config_with_tls.yml │ │ ├── single_pipeline_valid_empty_source_plugin_settings.yml │ │ ├── mockito-extensions │ │ │ └── org.mockito.plugins.MockMaker │ │ ├── missing_name_multiple_pipeline_configuration.yml │ │ ├── missing_pipeline_multiple_pipeline_configuration.yml │ │ ├── incorrect_source_multiple_pipeline_configuration.yml │ │ ├── connected_pipeline_incorrect_root_source.yml │ │ ├── cyclic_multiple_pipeline_configuration.yml │ │ ├── valid_multiple_pipeline_configuration.yml │ │ ├── valid_multiple_sinks.yml │ │ ├── connected_pipeline_incorrect_child_pipeline.yml │ │ └── valid_multiple_processors.yml │ └── java │ │ └── com │ │ └── amazon │ │ └── dataprepper │ │ ├── server │ │ ├── SslUtilTest.java │ │ └── CloudWatchMeterRegistryProviderTest.java │ │ └── plugins │ │ └── TestSink.java │ └── main │ ├── resources │ └── cloudwatch.properties │ └── java │ └── com │ └── amazon │ └── dataprepper │ ├── parser │ ├── ParseException.java │ └── model │ │ └── MetricRegistryType.java │ ├── DataPrepperExecute.java │ └── pipeline │ ├── server │ ├── ShutdownHandler.java │ ├── PrometheusMetricsHandler.java │ └── SslUtil.java │ └── common │ ├── FutureHelper.java │ └── PipelineThreadFactory.java ├── data-prepper-plugins ├── common │ ├── src │ │ ├── test │ │ │ ├── resources │ │ │ │ └── test-file-source.tst │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── amazon │ │ │ │ └── dataprepper │ │ │ │ └── plugins │ │ │ │ ├── ConstructorLessComponent.java │ │ │ │ ├── sink │ │ │ │ ├── StdOutSinkTests.java │ │ │ │ ├── TestSink.java │ │ │ │ └── SinkFactoryTests.java │ │ │ │ ├── prepper │ │ │ │ └── TestPrepper.java │ │ │ │ ├── PluginFactoryTests.java │ │ │ │ ├── source │ │ │ │ ├── RandomStringSourceTests.java │ │ │ │ └── TestSource.java │ │ │ │ └── buffer │ │ │ │ └── BufferFactoryTests.java │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── amazon │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ ├── PluginException.java │ │ │ ├── sink │ │ │ ├── SinkFactory.java │ │ │ └── StdOutSink.java │ │ │ ├── source │ │ │ └── SourceFactory.java │ │ │ ├── buffer │ │ │ └── BufferFactory.java │ │ │ └── prepper │ │ │ ├── PrepperFactory.java │ │ │ └── NoOpPrepper.java │ ├── build.gradle │ └── README.md ├── build.gradle ├── otel-trace-source │ ├── src │ │ ├── test │ │ │ └── resources │ │ │ │ └── mockito-extensions │ │ │ │ └── org.mockito.plugins.MockMaker │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── amazon │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ ├── certificate │ │ │ ├── CertificateProvider.java │ │ │ ├── model │ │ │ │ └── Certificate.java │ │ │ └── file │ │ │ │ └── FileCertificateProvider.java │ │ │ └── health │ │ │ └── HealthGrpcService.java │ └── data │ │ └── certificate │ │ ├── test_cert.crt │ │ ├── test_decrypted_key.key │ │ └── test_encrypted_key.key ├── peer-forwarder │ ├── src │ │ ├── test │ │ │ ├── resources │ │ │ │ ├── mockito-extensions │ │ │ │ │ └── org.mockito.plugins.MockMaker │ │ │ │ ├── log4j2.properties │ │ │ │ └── test-crt.crt │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── amazon │ │ │ │ └── dataprepper │ │ │ │ └── plugins │ │ │ │ └── prepper │ │ │ │ └── peerforwarder │ │ │ │ └── certificate │ │ │ │ └── file │ │ │ │ └── FileCertificateProviderTest.java │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── amazon │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── prepper │ │ │ └── peerforwarder │ │ │ ├── certificate │ │ │ ├── CertificateProvider.java │ │ │ ├── model │ │ │ │ └── Certificate.java │ │ │ └── file │ │ │ │ └── FileCertificateProvider.java │ │ │ └── discovery │ │ │ ├── DiscoveryUtils.java │ │ │ ├── PeerListProvider.java │ │ │ ├── PeerListProviderFactory.java │ │ │ └── DiscoveryMode.java │ └── build.gradle ├── otel-trace-group-prepper │ ├── src │ │ ├── test │ │ │ ├── resources │ │ │ │ ├── mockito-extensions │ │ │ │ │ └── org.mockito.plugins.MockMaker │ │ │ │ ├── raw-span-missing-trace-group-1.json │ │ │ │ ├── raw-span-missing-trace-group-2.json │ │ │ │ ├── raw-span-complete-1.json │ │ │ │ └── raw-span-complete-2.json │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── amazon │ │ │ │ └── dataprepper │ │ │ │ └── plugins │ │ │ │ └── prepper │ │ │ │ └── oteltracegroup │ │ │ │ └── OTelTraceGroupPrepperConfigTests.java │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── amazon │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── prepper │ │ │ └── oteltracegroup │ │ │ └── OTelTraceGroupPrepperConfig.java │ └── build.gradle ├── service-map-stateful │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── amazon │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── prepper │ │ │ ├── ServiceMapPrepperConfig.java │ │ │ └── OTelHelper.java │ ├── build.gradle │ └── README.md ├── elasticsearch │ ├── src │ │ ├── test │ │ │ ├── resources │ │ │ │ ├── service-map-1.json │ │ │ │ ├── test-index-template.json │ │ │ │ ├── test-index-template-v2.json │ │ │ │ ├── test-template-withshards.json │ │ │ │ ├── raw-span-error.json │ │ │ │ ├── raw-span-1.json │ │ │ │ ├── raw-span-2.json │ │ │ │ └── test-ca.pem │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── amazon │ │ │ │ └── dataprepper │ │ │ │ └── plugins │ │ │ │ └── sink │ │ │ │ └── elasticsearch │ │ │ │ ├── ODFETests.java │ │ │ │ ├── BackOffUtilsTests.java │ │ │ │ ├── RetryConfigurationTests.java │ │ │ │ └── ElasticsearchSinkConfigurationTests.java │ │ └── main │ │ │ ├── resources │ │ │ ├── raw-span-policy-no-ism-template.json │ │ │ ├── raw-span-policy-with-ism-template.json │ │ │ └── otel-v1-apm-service-map-index-template.json │ │ │ └── java │ │ │ └── com │ │ │ └── amazon │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── sink │ │ │ └── elasticsearch │ │ │ ├── RequestFunction.java │ │ │ ├── BackOffUtils.java │ │ │ └── RetryConfiguration.java │ └── odfe_security.md ├── blocking-buffer │ ├── build.gradle │ └── README.md ├── otel-trace-raw-prepper │ ├── src │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── amazon │ │ │ │ └── dataprepper │ │ │ │ └── plugins │ │ │ │ └── prepper │ │ │ │ └── oteltrace │ │ │ │ ├── OtelTraceRawPrepperConfig.java │ │ │ │ └── model │ │ │ │ ├── RawSpanSet.java │ │ │ │ └── RawEvent.java │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── amazon │ │ │ └── dataprepper │ │ │ └── plugins │ │ │ └── prepper │ │ │ └── oteltrace │ │ │ └── model │ │ │ └── TestUtils.java │ ├── build.gradle │ └── README.md └── mapdb-prepper-state │ └── build.gradle ├── release ├── archives │ ├── linux │ │ ├── build.gradle │ │ └── data-prepper-tar-install-with-jdk-x64.sh │ └── macos │ │ ├── build.gradle │ │ └── data-prepper-tar-install-with-jdk-x64.sh ├── README.md ├── release-notes │ ├── odfe-data-prepper.release-notes-1.0.0.md │ └── odfe-data-prepper.release-notes-0.7.0.md ├── docker │ ├── Dockerfile │ ├── build.gradle │ └── README.md ├── build.gradle └── build-resources.gradle ├── docs ├── dev │ └── images │ │ ├── TraceView.png │ │ ├── DashboardView.png │ │ ├── ServiceView.png │ │ ├── HighLevelDesign.jpg │ │ ├── DataPrepperPipeline.png │ │ └── TraceAnalyticsFeature.jpg └── readme │ ├── images │ ├── Components.jpg │ ├── DataPrepperPipeline.png │ └── TraceAnalyticsFeature.jpg │ ├── logs.md │ └── error_handling.md ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .dockerignore ├── .gitattributes ├── .gitignore ├── CODE_OF_CONDUCT.md ├── .github ├── pull_request_template.md ├── workflows │ ├── gradle.yml │ ├── data-prepper-trace-analytics-raw-span-e2e-tests.yml │ ├── data-prepper-trace-analytics-service-map-e2e-tests.yml │ ├── data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml │ ├── opensearch-sink-odfe-before-1_13_0-integration-tests.yml │ └── opensearch-sink-odfe-since-1_13_0-integration-tests.yml ├── ISSUE_TEMPLATE │ └── bug_report.md └── dependabot.yml ├── data-prepper-api ├── build.gradle └── src │ ├── test │ └── java │ │ └── com │ │ └── amazon │ │ └── dataprepper │ │ ├── model │ │ ├── CheckpointStateTest.java │ │ └── PluginTypeTests.java │ │ └── metrics │ │ └── MetricsTestUtil.java │ └── main │ └── java │ └── com │ └── amazon │ └── dataprepper │ └── model │ ├── CheckpointState.java │ ├── annotations │ ├── SingleThread.java │ └── DataPrepperPlugin.java │ ├── sink │ ├── Sink.java │ └── AbstractSink.java │ ├── source │ └── Source.java │ ├── PluginType.java │ ├── record │ └── Record.java │ ├── prepper │ └── Prepper.java │ └── buffer │ └── Buffer.java ├── deployment-template ├── ec2 │ └── README.md └── eks │ ├── ingress.yaml │ └── README.md ├── shared-config ├── log4j2.properties ├── log4j.properties └── log4j2-rolling.properties └── settings.gradle /research/build.gradle: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data-prepper-benchmarks/build.gradle: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=1.1.0 2 | -------------------------------------------------------------------------------- /examples/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /examples/config/example-data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /examples/dev/dns/hosts.prepper: -------------------------------------------------------------------------------- 1 | 10.10.1.2 prepper-cluster 2 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/data_prepper.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/resources/test-file-source.tst: -------------------------------------------------------------------------------- 1 | THIS IS A TEST -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config.yml: -------------------------------------------------------------------------------- 1 | badKey: badValue -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_some_default_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/resources/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_port_data_prepper_config.yml: -------------------------------------------------------------------------------- 1 | serverPort: -550 2 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config.yml: -------------------------------------------------------------------------------- 1 | serverPort: 5678 2 | ssl: false -------------------------------------------------------------------------------- /examples/dev/dns/dnsmasq.conf: -------------------------------------------------------------------------------- 1 | server=127.0.0.11 # Embedded Docker DNS 2 | addn-hosts=/etc/hosts.prepper 3 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_default_log4j.yml: -------------------------------------------------------------------------------- 1 | serverPort: 1234 2 | ssl: false -------------------------------------------------------------------------------- /release/archives/linux/build.gradle: -------------------------------------------------------------------------------- 1 | //This file is intentionally left blank to support commands for this project -------------------------------------------------------------------------------- /release/archives/macos/build.gradle: -------------------------------------------------------------------------------- 1 | //This file is intentionally left blank to support commands for this project -------------------------------------------------------------------------------- /examples/config/README.md: -------------------------------------------------------------------------------- 1 | Directory containing example configuration files to be bundled with release artifacts. 2 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'analytics-service' -------------------------------------------------------------------------------- /examples/aws/README.md: -------------------------------------------------------------------------------- 1 | ## AWS Examples 2 | This directory contains AWS-specific sample content (e.g. running on EC2). 3 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metricRegistries: [CloudWatch] -------------------------------------------------------------------------------- /docs/dev/images/TraceView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/dev/images/TraceView.png -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | metricRegistries: [Prometheus, CloudWatch] -------------------------------------------------------------------------------- /docs/dev/images/DashboardView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/dev/images/DashboardView.png -------------------------------------------------------------------------------- /docs/dev/images/ServiceView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/dev/images/ServiceView.png -------------------------------------------------------------------------------- /docs/readme/images/Components.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/readme/images/Components.jpg -------------------------------------------------------------------------------- /examples/demo/test_keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/examples/demo/test_keystore.p12 -------------------------------------------------------------------------------- /examples/dev/dns/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | RUN apk --no-cache add dnsmasq-dnssec 3 | EXPOSE 53 53/udp 4 | ENTRYPOINT ["dnsmasq", "-k"] 5 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /docs/dev/images/HighLevelDesign.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/dev/images/HighLevelDesign.jpg -------------------------------------------------------------------------------- /docs/dev/images/DataPrepperPipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/dev/images/DataPrepperPipeline.png -------------------------------------------------------------------------------- /docs/dev/images/TraceAnalyticsFeature.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/dev/images/TraceAnalyticsFeature.jpg -------------------------------------------------------------------------------- /docs/readme/images/DataPrepperPipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/readme/images/DataPrepperPipeline.png -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | #ignore gradle specific files and examples from copying to image 2 | build 3 | docs 4 | examples 5 | #research remove this after updating settings.gradle -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/no_pipelines_to_execute.yml: -------------------------------------------------------------------------------- 1 | fail-pipeline: 2 | source: 3 | non-existent-source: 4 | sink: 5 | - non-existent-sink: 6 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/test.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/data-prepper-core/src/test/resources/test.p12 -------------------------------------------------------------------------------- /docs/readme/images/TraceAnalyticsFeature.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/docs/readme/images/TraceAnalyticsFeature.jpg -------------------------------------------------------------------------------- /data-prepper-core/src/main/resources/cloudwatch.properties: -------------------------------------------------------------------------------- 1 | cloudwatch.enabled=true 2 | cloudwatch.namespace=dataprepper 3 | cloudwatch.batchSize=20 4 | cloudwatch.step=PT1M -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/data-prepper-core/src/test/resources/keystore.jks -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/cloudwatch_test.properties: -------------------------------------------------------------------------------- 1 | cloudwatch.enabled=true 2 | cloudwatch.namespace=dataprepper-test 3 | cloudwatch.batchSize=20 4 | cloudwatch.step=PT1S -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | # These are explicitly windows files and should use crlf 5 | *.bat text eol=crlf 6 | -------------------------------------------------------------------------------- /data-prepper-plugins/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | dependencies { 6 | subprojects.forEach { api project(':data-prepper-plugins:' + it.name) } 7 | } -------------------------------------------------------------------------------- /examples/dev/README.md: -------------------------------------------------------------------------------- 1 | # Dev Examples 2 | This directory contains example projects intended for developing/testing new features. Examples are not guaranteed to be in a working state. 3 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=8087 2 | elasticsearch.host=elasticsearch 3 | elasticsearch.port=9200 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/test_keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/data-prepper-core/src/test/resources/tls/test_keystore.jks -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/test_keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/data-prepper-core/src/test/resources/tls/test_keystore.p12 -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/HighLevelDiagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/examples/trace-analytics-sample-app/HighLevelDiagram.png -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/invalid_data_prepper_config_with_bad_keystore_password.yml: -------------------------------------------------------------------------------- 1 | serverPort: 1234 2 | ssl: true 3 | keyStoreFilePath: "src/test/resources/test.p12" 4 | keyStorePassword: "bad_password" 5 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/tls/test_keystore_with_passwords.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/data-prepper-core/src/test/resources/tls/test_keystore_with_passwords.p12 -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_data_prepper_config_with_tls.yml: -------------------------------------------------------------------------------- 1 | serverPort: 1234 2 | ssl: true 3 | keyStoreFilePath: "src/test/resources/test.p12" 4 | keyStorePassword: "store" 5 | privateKeyPassword: "key" 6 | -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/images/grafana_metrics_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/examples/dev/trace-analytics-sample-app/images/grafana_metrics_sample.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Gradle project-specific cache directory 2 | .DS_Store 3 | .idea 4 | .gradle 5 | 6 | # Ignore Gradle build output directory 7 | build 8 | 9 | # Ignore things downloaded for gradle es plugin 10 | gradle/tools 11 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/single_pipeline_valid_empty_source_plugin_settings.yml: -------------------------------------------------------------------------------- 1 | test-pipeline-1: 2 | source: 3 | stdin: 4 | buffer: 5 | bounded_blocking: #to check non object nodes for plugins 6 | sink: 7 | - stdout: -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/data-prepper/HEAD/examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /examples/kibana-trace-analytics/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazon/opendistro-for-elasticsearch-kibana:1.12.0 2 | RUN /usr/share/kibana/bin/kibana-plugin install https://github.com/opendistro-for-elasticsearch/trace-analytics/releases/download/v1.12.0.0-alpha/trace-analytics-v1.12.0.0-alpha.zip 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Aug 03 12:43:56 CDT 2020 2 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip 3 | distributionBase=GRADLE_USER_HOME 4 | distributionPath=wrapper/dists 5 | zipStorePath=wrapper/dists 6 | zipStoreBase=GRADLE_USER_HOME 7 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | # To enable mocking of final classes with vanilla Mockito 2 | # https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods 3 | mock-maker-inline 4 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | # To enable mocking of final classes with vanilla Mockito 2 | # https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods 3 | mock-maker-inline -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | # To enable mocking of final classes with vanilla Mockito 2 | # https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods 3 | mock-maker-inline 4 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | # To enable mocking of final classes with vanilla Mockito 2 | # https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods 3 | mock-maker-inline -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /examples/zipkin-sleuth-webmvc-example/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | zipkin: 3 | 4 | exporters: 5 | otlp/2: 6 | endpoint: data-prepper:21890 7 | insecure: true 8 | logging: 9 | 10 | service: 11 | pipelines: 12 | traces: 13 | receivers: [zipkin] 14 | exporters: [logging, otlp/2] 15 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/requirements.txt: -------------------------------------------------------------------------------- 1 | dash==1.17.0 2 | mysql-connector==2.2.9 3 | opentelemetry-exporter-otlp==1.0.0 4 | opentelemetry-instrumentation-flask==0.19b0 5 | opentelemetry-instrumentation-mysql==0.19b0 6 | opentelemetry-instrumentation-requests==0.19b0 7 | opentelemetry-sdk==1.0.0 8 | protobuf==3.15.6 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.certificate; 2 | 3 | import com.amazon.dataprepper.plugins.certificate.model.Certificate; 4 | 5 | public interface CertificateProvider { 6 | Certificate getCertificate(); 7 | } 8 | -------------------------------------------------------------------------------- /examples/jaeger-hotrod/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | jaeger: 3 | protocols: 4 | grpc: 5 | 6 | exporters: 7 | otlp/2: 8 | endpoint: data-prepper:21890 9 | insecure: true 10 | logging: 11 | 12 | service: 13 | pipelines: 14 | traces: 15 | receivers: [jaeger] 16 | exporters: [logging, otlp/2] 17 | -------------------------------------------------------------------------------- /examples/dev/k8s/build_images_for_minikube.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | eval $(minikube docker-env) 3 | docker build ../../trace-analytics-sample-app/sample-app -t example-k8s/sample-app 4 | docker build ../../kibana-trace-analytics -t example-k8s/kibana 5 | docker build ../../.. -f ../../../examples/dev/trace-analytics-sample-app/Dockerfile -t example-k8s/data-prepper 6 | -------------------------------------------------------------------------------- /data-prepper-benchmarks/mapdb-benchmarks/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id "me.champeau.gradle.jmh" version "0.5.0" 4 | } 5 | 6 | group 'com.amazon' 7 | version '0.1-beta' 8 | 9 | sourceCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation project(':data-prepper-plugins:mapdb-prepper-state') 17 | } 18 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; 2 | 3 | import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; 4 | 5 | public interface CertificateProvider { 6 | Certificate getCertificate(); 7 | } 8 | -------------------------------------------------------------------------------- /data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapPrepperConfig.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper; 2 | 3 | public class ServiceMapPrepperConfig { 4 | static final String WINDOW_DURATION = "window_duration"; 5 | static final int DEFAULT_WINDOW_DURATION = 180; 6 | static final String DEFAULT_DB_PATH = "data/service-map/"; 7 | } 8 | -------------------------------------------------------------------------------- /release/README.md: -------------------------------------------------------------------------------- 1 | ## Release project for Data Prepper 2 | 3 | This repository includes different mechanisms for distributing/releasing the data prepper. The directories/sub-projects represent 4 | the specific mechanism and holds README with instructions. For example docker directory holds scripts for building the docker 5 | image of the data prepper, and the instructions to build the image. 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/service-map-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "hashId": "aQ/2NNEmtuwsGAOR5ntCNwk=", 3 | "serviceName": "Payment", 4 | "kind": "Client", 5 | "target": 6 | { 7 | "domain": "Purchase", 8 | "resource": "Buy" 9 | }, 10 | "destination": 11 | { 12 | "domain": "Purchase", 13 | "resource": "Buy" 14 | }, 15 | "traceGroupName": "MakePayement.auto" 16 | } -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/resources/data-prepper-wait-for-odfe-and-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | until [[ $(curl --write-out %{http_code} --output /dev/null --silent --head --fail https://node-0.example.com:9200 -u admin:admin --insecure) == 200 ]]; do 4 | echo "Waiting for ODFE to be ready" 5 | sleep 1 6 | done 7 | 8 | java -Xms128m -Xmx128m -jar data-prepper.jar pipelines.yaml data-prepper-config.yaml -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/opentelemetry-collector/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:55680 6 | 7 | exporters: 8 | otlp/2: 9 | endpoint: data-prepper:21890 10 | insecure: true 11 | logging: 12 | 13 | service: 14 | pipelines: 15 | traces: 16 | receivers: [otlp] 17 | exporters: [logging, otlp/2] 18 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | appender.console.type = Console 2 | appender.console.name = STDOUT 3 | appender.console.layout.type = PatternLayout 4 | appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n 5 | 6 | rootLogger.level = warn 7 | rootLogger.appenderRef.stdout.ref = STDOUT 8 | 9 | logger.core.name = com.amazon.dataprepper 10 | logger.core.level = debug 11 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | ----- 6 | 7 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 8 | For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md). 9 | -------------------------------------------------------------------------------- /release/release-notes/odfe-data-prepper.release-notes-1.0.0.md: -------------------------------------------------------------------------------- 1 | # 2021-05-11 Version 1.0.0 2 | 3 | ## Highlights 4 | * Now builds using [version 1.0+](https://github.com/open-telemetry/opentelemetry-specification/pull/1372) of the OpenTelemetry tracing specification 5 | * Additional TraceGroup fields are emitted for enhanced searching and filtering 6 | 7 | ## Compatibility 8 | * No compatibility issues with previous versions of Data Prepper. 9 | -------------------------------------------------------------------------------- /examples/adot/aws-ot-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:55680 6 | 7 | exporters: 8 | otlp/2: 9 | endpoint: data-prepper:21890 10 | insecure: true 11 | logging: 12 | 13 | service: 14 | pipelines: 15 | traces: 16 | receivers: [otlp] 17 | exporters: [otlp/2, logging] 18 | metrics: 19 | receivers: [ otlp ] 20 | exporters: [ logging ] 21 | -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/resources/data-prepper-wait-for-odfe-and-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | until [[ $(curl --write-out %{http_code} --output /dev/null --silent --head --fail https://node-0.example.com:9200 -u admin:admin --insecure) == 200 ]]; do 4 | echo "Waiting for ODFE to be ready" 5 | sleep 1 6 | done 7 | 8 | java -Dlog4j.configurationFile=log4j.properties -Xms128m -Xmx128m -jar data-prepper.jar pipelines.yaml data-prepper-config.yaml -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/opentelemetry-collector/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | endpoint: 0.0.0.0:55680 6 | 7 | exporters: 8 | otlp/2: 9 | endpoint: data-prepper:21890 10 | insecure: false 11 | insecure_skip_verify: true 12 | logging: 13 | 14 | service: 15 | pipelines: 16 | traces: 17 | receivers: [otlp] 18 | exporters: [logging, otlp/2] 19 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/missing_name_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | someProperty: "someValue" 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - pipeline: 10 | test-pipeline-2: 11 | source: 12 | pipeline: 13 | name: "test-pipeline-1" 14 | sink: 15 | - file: 16 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | elasticsearch: 4 | image: docker.elastic.co/elasticsearch/elasticsearch:7.8.1 5 | ports: 6 | - "9200:9200" 7 | - "9300:9300" 8 | environment: 9 | - discovery.type=single-node 10 | analytics-service: 11 | build: 12 | context: . 13 | dockerfile: Dockerfile 14 | ports: 15 | - "8087:8087" 16 | -------------------------------------------------------------------------------- /data-prepper-plugins/blocking-buffer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | dependencies { 5 | implementation project(':data-prepper-api') 6 | testImplementation "junit:junit:4.13.2" 7 | } 8 | 9 | jacocoTestCoverageVerification { 10 | dependsOn jacocoTestReport 11 | violationRules { 12 | rule { //in addition to core projects rule 13 | limit { 14 | minimum = 0.90 15 | } 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id "me.champeau.gradle.jmh" version "0.5.0" 4 | } 5 | 6 | group 'com.amazon' 7 | version '0.1-beta' 8 | 9 | sourceCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation project(':data-prepper-plugins:service-map-stateful') 17 | jmh "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" 18 | } 19 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/java/com/example/restservice/RestServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.example.restservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class RestServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(RestServiceApplication.class, args); 11 | } 12 | } -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.oteltrace; 2 | 3 | public class OtelTraceRawPrepperConfig { 4 | static final String TRACE_FLUSH_INTERVAL = "trace_flush_interval"; 5 | static final long DEFAULT_TG_FLUSH_INTERVAL_SEC = 180L; 6 | static final long DEFAULT_TRACE_ID_TTL_SEC = 15L; 7 | static final long MAX_TRACE_ID_CACHE_SIZE = 1000_000L; 8 | } 9 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/parser/ParseException.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.parser; 2 | 3 | public class ParseException extends RuntimeException { 4 | public ParseException(final Throwable cause) { 5 | super(cause); 6 | } 7 | 8 | public ParseException(final String message) { 9 | super(message); 10 | } 11 | 12 | public ParseException(final String message, final Throwable cause) { 13 | super(message, cause); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TestUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.oteltrace.model; 2 | 3 | import java.util.Random; 4 | 5 | public class TestUtils { 6 | 7 | private static final Random RANDOM = new Random(); 8 | 9 | public static byte[] getRandomBytes(int len) { 10 | byte[] bytes = new byte[len]; 11 | RANDOM.nextBytes(bytes); 12 | return bytes; 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gradle:jdk14 AS builder 2 | COPY . /home/gradle/src 3 | WORKDIR /home/gradle/src/data-prepper-core 4 | RUN gradle clean jar --daemon 5 | 6 | FROM amazoncorretto:15-al2-full 7 | EXPOSE 21890 8 | WORKDIR /usr/share/data-prepper/ 9 | COPY --from=builder /home/gradle/src/data-prepper-core/build/libs/data-prepper*.jar /usr/share/data-prepper/data-prepper.jar 10 | CMD ["java", "-Xms128m", "-Xmx128m", "-jar", "data-prepper.jar", "pipelines.yaml", "data-prepper-config.yaml"] -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/PluginException.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins; 2 | 3 | public class PluginException extends RuntimeException { 4 | public PluginException(final Throwable cause) { 5 | super(cause); 6 | } 7 | 8 | public PluginException(final String message) { 9 | super(message); 10 | } 11 | 12 | public PluginException(final String message, final Throwable cause) { 13 | super(message, cause); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/missing_pipeline_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | someProperty: "someValue" 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - pipeline: 10 | name: "test-pipeline-4" 11 | test-pipeline-2: 12 | source: 13 | pipeline: 14 | name: "test-pipeline-1" 15 | sink: 16 | - pipeline: 17 | name: "test-pipeline-3" -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-no-ism-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "policy": { 3 | "description": "Managing raw spans for trace analytics", 4 | "default_state": "current_write_index", 5 | "states": [ 6 | { 7 | "name": "current_write_index", 8 | "actions": [ 9 | { 10 | "rollover": { 11 | "min_size": "50gb", 12 | "min_index_age": "24h" 13 | } 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/prometheus/prometheus.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | 5 | scrape_configs: 6 | - job_name: "prometheus" 7 | static_configs: 8 | - targets: ["localhost:9090"] 9 | 10 | - job_name: "data-prepper" 11 | metrics_path: "/metrics/prometheus" 12 | static_configs: 13 | - targets: ["data-prepper:4900"] 14 | 15 | - job_name: "sys" 16 | metrics_path: "/metrics/sys" 17 | static_configs: 18 | - targets: ["data-prepper:4900"] -------------------------------------------------------------------------------- /data-prepper-api/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | 3 | } 4 | dependencies { 5 | implementation "io.micrometer:micrometer-core:1.7.2" 6 | 7 | testImplementation "org.hamcrest:hamcrest:2.2" 8 | } 9 | jacocoTestCoverageVerification { 10 | dependsOn jacocoTestReport 11 | violationRules { 12 | rule { //in addition to core projects rule - this one checks for 100% code coverage for this project 13 | limit { 14 | minimum = 1.0 //keep this at 100% 15 | } 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/RequestFunction.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink.elasticsearch; 2 | 3 | /** 4 | * Represents a function that accepts one argument and produces a result or throws Exception. 5 | * 6 | * @param the type of the input to the function 7 | * @param the type of the result of the function 8 | */ 9 | @FunctionalInterface 10 | public interface RequestFunction { 11 | 12 | R apply(T t) throws Exception; 13 | } 14 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/java/com/example/restservice/Metrics.java: -------------------------------------------------------------------------------- 1 | package com.example.restservice; 2 | 3 | public class Metrics { 4 | 5 | private final long id; 6 | private final String service; 7 | 8 | public Metrics(long id, String service) { 9 | this.id = id; 10 | this.service = service; 11 | } 12 | 13 | public long getId() { 14 | return id; 15 | } 16 | 17 | public String getService() { 18 | return service; 19 | } 20 | } -------------------------------------------------------------------------------- /release/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazoncorretto:15-al2-full 2 | ARG PIPELINE_FILEPATH 3 | ARG CONFIG_FILEPATH 4 | ARG JAR_FILE 5 | 6 | ENV DATA_PREPPER_PATH /usr/share/data-prepper 7 | ENV ENV_CONFIG_FILEPATH=$CONFIG_FILEPATH 8 | ENV ENV_PIPELINE_FILEPATH=$PIPELINE_FILEPATH 9 | 10 | RUN mkdir -p $DATA_PREPPER_PATH 11 | RUN mkdir -p /var/log/data-prepper 12 | COPY $JAR_FILE /usr/share/data-prepper/data-prepper.jar 13 | 14 | WORKDIR $DATA_PREPPER_PATH 15 | CMD java $JAVA_OPTS -jar data-prepper.jar ${ENV_PIPELINE_FILEPATH} ${ENV_CONFIG_FILEPATH} 16 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/Error.py: -------------------------------------------------------------------------------- 1 | from flask import jsonify 2 | 3 | class Error(Exception): 4 | status_code = 400 5 | 6 | def __init__(self, message, status_code=None, payload=None): 7 | Exception.__init__(self) 8 | self.message = message 9 | if status_code is not None: 10 | self.status_code = status_code 11 | self.payload = payload 12 | 13 | def to_dict(self): 14 | result = dict(self.payload or ()) 15 | result['message'] = self.message 16 | return result -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; 2 | 3 | import com.google.common.net.InternetDomainName; 4 | import org.apache.commons.validator.routines.InetAddressValidator; 5 | 6 | class DiscoveryUtils { 7 | static boolean validateEndpoint(final String endpoint) { 8 | return InternetDomainName.isValid(endpoint) || InetAddressValidator.getInstance().isValid(endpoint); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /deployment-template/ec2/README.md: -------------------------------------------------------------------------------- 1 | # Deploying with CloudFormation 2 | This directory contains an [AWS CloudFormation](https://docs.aws.amazon.com/cloudformation/index.html) template which provisions a single Data Prepper instance on an EC2 host. The template's parameters allow you to configure settings such as the EC2 instance type, Amazon Elasticsearch Service endpoint, and which SSH key pair can be used to access the host. 3 | 4 | Data Prepper server and pipeline configuration values can be manually adjusted by editing the _Resources_ section of the yaml template. 5 | -------------------------------------------------------------------------------- /release/build.gradle: -------------------------------------------------------------------------------- 1 | apply from: file("build-resources.gradle") 2 | 3 | allprojects { 4 | dependencies { 5 | implementation project(':data-prepper-core') 6 | } 7 | } 8 | 9 | task endToEndTests { 10 | // TODO add benchmark test and enable 11 | } 12 | 13 | task benchmarkTests { 14 | // TODO add benchmark test and enable 15 | } 16 | 17 | task buildCore { 18 | dependsOn ':data-prepper-core:build' 19 | } 20 | 21 | task releasePrerequisites { 22 | dependsOn 'buildCore' 23 | dependsOn 'endToEndTests' 24 | dependsOn 'benchmarkTests' 25 | } -------------------------------------------------------------------------------- /data-prepper-api/src/test/java/com/amazon/dataprepper/model/CheckpointStateTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertEquals; 6 | 7 | public class CheckpointStateTest { 8 | private static final int TEST_NUM_CHECKED_RECORDS = 3; 9 | 10 | @Test 11 | public void testSimple() { 12 | final CheckpointState checkpointState = new CheckpointState(TEST_NUM_CHECKED_RECORDS); 13 | assertEquals(TEST_NUM_CHECKED_RECORDS, checkpointState.getNumRecordsToBeChecked()); 14 | } 15 | } -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/CheckpointState.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model; 2 | 3 | /** 4 | * CheckpointState keeps track of a summary of records processed by a data-prepper worker thread. 5 | */ 6 | public class CheckpointState { 7 | private final int numRecordsToBeChecked; 8 | 9 | public CheckpointState(final int numRecordsToBeChecked) { 10 | this.numRecordsToBeChecked = numRecordsToBeChecked; 11 | } 12 | 13 | public int getNumRecordsToBeChecked() { 14 | return numRecordsToBeChecked; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/incorrect_source_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | buffer: 6 | bounded_blocking: #to check non object nodes for plugins 7 | sink: 8 | - pipeline: 9 | name: "test-pipeline-2" 10 | test-pipeline-2: 11 | source: 12 | pipeline: 13 | name: "test-pipeline-4" 14 | sink: 15 | - pipeline: 16 | name: "test-pipeline-3" 17 | test-pipeline-3: 18 | source: 19 | pipeline: 20 | name: "test-pipeline-2" 21 | sink: 22 | - file: -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "policy": { 3 | "description": "Managing raw spans for trace analytics", 4 | "default_state": "current_write_index", 5 | "states": [ 6 | { 7 | "name": "current_write_index", 8 | "actions": [ 9 | { 10 | "rollover": { 11 | "min_size": "50gb", 12 | "min_index_age": "24h" 13 | } 14 | } 15 | ] 16 | } 17 | ], 18 | "ism_template": { 19 | "index_patterns": ["otel-v1-apm-span-*"] 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/annotations/SingleThread.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.annotations; 2 | 3 | import java.lang.annotation.Documented; 4 | import java.lang.annotation.ElementType; 5 | import java.lang.annotation.Retention; 6 | import java.lang.annotation.RetentionPolicy; 7 | import java.lang.annotation.Target; 8 | 9 | /** 10 | * Annotates a Prepper plugin class of which an instance is required to be confined to a single thread. 11 | */ 12 | 13 | @Documented 14 | @Retention(RetentionPolicy.RUNTIME) 15 | @Target({ElementType.TYPE}) 16 | public @interface SingleThread { 17 | } 18 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/test-index-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "mappings": { 4 | "date_detection": false, 5 | "dynamic_templates": [ 6 | { 7 | "strings_as_keyword": { 8 | "mapping": { 9 | "ignore_above": 1024, 10 | "type": "keyword" 11 | }, 12 | "match_mapping_type": "string" 13 | } 14 | } 15 | ], 16 | "_source": { 17 | "enabled": false 18 | }, 19 | "properties": { 20 | "name": { 21 | "ignore_above": 1024, 22 | "type": "keyword" 23 | } 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /data-prepper-benchmarks/mapdb-benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # MapDb Benchmarks 2 | 3 | This package uses JMH (https://openjdk.java.net/projects/code-tools/jmh/) to benchmark the MapDb Prepper State plugin. 4 | To use jmh benchmarking easily with gradle, this package uses a jmh gradle plugin (https://github.com/melix/jmh-gradle-plugin/) . 5 | Details on configuration and other options can be found there. 6 | 7 | To run the benchmarks from this directory, run the following command: 8 | 9 | ``` 10 | ../../gradlew jmh 11 | ``` 12 | 13 | To build an executable standalone jar of these benchmarks, run: 14 | 15 | ``` 16 | ../../gradlew jmhJar 17 | ``` 18 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/test-index-template-v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 2, 3 | "mappings": { 4 | "date_detection": false, 5 | "dynamic_templates": [ 6 | { 7 | "strings_as_keyword": { 8 | "mapping": { 9 | "ignore_above": 1024, 10 | "type": "keyword" 11 | }, 12 | "match_mapping_type": "string" 13 | } 14 | } 15 | ], 16 | "_source": { 17 | "enabled": false 18 | }, 19 | "properties": { 20 | "name": { 21 | "ignore_above": 1024, 22 | "type": "keyword" 23 | } 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/connected_pipeline_incorrect_root_source.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: #this plugin fails as it requires path attribute 5 | buffer: 6 | bounded_blocking: 7 | sink: 8 | - pipeline: 9 | name: "test-pipeline-2" 10 | test-pipeline-2: 11 | source: 12 | pipeline: 13 | name: "test-pipeline-1" 14 | sink: 15 | - pipeline: 16 | name: "test-pipeline-3" 17 | test-pipeline-3: 18 | source: 19 | pipeline: 20 | name: "test-pipeline-2" 21 | sink: 22 | - file: 23 | path: "/tmp/todelete.txt" -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/sink/SinkFactory.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.model.sink.Sink; 5 | import com.amazon.dataprepper.plugins.PluginFactory; 6 | import com.amazon.dataprepper.plugins.PluginRepository; 7 | 8 | @SuppressWarnings({"rawtypes"}) 9 | public class SinkFactory extends PluginFactory { 10 | 11 | public static Sink newSink(final PluginSetting pluginSetting) { 12 | return (Sink) newPlugin(pluginSetting, PluginRepository.getSinkClass(pluginSetting.getName())); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model; 2 | 3 | import static java.util.Objects.requireNonNull; 4 | 5 | public class Certificate { 6 | /** 7 | * The base64 PEM-encoded certificate. 8 | */ 9 | private String certificate; 10 | 11 | public Certificate(final String certificate) { 12 | this.certificate = requireNonNull(certificate, "certificate must not be null"); 13 | } 14 | 15 | public String getCertificate() { 16 | return certificate; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /docs/readme/logs.md: -------------------------------------------------------------------------------- 1 | # Logging 2 | Data Prepper uses [SLF4J](http://www.slf4j.org/) with a [Log4j 2 binding](http://logging.apache.org/log4j/2.x/log4j-slf4j-impl/). 3 | 4 | Default properties for Log4j 2 can be found in the log4j2.properties file in the *shared-config* directory. 5 | 6 | Users are able to override these logging settings by setting their own "log4j.configurationFile" system property (see [Log4j 2 configuration docs](https://logging.apache.org/log4j/2.x/manual/configuration.html)). 7 | 8 | Example: 9 | ``` 10 | java "-Dlog4j.configurationFile=config/custom-log4j2.properties" -jar data-prepper-core-$VERSION.jar pipelines.yaml data-prepper-config.yaml 11 | ``` 12 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/cyclic_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | pipeline: 5 | name: "test-pipeline-3" 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - pipeline: 10 | name: "test-pipeline-2" 11 | test-pipeline-2: 12 | source: 13 | pipeline: 14 | name: "test-pipeline-1" 15 | sink: 16 | - pipeline: 17 | name: "test-pipeline-3" 18 | test-pipeline-3: 19 | source: 20 | pipeline: 21 | name: "test-pipeline-2" 22 | sink: 23 | - pipeline: 24 | name: "test-pipeline-1" -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_multiple_pipeline_configuration.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | path: "/tmp/file-source.tmp" 6 | buffer: 7 | bounded_blocking: #to check non object nodes for plugins 8 | sink: 9 | - pipeline: 10 | name: "test-pipeline-2" 11 | test-pipeline-2: 12 | source: 13 | pipeline: 14 | name: "test-pipeline-1" 15 | sink: 16 | - pipeline: 17 | name: "test-pipeline-3" 18 | test-pipeline-3: 19 | source: 20 | pipeline: 21 | name: "test-pipeline-2" 22 | sink: 23 | - file: 24 | path: "/tmp/todelete.txt" -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/source/SourceFactory.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.source; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.plugins.PluginFactory; 5 | import com.amazon.dataprepper.plugins.PluginRepository; 6 | import com.amazon.dataprepper.model.source.Source; 7 | 8 | @SuppressWarnings({"rawtypes"}) 9 | public class SourceFactory extends PluginFactory { 10 | 11 | public static Source newSource(final PluginSetting pluginSetting) { 12 | return (Source) newPlugin(pluginSetting, PluginRepository.getSourceClass(pluginSetting.getName())); 13 | } 14 | } -------------------------------------------------------------------------------- /release/docker/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id "com.palantir.docker" version "0.25.0" 3 | } 4 | 5 | docker { 6 | name "${project.rootProject.name}:${project.version}" 7 | tag "${project.rootProject.name}", "${project.version}" 8 | files project(':data-prepper-core').jar.archivePath 9 | buildArgs(['JAR_FILE' : project(':data-prepper-core').jar.archiveName, 10 | 'CONFIG_FILEPATH': '/usr/share/data-prepper/data-prepper-config.yaml', 11 | 'PIPELINE_FILEPATH': '/usr/share/data-prepper/pipelines.yaml']) 12 | dockerfile file('Dockerfile') 13 | } 14 | 15 | dockerPrepare.dependsOn ':release:releasePrerequisites' 16 | dockerPush.dependsOn docker -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/buffer/BufferFactory.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.buffer; 2 | 3 | import com.amazon.dataprepper.model.buffer.Buffer; 4 | import com.amazon.dataprepper.model.configuration.PluginSetting; 5 | import com.amazon.dataprepper.plugins.PluginFactory; 6 | import com.amazon.dataprepper.plugins.PluginRepository; 7 | 8 | @SuppressWarnings({"rawtypes"}) 9 | public class BufferFactory extends PluginFactory { 10 | 11 | public static Buffer newBuffer(final PluginSetting pluginSetting) { 12 | return (Buffer) newPlugin(pluginSetting, PluginRepository.getBufferClass(pluginSetting.getName())); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_multiple_sinks.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | source: 3 | otel_trace_source: 4 | sink: 5 | - pipeline: 6 | name: "raw-pipeline" 7 | - pipeline: 8 | name: "service-map-pipeline" 9 | raw-pipeline: 10 | source: 11 | pipeline: 12 | name: "entry-pipeline" 13 | prepper: 14 | - string_converter: 15 | upper_case: true 16 | sink: 17 | - file: 18 | someProperty: "someValue" 19 | service-map-pipeline: 20 | source: 21 | pipeline: 22 | name: "entry-pipeline" 23 | prepper: 24 | - string_converter: 25 | upper_case: true 26 | sink: 27 | - file: 28 | someProperty: "someValue" -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/connected_pipeline_incorrect_child_pipeline.yml: -------------------------------------------------------------------------------- 1 | # this configuration file is solely for testing formatting 2 | test-pipeline-1: 3 | source: 4 | file: 5 | path: "/tmp/file-source.tmp" 6 | buffer: 7 | bounded_blocking: 8 | sink: 9 | - pipeline: 10 | name: "test-pipeline-2" 11 | test-pipeline-2: 12 | source: 13 | pipeline: 14 | name: "test-pipeline-1" 15 | sink: 16 | - pipeline: 17 | name: "test-pipeline-3" 18 | - file1: #this will fail file plugin creation. 19 | test-pipeline-3: 20 | source: 21 | pipeline: 22 | name: "test-pipeline-2" 23 | sink: 24 | - file: 25 | path: "/tmp/todelete.txt" -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/test-template-withshards.json: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "number_of_shards": 100, 4 | "number_of_replicas": 2 5 | }, 6 | "mappings": { 7 | "date_detection": false, 8 | "dynamic_templates": [ 9 | { 10 | "strings_as_keyword": { 11 | "mapping": { 12 | "ignore_above": 1024, 13 | "type": "keyword" 14 | }, 15 | "match_mapping_type": "string" 16 | } 17 | } 18 | ], 19 | "_source": { 20 | "enabled": false 21 | }, 22 | "properties": { 23 | "name": { 24 | "ignore_above": 1024, 25 | "type": "keyword" 26 | } 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /data-prepper-plugins/common/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | dependencies { 5 | api project(':data-prepper-api') 6 | implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" 7 | implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" 8 | implementation "org.reflections:reflections:0.9.12" 9 | testImplementation "junit:junit:4.13.2" 10 | testImplementation "commons-io:commons-io:2.11.0" 11 | } 12 | 13 | jacocoTestCoverageVerification { 14 | dependsOn jacocoTestReport 15 | violationRules { 16 | rule { //in addition to core projects rule 17 | limit { 18 | minimum = 0.90 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | source: 3 | otel_trace_source: 4 | ssl: false 5 | prepper: 6 | - peer_forwarder: 7 | discovery_mode: "static" 8 | static_endpoints: ["dataprepper1", "dataprepper2"] 9 | ssl: false 10 | sink: 11 | - pipeline: 12 | name: "service-map-pipeline" 13 | service-map-pipeline: 14 | source: 15 | pipeline: 16 | name: "entry-pipeline" 17 | prepper: 18 | - service_map_stateful: 19 | window_duration: 3 20 | sink: 21 | - elasticsearch: 22 | hosts: ["https://node-0.example.com:9200"] 23 | username: "admin" 24 | password: "admin" 25 | trace_analytics_service_map: true -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/health/HealthGrpcService.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.health; 2 | 3 | import io.grpc.health.v1.HealthCheckRequest; 4 | import io.grpc.health.v1.HealthCheckResponse; 5 | import io.grpc.health.v1.HealthGrpc; 6 | import io.grpc.stub.StreamObserver; 7 | 8 | public class HealthGrpcService extends HealthGrpc.HealthImplBase { 9 | 10 | @Override 11 | public void check(final HealthCheckRequest request, final StreamObserver responseObserver) { 12 | responseObserver.onNext( 13 | HealthCheckResponse.newBuilder().setStatus(HealthCheckResponse.ServingStatus.SERVING).build()); 14 | responseObserver.onCompleted(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/sink/Sink.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.sink; 2 | 3 | import com.amazon.dataprepper.model.record.Record; 4 | 5 | import java.util.Collection; 6 | 7 | /** 8 | * Data Prepper sink interface. Sink may publish records to a disk or a file or 9 | * to elasticsearch or other pipelines or external systems 10 | */ 11 | public interface Sink> { 12 | 13 | /** 14 | * outputs collection of records which extend {@link Record}. 15 | * 16 | * @param records the records to write to the sink. 17 | */ 18 | void output(Collection records); 19 | 20 | /** 21 | * Prepare sink for shutdown, by cleaning up resources and threads. 22 | */ 23 | void shutdown(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/README.md: -------------------------------------------------------------------------------- 1 | # Common Data Prepper plugins 2 | 3 | ## `string_coverter` 4 | 5 | A prepper plugin to generate new string records with upper or lower case conversion on the content of input records. 6 | 7 | - upper_case (boolean): convert to upper case if true; otherwise convert to lower case 8 | 9 | ## `file` (source) 10 | 11 | A source plugin to read input data from the specified file path. 12 | 13 | - path (String): absolute input data file path 14 | 15 | ## `file` (sink) 16 | 17 | A sink plugin to write output data to the specified file path. 18 | 19 | - path (String): absolute output file path 20 | 21 | ## `stdin` 22 | 23 | A source plugin to read input data from console. 24 | 25 | ## `stdout` 26 | 27 | A sink plugin to write output data to console. 28 | -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | source: 3 | otel_trace_source: 4 | ssl: false 5 | sink: 6 | - pipeline: 7 | name: "raw-pipeline" 8 | raw-pipeline: 9 | source: 10 | pipeline: 11 | name: "entry-pipeline" 12 | prepper: 13 | - otel_trace_raw_prepper: 14 | root_span_flush_delay: 1 # TODO: remove after 1.1 release 15 | trace_flush_interval: 5 16 | - otel_trace_group_prepper: 17 | hosts: [ "https://node-0.example.com:9200" ] 18 | username: "admin" 19 | password: "admin" 20 | sink: 21 | - elasticsearch: 22 | hosts: [ "https://node-0.example.com:9200" ] 23 | username: "admin" 24 | password: "admin" 25 | trace_analytics_raw: true -------------------------------------------------------------------------------- /shared-config/log4j2.properties: -------------------------------------------------------------------------------- 1 | status = error 2 | dest = err 3 | name = PropertiesConfig 4 | 5 | property.filename = log/data-prepper/data-prepper.log 6 | 7 | appender.console.type = Console 8 | appender.console.name = STDOUT 9 | appender.console.layout.type = PatternLayout 10 | appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n 11 | 12 | rootLogger.level = warn 13 | rootLogger.appenderRef.stdout.ref = STDOUT 14 | 15 | logger.core.name = com.amazon.dataprepper 16 | logger.core.level = info 17 | 18 | logger.pipeline.name = com.amazon.dataprepper.pipeline 19 | logger.pipeline.level = info 20 | 21 | logger.parser.name = com.amazon.dataprepper.parser 22 | logger.parser.level = info 23 | 24 | logger.plugins.name = com.amazon.dataprepper.plugins 25 | logger.plugins.level = info 26 | -------------------------------------------------------------------------------- /release/build-resources.gradle: -------------------------------------------------------------------------------- 1 | //preferably try to main the alphabetical order 2 | ext { 3 | architectures = [ 4 | linux: ['x64'], 5 | macos: ['x64'] 6 | ] 7 | jdkSources = [ 8 | linux_x64: 'https://download.java.net/java/GA/jdk15.0.1/51f4f36ad4ef43e39d0dfdbaf6549e32/9/GPL/openjdk-15.0.1_linux-x64_bin.tar.gz', 9 | linux_arm64: 'https://hg.openjdk.java.net/aarch64-port/jdk8/archive/tip.tar.gz', 10 | macos_x64: 'https://download.java.net/java/GA/jdk15.0.1/51f4f36ad4ef43e39d0dfdbaf6549e32/9/GPL/openjdk-15.0.1_osx-x64_bin.tar.gz' 11 | ] 12 | awsResources = [ 13 | default_region: 'us-east-1', 14 | default_profile: 'default', 15 | default_bucket: 'artifacts.dataprepper.amazon.com' 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/source/Source.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.source; 2 | 3 | import com.amazon.dataprepper.model.buffer.Buffer; 4 | import com.amazon.dataprepper.model.record.Record; 5 | 6 | /** 7 | * Data Prepper source interface. Source acts as receiver of the events that flow 8 | * through the transformation pipeline. 9 | */ 10 | public interface Source> { 11 | 12 | /** 13 | * Notifies the source to start writing the records into the buffer 14 | * 15 | * @param buffer Buffer to which the records will be queued or written to. 16 | */ 17 | void start(Buffer buffer); 18 | 19 | /** 20 | * Notifies the source to stop consuming/writing records into Buffer. 21 | */ 22 | void stop(); 23 | } 24 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/resources/valid_multiple_processors.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | source: 3 | otel_trace_source: 4 | sink: 5 | - pipeline: 6 | name: "raw-pipeline" 7 | - pipeline: 8 | name: "service-map-pipeline" 9 | raw-pipeline: 10 | source: 11 | pipeline: 12 | name: "entry-pipeline" 13 | prepper: 14 | - string_converter: 15 | upper_case: true 16 | - string_converter: 17 | upper_case: false 18 | sink: 19 | - file: 20 | someProperty: "someValue" 21 | service-map-pipeline: 22 | source: 23 | pipeline: 24 | name: "entry-pipeline" 25 | prepper: 26 | - string_converter: 27 | upper_case: true 28 | - string_converter: 29 | upper_case: false 30 | sink: 31 | - file: 32 | someProperty: "someValue" -------------------------------------------------------------------------------- /data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | source: 3 | otel_trace_source: 4 | ssl: false 5 | prepper: 6 | - peer_forwarder: 7 | discovery_mode: "static" 8 | static_endpoints: ["dataprepper1", "dataprepper2"] 9 | ssl: false 10 | sink: 11 | - pipeline: 12 | name: "raw-pipeline" 13 | raw-pipeline: 14 | source: 15 | pipeline: 16 | name: "entry-pipeline" 17 | prepper: 18 | - otel_trace_raw_prepper: 19 | root_span_flush_delay: 3 # TODO: remove after 1.1 release 20 | trace_flush_interval: 5 21 | sink: 22 | - elasticsearch: 23 | hosts: [ "https://node-0.example.com:9200" ] 24 | username: "admin" 25 | password: "admin" 26 | trace_analytics_raw: true -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | ext { 6 | es_version = System.getProperty("es.version", "7.10.2") 7 | } 8 | 9 | dependencies { 10 | implementation project(':data-prepper-api') 11 | implementation project(':data-prepper-plugins:elasticsearch') 12 | testImplementation project(':data-prepper-api').sourceSets.test.output 13 | implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" 14 | implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" 15 | implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" 16 | implementation "io.micrometer:micrometer-core:1.7.2" 17 | testImplementation "org.mockito:mockito-core:3.11.2" 18 | testImplementation "junit:junit:4.13.2" 19 | } -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/java/com/example/restservice/MetricsController.java: -------------------------------------------------------------------------------- 1 | package com.example.restservice; 2 | 3 | import org.springframework.web.bind.annotation.GetMapping; 4 | import org.springframework.web.bind.annotation.RequestParam; 5 | import org.springframework.web.bind.annotation.RestController; 6 | import java.util.concurrent.atomic.AtomicLong; 7 | 8 | @RestController 9 | public class MetricsController { 10 | 11 | private static final String template = "Hello, %s!"; 12 | private final AtomicLong counter = new AtomicLong(); 13 | 14 | @GetMapping("/metrics") 15 | public Metrics metrics(@RequestParam(value = "name", defaultValue = "Service-X") String name) { 16 | return new Metrics(counter.incrementAndGet(), String.format(template, name)); 17 | } 18 | } -------------------------------------------------------------------------------- /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Data Prepper Java CI with Gradle 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | matrix: 16 | java: [14] 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up JDK ${{ matrix.java }} 23 | uses: actions/setup-java@v1 24 | with: 25 | java-version: ${{ matrix.java }} 26 | - name: Grant execute permission for gradlew 27 | run: chmod +x gradlew 28 | - name: Build with Gradle 29 | run: ./gradlew build 30 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanSet.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.oteltrace.model; 2 | 3 | import com.google.common.collect.Sets; 4 | 5 | import java.util.Set; 6 | 7 | public class RawSpanSet { 8 | 9 | private final Set rawSpans; 10 | private final long timeSeen; 11 | 12 | public RawSpanSet() { 13 | this.rawSpans = Sets.newConcurrentHashSet(); 14 | this.timeSeen = System.currentTimeMillis(); 15 | } 16 | 17 | public Set getRawSpans() { 18 | return rawSpans; 19 | } 20 | 21 | public long getTimeSeen() { 22 | return timeSeen; 23 | } 24 | 25 | public void addRawSpan(final RawSpan rawSpan) { 26 | rawSpans.add(rawSpan); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProvider.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; 2 | 3 | import com.linecorp.armeria.client.Endpoint; 4 | import com.linecorp.armeria.common.util.Listenable; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * Provides a list other Data Prepper instance endpoints within the same cluster. 10 | * Coupling this to Armeria's Listenable interface for now to leverage existing implementers of it 11 | * (see DynamicEndpointGroup), though this can be redefined in the future if more Provider implementations are necessary. 12 | */ 13 | public interface PeerListProvider extends Listenable> { 14 | String PEER_ENDPOINTS = "peerEndpoints"; 15 | 16 | List getPeerList(); 17 | } 18 | -------------------------------------------------------------------------------- /research/zipkin-elastic-to-otel/README.md: -------------------------------------------------------------------------------- 1 | # Zipkin trace data from ODFE to Otel Trace Source 2 | 3 | This is a demo on reading zipkin trace analytics data from local opendistro-elasticsearch cluster to Otel Trace source. 4 | 5 | ## Prerequisite 6 | 7 | * ODFE docker: listening on https://localhost:9200 with `admin` as user and password. 8 | * Otel Trace Source: listening on http://127.0.0.1:21890. 9 | 10 | ## Run 11 | 12 | ``` 13 | ./gradlew :research:zipkin-elastic-to-otel:run --args YOUR_INDEX_PATTERN FIELD VALUE 14 | ``` 15 | 16 | Notice that `FIELD` and `VALUE` are optional arguments. 17 | 18 | * Test mode: with environment variable test set to `true`. A test Otel trace source will be launched listening on http://127.0.0.1:21890. 19 | 20 | ``` 21 | ./gradlew :research:zipkin-elastic-to-otel:run -Dtest=true --args YOUR_INDEX_PATTERN 22 | ``` 23 | -------------------------------------------------------------------------------- /examples/aws/jaeger-hotrod-on-ec2/README.md: -------------------------------------------------------------------------------- 1 | ## Jaeger Hot R.O.D. on EC2 2 | This directory contains a script to setup install and run the Jaeger Hot R.O.D. sample application 3 | using [Docker](https://docs.docker.com/) and [Docker Compose](https://docs.docker.com/compose/). 4 | 5 | The script downloads/installs Docker and Docker Compose, writes necessary application configuration files, then creates the following containers: 6 | 1. [Jaeger HotROD](https://github.com/jaegertracing/jaeger/tree/master/examples/hotrod) - the example application to generate trace data 7 | 2. [Jaeger Agent](https://www.jaegertracing.io/docs/1.21/architecture/#agent) - a network daemon which batches trace spans and sends to the collector 8 | 3. [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector) - a vendor-agnostic implementation on how to receive, process and export telemetry data 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: This project is no longer making changes. Please migrate to https://github.com/opensearch-project/data-prepper 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment (please complete the following information):** 27 | - OS: [e.g. Ubuntu 20.04 LTS] 28 | - Version [e.g. 22] 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICHTCCAYYCCQD4hqYeYDQZADANBgkqhkiG9w0BAQUFADBSMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCVFgxDzANBgNVBAcMBkF1c3RpbjEPMA0GA1UECgwGQW1hem9u 4 | MRQwEgYDVQQLDAtEYXRhcHJlcHBlcjAgFw0yMTA2MjUxOTIzMTBaGA8yMTIxMDYw 5 | MTE5MjMxMFowUjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlRYMQ8wDQYDVQQHDAZB 6 | dXN0aW4xDzANBgNVBAoMBkFtYXpvbjEUMBIGA1UECwwLRGF0YXByZXBwZXIwgZ8w 7 | DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKrb3YhdKbQ5PtLHall10iLZC9ZdDVrq 8 | HOvqVSM8NHlL8f82gJ3l0n9k7hYc5eKisutaS9eDTmJ+Dnn8xn/qPSKTIq9Wh+OZ 9 | O+e9YEEpI/G4F9KpGULgMyRg9sJK0GlZdEt9o5GJNJIJUkptJU5eiLuE0IV+jyJo 10 | Nvm8OE6EJPqxAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAjgnX5n/Tt7eo9uakIGAb 11 | uBhvYdR8JqKXqF9rjFJ/MIK7FdQSF/gCdjnvBhzLlZFK/Nb6MGKoSKm5Lcr75LgC 12 | FyhIwp3WlqQksiMFnOypYVY71vqDgj6UKdMaOBgthsYhngj8lC+wsVzWqQvkJ2Qg 13 | /GAIzJwiZfXiaevQHRk79qI= 14 | -----END CERTIFICATE----- 15 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'org.springframework.boot' version '2.3.1.RELEASE' 4 | id 'io.spring.dependency-management' version '1.0.9.RELEASE' 5 | } 6 | 7 | 8 | group = 'com.example.restservice' 9 | version = '0.0.1-SNAPSHOT' 10 | sourceCompatibility = '1.8' 11 | 12 | repositories { 13 | mavenCentral() 14 | } 15 | 16 | dependencies { 17 | implementation('org.springframework.boot:spring-boot-starter-web') 18 | testImplementation('org.springframework.boot:spring-boot-starter-test') 19 | implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.8.1' 20 | implementation 'org.elasticsearch.client:elasticsearch-rest-client:7.8.1' 21 | implementation 'org.elasticsearch:elasticsearch:7.8.1' 22 | } 23 | 24 | bootJar { 25 | mainClassName = 'com.example.restservice.RestServiceApplication' 26 | } 27 | 28 | -------------------------------------------------------------------------------- /examples/jaeger-hotrod/README.md: -------------------------------------------------------------------------------- 1 | ## Jaeger Hot Rod 2 | 3 | This demo will use the revered Jaeger Hotrod app. The purpose of this demo is to experience the Trace Analytics feature using an existing Jaeger based application. For this demo we use the Jager Hotrod App, Jaeger Agent, OpenTelemetry collector and Data Prepper. 4 | 5 | #### Required: 6 | * Docker - we recommend allowing Docker to use at least 4 GB of RAM in Preferences > Resources. 7 | 8 | #### Demo 9 | ``` 10 | docker-compose up -d --build 11 | ``` 12 | 13 | The above command will start the Jaeger Hotrod sample, Jaeger Agent, OpenTelemetry Collector, Data Prepper, ODFE and Kibana. Wait for few minutes for all the containers to come up, the DataPrepper will restart till elasticsearch becomes available. 14 | 15 | After successful start, 16 | 17 | * use the hot rod app at localhost:8080 18 | * check the traces in kibana trace analytics dashboards at localhost:5601/app/traceAnalytics#/dashboard 19 | 20 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/PluginType.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model; 2 | 3 | import com.amazon.dataprepper.model.buffer.Buffer; 4 | import com.amazon.dataprepper.model.prepper.Prepper; 5 | import com.amazon.dataprepper.model.sink.Sink; 6 | import com.amazon.dataprepper.model.source.Source; 7 | 8 | public enum PluginType { 9 | SOURCE("source", Source.class), 10 | BUFFER("buffer", Buffer.class), 11 | PREPPER("prepper", Prepper.class), 12 | SINK("sink", Sink.class); 13 | 14 | private final String pluginName; 15 | private final Class pluginClass; 16 | 17 | PluginType(final String pluginName, final Class pluginClass) { 18 | this.pluginName = pluginName; 19 | this.pluginClass = pluginClass; 20 | } 21 | 22 | public String pluginName() { 23 | return pluginName; 24 | } 25 | 26 | public Class pluginClass() { 27 | return pluginClass; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/dev/k8s/README.md: -------------------------------------------------------------------------------- 1 | # Example Kubernetes Deployment with Minikube 2 | 3 | This set of Kubernetes config files recreates the same sample app under the /examples directory, but within a minikube environment. 4 | * _data-prepper.yaml_ contains a deployment, service, headless service, and configmap required to run Data Prepper with Kubernetes 5 | * _sample-project-applications.yaml_ contains configuration required to setup the remaining elements of the sample project (Kibana, Elasticsearch, etc.) 6 | 7 | 1. Setup kubectl and minikube locally 8 | 1. https://kubernetes.io/docs/tasks/tools/install-kubectl/ 9 | 2. https://minikube.sigs.k8s.io/docs/start/ 10 | 2. `minikube start` 11 | 3. `./build_images_for_minikube.sh` to build required Docker images locally 12 | 4. `kubectl apply -f .` 13 | 5. To cleanup, `kubectl delete -f .` 14 | 15 | Once the k8s deployment is running, you can check the status via `minikube dashboard` and visiting the output URL in a browser. 16 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | /** 7 | * Execute entry into Data Prepper. 8 | */ 9 | public class DataPrepperExecute { 10 | private static final Logger LOG = LoggerFactory.getLogger(DataPrepperExecute.class); 11 | 12 | public static void main(String[] args) { 13 | java.security.Security.setProperty("networkaddress.cache.ttl", "60"); 14 | 15 | if(args.length > 1) { 16 | DataPrepper.configure(args[1]); 17 | } else { 18 | DataPrepper.configureWithDefaults(); 19 | } 20 | final DataPrepper dataPrepper = DataPrepper.getInstance(); 21 | if (args.length > 0) { 22 | dataPrepper.execute(args[0]); 23 | } else { 24 | LOG.error("Configuration file is required"); 25 | System.exit(1); 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Data Prepper Trace Analytics Raw Span End-to-end test with Gradle 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | matrix: 16 | java: [14] 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up JDK ${{ matrix.java }} 23 | uses: actions/setup-java@v1 24 | with: 25 | java-version: ${{ matrix.java }} 26 | - name: Grant execute permission for gradlew 27 | run: chmod +x gradlew 28 | - name: Run raw-span end-to-end tests with Gradle 29 | run: ./gradlew :data-prepper-core:rawSpanEndToEndTest -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/README.md: -------------------------------------------------------------------------------- 1 | # Analytics Service 2 | This directory contains a RESTful Spring Boot Application running on port 8087. 3 | 4 | The app contains two endpoints: 5 | - /logs 6 | - /metrics 7 | 8 | ##### Commands to build app and run the JAR: 9 | ``` 10 | $ ./gradlew clean build 11 | ``` 12 | ``` 13 | $ java -jar build/libs/analytics-service-0.0.1-SNAPSHOT.jar 14 | ``` 15 | 16 | ##### Running the app using Docker: 17 | The Dockerfile contains a DEPENDENCY param which unpacks the fat jar in build/dependency. This is for faster performance. 18 | ``` 19 | $ docker build -t analytics-service . 20 | ``` 21 | Run the image: 22 | ``` 23 | $ docker run -p 8087:8087 -t analytics-service 24 | ``` 25 | ##### Alternatively, from the root project directory, this service can be run alongside other services using docker-compose 26 | ``` 27 | $ cd analytics-service 28 | $ ./gradlew clean build 29 | ``` 30 | ``` 31 | $ docker-compose up --build 32 | ``` -------------------------------------------------------------------------------- /.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Data Prepper Trace Analytics Service Map End-to-end test with Gradle 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | matrix: 16 | java: [14] 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up JDK ${{ matrix.java }} 23 | uses: actions/setup-java@v1 24 | with: 25 | java-version: ${{ matrix.java }} 26 | - name: Grant execute permission for gradlew 27 | run: chmod +x gradlew 28 | - name: Run service-map end-to-end tests with Gradle 29 | run: ./gradlew :data-prepper-core:serviceMapEndToEndTest -------------------------------------------------------------------------------- /examples/config/example-pipelines.yaml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | delay: "100" 3 | source: 4 | otel_trace_source: 5 | ssl: false 6 | sink: 7 | - pipeline: 8 | name: "raw-pipeline" 9 | - pipeline: 10 | name: "service-map-pipeline" 11 | raw-pipeline: 12 | source: 13 | pipeline: 14 | name: "entry-pipeline" 15 | prepper: 16 | - otel_trace_raw_prepper: 17 | sink: 18 | - elasticsearch: 19 | hosts: [ "https://..es.amazonaws.com" ] 20 | aws_sigv4: true 21 | aws_region: "us-east-1" 22 | trace_analytics_raw: true 23 | service-map-pipeline: 24 | delay: "100" 25 | source: 26 | pipeline: 27 | name: "entry-pipeline" 28 | prepper: 29 | - service_map_stateful: 30 | sink: 31 | - elasticsearch: 32 | hosts: [ "https://..es.amazonaws.com" ] 33 | aws_sigv4: true 34 | aws_region: "us-east-1" 35 | trace_analytics_service_map: true 36 | -------------------------------------------------------------------------------- /shared-config/log4j.properties: -------------------------------------------------------------------------------- 1 | # Logging Level 2 | log4j.rootLogger=WARN, CONSOLE 3 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 4 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.CONSOLE.Threshold=INFO 6 | log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x \u2013 %m%n 7 | #- size rotation with log cleanup. 8 | log4j.appender.file=org.apache.log4j.RollingFileAppender 9 | log4j.appender.file.File=logs/Data-Prepper.log 10 | log4j.appender.file.MaxFileSize=10MB 11 | log4j.appender.file.MaxBackupIndex=5 12 | #setting threshold to INFO if the logger level is changed 13 | log4j.appender.file.Threshold=INFO 14 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 15 | log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %40C - %m%n 16 | 17 | #package specific levels 18 | log4j.logger.com.amazon.dataprepper.pipeline=INFO 19 | log4j.logger.com.amazon.dataprepper.parser=INFO 20 | log4j.logger.com.amazon.dataprepper.plugins=INFO 21 | -------------------------------------------------------------------------------- /examples/zipkin-sleuth-webmvc-example/README.md: -------------------------------------------------------------------------------- 1 | ## Zipkin sleuth-webmvc-example 2 | 3 | This demo will use the [sleuth-webmvc-example](https://github.com/openzipkin/sleuth-webmvc-example). 4 | 5 | #### Required: 6 | * Docker - If you run Docker locally, we recommend allowing Docker to use at least 4 GB of RAM in Preferences > Resources. 7 | 8 | #### Demo 9 | ``` 10 | docker-compose up -d --build 11 | ``` 12 | 13 | The above command will start the frontend, backend, Zipkin, OpenTelemetry Collector, Data Prepper, ODFE and Kibana. 14 | 15 | After successful start, one can send request to the frontend by opening http://localhost:8081/ or running curl command, e.g. 16 | 17 | ```$xslt 18 | curl -s localhost:8081 -H'user_name: JC' 19 | ``` 20 | 21 | Now check the traces in kibana which runs at localhost:5601. 22 | 23 | You will see two indicies, 24 | 25 | * otel-v1-apm-span - This index alias will store the raw traces. 26 | * otel-v1-apm-service-map - This index will store the relationship between services. -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICXAIBAAKBgQCq292IXSm0OT7Sx2pZddIi2QvWXQ1a6hzr6lUjPDR5S/H/NoCd 3 | 5dJ/ZO4WHOXiorLrWkvXg05ifg55/MZ/6j0ikyKvVofjmTvnvWBBKSPxuBfSqRlC 4 | 4DMkYPbCStBpWXRLfaORiTSSCVJKbSVOXoi7hNCFfo8iaDb5vDhOhCT6sQIDAQAB 5 | AoGANrrhFqpJDpr7vcb1ER0Fp/YArbT27zVo+EUC6puBb41dQlQyFOImcHpjLaAq 6 | H1PgnjU5cBp2hGQ+vOK0rwrYc/HNl6vfh6N3NbDptMiuoBafRJA9JzYourAM09BU 7 | zmXyr61Yn3KHzx1PRwWe37icX93oXP3P0qHb3dI1ZF4jG0ECQQDU5N/a7ogoz2zn 8 | ZssD6FvUOUQDsdBWdXmhUvg+YdZrV44e4xk+FVzwEONoRktEYKz9MFXlsgNHr445 9 | KRguHWcJAkEAzXQkwOkN8WID1wrwoobUIMbZSGAZzofwkKXgTTnllnT1qOQXuRbS 10 | aCMejFEymBBef4aXP6N4+va2FKW/MF34aQJAO2oMl1sOoOUSrZngepy0VAwPUUCk 11 | thxe74jqQu6nGpn6zd/vQYZQw6bS8Fz90H1yic6dilcd1znFZWp0lxoZkQJBALeI 12 | xoBycRsuFQIYasi1q3AwUtBd0Q/3zkZZeBtk2hzjFMUwJaUZpxKSNOrialD/ZnuD 13 | jz+xWBTRKe0d98JMX+kCQCmsJEj/HYQAC1GamZ7JQWogRSRF2KTgTWRaDXDxy0d4 14 | yUQgwHB+HZLFcbi1JEK6eIixCsX8iifrrkteh+1npJ0= 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gradle:6.5.0-jdk8 as cache 2 | RUN mkdir -p /home/gradle/cache_home 3 | ENV GRADLE_USER_HOME /home/gradle/cache_home 4 | COPY build.gradle /home/gradle/src/ 5 | WORKDIR /home/gradle/src 6 | RUN gradle clean build --daemon 7 | 8 | FROM gradle:6.5.0-jdk8 AS build 9 | COPY --from=cache /home/gradle/cache_home /home/gradle/.gradle 10 | COPY . /home/gradle/src/ 11 | WORKDIR /home/gradle/src 12 | RUN gradle bootJar --daemon 13 | 14 | RUN wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.0.1/opentelemetry-javaagent-all.jar 15 | 16 | FROM openjdk:8-jdk-alpine 17 | EXPOSE 8087 18 | RUN mkdir /app 19 | COPY --from=build /home/gradle/src/build/libs/*.jar /app/spring-boot-application.jar 20 | COPY --from=build /home/gradle/src/opentelemetry-javaagent-all.jar /app/opentelemetry-javaagent-all.jar 21 | ENTRYPOINT ["java","-javaagent:/app/opentelemetry-javaagent-all.jar","-jar","/app/spring-boot-application.jar"] -------------------------------------------------------------------------------- /.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Data Prepper Trace Analytics Raw Span Compatibility End-to-end test with Gradle 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | matrix: 16 | java: [14] 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up JDK ${{ matrix.java }} 23 | uses: actions/setup-java@v1 24 | with: 25 | java-version: ${{ matrix.java }} 26 | - name: Grant execute permission for gradlew 27 | run: chmod +x gradlew 28 | - name: Run raw-span compatibility end-to-end tests with Gradle 29 | run: ./gradlew :data-prepper-core:rawSpanCompatibilityEndToEndTest 30 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/record/Record.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.record; 2 | 3 | /** 4 | * Data Prepper record - represents the fundamental data unit of TI, the idea is to encapsulate different 5 | * types of data we will be supporting in TI. 6 | *

7 | * TODO: The current implementation focuses on proving the bare bones for which this class only need to 8 | * TODO: support sample test cases. 9 | */ 10 | public class Record { 11 | private final T data; 12 | private final RecordMetadata metadata; 13 | 14 | public Record(final T data) { 15 | this.data = data; 16 | metadata = RecordMetadata.defaultMetadata(); 17 | } 18 | 19 | public Record(final T data, final RecordMetadata metadata) { 20 | this.data = data; 21 | this.metadata = metadata; 22 | } 23 | 24 | public T getData() { 25 | return data; 26 | } 27 | 28 | public RecordMetadata getMetadata() { 29 | return metadata; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/ConstructorLessComponent.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins; 2 | 3 | import com.amazon.dataprepper.model.PluginType; 4 | import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; 5 | import com.amazon.dataprepper.model.buffer.Buffer; 6 | import com.amazon.dataprepper.model.record.Record; 7 | import com.amazon.dataprepper.model.source.Source; 8 | 9 | import java.util.concurrent.TimeoutException; 10 | 11 | @DataPrepperPlugin(name = "junit-test", type = PluginType.SOURCE) 12 | public class ConstructorLessComponent implements Source> { 13 | 14 | @Override 15 | public void start(Buffer> buffer) { 16 | try { 17 | buffer.write(new Record<>("Junit Testing"), 1_000); 18 | } catch (TimeoutException ex) { 19 | throw new RuntimeException("Timed out writing to buffer"); 20 | } 21 | } 22 | 23 | @Override 24 | public void stop() { 25 | //no op 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /release/release-notes/odfe-data-prepper.release-notes-0.7.0.md: -------------------------------------------------------------------------------- 1 | # 2020-12-10 Version 0.7.0 (alpha) 2 | 3 | This is the first official release of Data Prepper. 4 | 5 | Data Prepper is a server side data collector with abilities to filter, enrich, transform, normalize and aggregate data for downstream analytics and visualization. The broader vision for Data Prepper is to enable an end-to-end data analysis life cycle from gathering raw logs to facilitating sophisticated and actionable interactive ad-hoc analyses on the data. 6 | In the first release of Data Prepper, it only supports plugins that are required for Trace Analytics feature. More details check the documentations below. 7 | 8 | ### Documentation 9 | * [Overview](https://github.com/opendistro-for-elasticsearch/Data-Prepper/blob/e51238b2cea55bba138466f146fc19622331e59c/docs/readme/overview.md): Overview of Data Prepper 10 | * [Trace Overview](https://github.com/opendistro-for-elasticsearch/Data-Prepper/blob/e51238b2cea55bba138466f146fc19622331e59c/docs/readme/trace_overview.md): Trace overview 11 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.certificate.model; 2 | 3 | import static java.util.Objects.requireNonNull; 4 | 5 | public class Certificate { 6 | /** 7 | * The base64 PEM-encoded certificate. 8 | */ 9 | private String certificate; 10 | 11 | /** 12 | * The decrypted private key associated with the public key in the certificate. The key is output in PKCS #8 format 13 | * and is base64 PEM-encoded. 14 | */ 15 | private String privateKey; 16 | 17 | public Certificate(final String certificate, final String privateKey) { 18 | this.certificate = requireNonNull(certificate, "certificate must not be null"); 19 | this.privateKey = requireNonNull(privateKey, "privateKey must not be null"); 20 | } 21 | 22 | public String getCertificate() { 23 | return certificate; 24 | } 25 | 26 | public String getPrivateKey() { 27 | return privateKey; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/trace_analytics_no_ssl.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | delay: "100" 3 | source: 4 | otel_trace_source: 5 | ssl: false 6 | sink: 7 | - pipeline: 8 | name: "raw-pipeline" 9 | - pipeline: 10 | name: "service-map-pipeline" 11 | raw-pipeline: 12 | source: 13 | pipeline: 14 | name: "entry-pipeline" 15 | prepper: 16 | - otel_trace_raw_prepper: 17 | sink: 18 | - elasticsearch: 19 | hosts: [ "https://node-0.example.com:9200" ] 20 | cert: "/usr/share/data-prepper/root-ca.pem" 21 | username: "admin" 22 | password: "admin" 23 | trace_analytics_raw: true 24 | service-map-pipeline: 25 | delay: "100" 26 | source: 27 | pipeline: 28 | name: "entry-pipeline" 29 | prepper: 30 | - service_map_stateful: 31 | sink: 32 | - elasticsearch: 33 | hosts: ["https://node-0.example.com:9200"] 34 | cert: "/usr/share/data-prepper/root-ca.pem" 35 | username: "admin" 36 | password: "admin" 37 | trace_analytics_service_map: true 38 | -------------------------------------------------------------------------------- /examples/demo/demo-data-prepper.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICqjCCAZICCQDBYJxuuzsbczANBgkqhkiG9w0BAQsFADAXMRUwEwYDVQQDDAxk 3 | YXRhLXByZXBwZXIwHhcNMjAxMTMwMTc0NDM4WhcNMzAxMTI4MTc0NDM4WjAXMRUw 4 | EwYDVQQDDAxkYXRhLXByZXBwZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK 5 | AoIBAQDBuSTI3oz9ZJRiVQCdTfbQlG9TWCXwuIjjJcVwvgVjhIuj5VLEOWUpC8nr 6 | be4/3ttGLuqOBViLlw2aelN7ZANgFy33ONF7kDh4xFMp4rZdFKxCOsmUpbSD7hRf 7 | 8mVRFv30bx9//bVjLP8qi2QKuHc8WOevMsdNENFI6BgiI52EoVPyYJIIMaE2ripG 8 | qkQVn04Tk+rWz3zQ+NyOcExow98+pxMIjAanMUELTpMjQCWq2HtV2nnqcfqqNCae 9 | ig7NSDRvQtPL84Fjy7y4B1pS558hBmfOyZcGHo/gxLxQmvb1UKy4bcL7Kd8H3aNr 10 | LZ8zsr+he3cM9a7SZbiilALdcv0NAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAKvw 11 | fLiflX0Wd54ExsQvLvwZbrVU0z110h2N+E7SXt9K4NhHsANsX2mWxbqU/0YMYQpo 12 | fPO0XJSqmo28YD/eyKEazJg7tFISEqQ5gfWZM8BkfHcjbHMixtRDm6SpNzfeQT97 13 | C0zZo5kAD4d0Y3FwVjQ5FkvlmM6rCaJ0NyvZBUJseCsWpZsV2vF0QbMQcQ0lxjqY 14 | Vxs8Cnwp9bfF7ZP3i5KD5JxtVm80g6MIQdoYDQqtMjhZ8rCg5e1S6DbPnLRlmXkz 15 | 4cFfD4V/7+ldSpEt38QBtCFWalfem+4dtbEX8FoTciNZfCkeUotpO2jPS1IzzghA 16 | yy6bLJd2MHl8AnGRdQg= 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/README.md: -------------------------------------------------------------------------------- 1 | # Dev Trace Analytics Sample App 2 | This directory contains the same sample app under the /examples directory, except with the following changes: 3 | 1. Data Prepper is built using the code in this repository, instead of a pulled image from DockerHub 4 | 2. A container for dnsmasq is included for network tests 5 | 3. Prometheus and Grafana containers for metrics visualization 6 | 1. Prometheus can be accessed at http://localhost:9090/ 7 | 2. Grafana can be accessed at http://localhost:3000/. [Add a Prometheus data source](https://prometheus.io/docs/visualization/grafana/) 8 | with one of the following HTTP config: 9 | - URL: `prometheus:9090`; Access: `Server (default)` 10 | - URL: `http://localhost:9090`; Access: `Browser` 11 | Then checkout the metrics graph in `Explore` of the sidebar such as the following: 12 | 13 | ![grafana_metrics_sample](images/grafana_metrics_sample.png) 14 | 15 |
16 | 17 | Start the project with with `docker-compose up -d --build` (--build is optional) 18 | 19 | 20 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/java/com/example/restservice/Logging.java: -------------------------------------------------------------------------------- 1 | package com.example.restservice; 2 | 3 | public class Logging { 4 | 5 | private String service; 6 | private String message; 7 | 8 | public Logging(String service, String message) { 9 | this.service = service; 10 | this.message = message; 11 | } 12 | 13 | public String getService() { 14 | return service; 15 | } 16 | 17 | public String getMessage() { 18 | return message; 19 | } 20 | 21 | public void setService(String service) { 22 | this.service = service; 23 | } 24 | 25 | public void setMessage(String message) { 26 | this.message = message; 27 | } 28 | 29 | @Override 30 | public String toString() { 31 | final StringBuilder sb = new StringBuilder("Log{"); 32 | sb.append("service='").append(service).append('\''); 33 | sb.append(", message='").append(message).append('\''); 34 | sb.append('}'); 35 | return sb.toString(); 36 | } 37 | } -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/sink/StdOutSinkTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.model.record.Record; 5 | import org.junit.Test; 6 | 7 | import java.util.Arrays; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | 11 | public class StdOutSinkTests { 12 | private static String PLUGIN_NAME = "stdout"; 13 | 14 | private final String TEST_DATA_1 = "data_prepper"; 15 | private final String TEST_DATA_2 = "stdout_sink"; 16 | private final Record TEST_RECORD_1 = new Record<>(TEST_DATA_1); 17 | private final Record TEST_RECORD_2 = new Record<>(TEST_DATA_2); 18 | private final List> TEST_RECORDS = Arrays.asList(TEST_RECORD_1, TEST_RECORD_2); 19 | 20 | @Test 21 | public void testSimple() { 22 | final StdOutSink stdOutSink = new StdOutSink(new PluginSetting(PLUGIN_NAME, new HashMap<>())); 23 | stdOutSink.output(TEST_RECORDS); 24 | stdOutSink.shutdown(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key: -------------------------------------------------------------------------------- 1 | -----BEGIN ENCRYPTED PRIVATE KEY----- 2 | MIICojAcBgoqhkiG9w0BDAEDMA4ECAd2FKZw2oGwAgIIAASCAoDTgiaXkazaotc7 3 | SxQK3bX34sEvdkLXg/O4ZpHTb0f4gLPxNhDe7ZPKrAS2TdywpSHT0189MVl+PIvw 4 | 4YQDaGVHL1SM5ukJu+PQkfQAMigdCJ+bUsG6hkrUDC74qYhHZHj1yVGavL6I4KHT 5 | Ixh9IV2GMRS4m6HGJ17nYsdiTFFNdK++WcTMxbVWv3SNdKGZG79T32pjMxuIUPWr 6 | 3dB+ZXM+FSqwlBLZxPvvjlP6ETw7QXrlBHcQh1tHSh10bM+q0c5CktZoXLwpc+gv 7 | ZsGXzpVjqFrAw4Vw0ikJl1mUCoGOtqqP0P6QWwbIJZBxNoO0MvWcXW+U3AGNFFze 8 | nMR8UTXdga2l1Lx7pokQkWUpp48SDRjDx/RdZTRXCgtRcKuBcm0x2lxNILdwOzjJ 9 | 5GlKMvvc2OXXTnYqSCTqdfbuR3XBYmWgFki92D6JnVIYq+QJr5qj8IJDJ7mADQ1i 10 | Za6PEJnrT641fLeSKRq7QiTydMQ3JXa9DFqUPwdZPPHLr/hC19sWHrq7gxvhkcLI 11 | wrTtTIi8/iV4IVaiHk7YU83IM6sGExabQ3BRXcHMr+7i1vVxtEsFNC6ycTfJ8gpJ 12 | YsnpXUQe912l5sk7GRSP1InNRF7kzMD0QeOAQ0UVfmsbHOPSXvD7fXkJWIb6N+zW 13 | qCQYRmBwc7Bz2KZein5MLsMeNz2AWj/DcA2fMC+4+QtI0nF5BFtaR0V0npWhsbPu 14 | 3rj+AXipnvVhDIkfl8495j7ybCBj7HAZk8Ux8GmiZ3PGFO1C7XCQaLPWJ4Aw4Kb3 15 | EUqtVtpbwsCov5PDmMDXgz8qOxWMdQsP/dPF1HnVAg7SoFG9xA4nHdZ2WAFZwYtf 16 | rRxEd7br 17 | -----END ENCRYPTED PRIVATE KEY----- 18 | -------------------------------------------------------------------------------- /examples/trace_analytics.yml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | delay: "100" 3 | source: 4 | otel_trace_source: 5 | ssl: true 6 | sslKeyCertChainFile: "/app/demo-data-prepper.crt" 7 | sslKeyFile: "/app/demo-data-prepper.key" 8 | sink: 9 | - pipeline: 10 | name: "raw-pipeline" 11 | - pipeline: 12 | name: "service-map-pipeline" 13 | raw-pipeline: 14 | source: 15 | pipeline: 16 | name: "entry-pipeline" 17 | prepper: 18 | - otel_trace_raw_prepper: 19 | sink: 20 | - elasticsearch: 21 | hosts: [ "https://node-0.example.com:9200" ] 22 | cert: "/app/root-ca.pem" 23 | username: "admin" 24 | password: "admin" 25 | trace_analytics_raw: true 26 | service-map-pipeline: 27 | delay: "100" 28 | source: 29 | pipeline: 30 | name: "entry-pipeline" 31 | prepper: 32 | - service_map_stateful: 33 | sink: 34 | - elasticsearch: 35 | hosts: ["https://node-0.example.com:9200"] 36 | cert: "/app/root-ca.pem" 37 | username: "admin" 38 | password: "admin" 39 | trace_analytics_service_map: true 40 | -------------------------------------------------------------------------------- /data-prepper-plugins/service-map-stateful/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.amazon' 6 | version '0.1-beta' 7 | 8 | sourceCompatibility = 1.8 9 | 10 | repositories { 11 | mavenCentral() 12 | } 13 | 14 | dependencies { 15 | implementation project(':data-prepper-api') 16 | implementation project(':data-prepper-plugins:common') 17 | implementation project(':data-prepper-plugins:mapdb-prepper-state') 18 | testImplementation project(':data-prepper-api').sourceSets.test.output 19 | implementation "io.micrometer:micrometer-core:1.7.2" 20 | implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" 21 | implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" 22 | testImplementation "org.hamcrest:hamcrest:2.2" 23 | testImplementation "org.mockito:mockito-inline:3.11.2" 24 | } 25 | 26 | jacocoTestCoverageVerification { 27 | dependsOn jacocoTestReport 28 | violationRules { 29 | rule { //in addition to core projects rule 30 | limit { 31 | minimum = 0.9 32 | } 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'opendistroforelasticsearch-data-prepper' 2 | 3 | if(startParameter.getProjectProperties().containsKey("release")){ 4 | include 'release' 5 | include 'release:docker' 6 | include 'release:archives' 7 | include 'release:archives:linux' 8 | include 'release:archives:macos' 9 | } 10 | include 'data-prepper-api' 11 | include 'data-prepper-plugins' 12 | include 'data-prepper-core' 13 | include 'data-prepper-plugins:common' 14 | include 'data-prepper-plugins:elasticsearch' 15 | include 'data-prepper-plugins:service-map-stateful' 16 | include 'data-prepper-plugins:mapdb-prepper-state' 17 | include 'data-prepper-benchmarks' 18 | include 'data-prepper-benchmarks:mapdb-benchmarks' 19 | include 'research' 20 | include 'research:zipkin-elastic-to-otel' 21 | include 'data-prepper-benchmarks:service-map-stateful-benchmarks' 22 | include 'data-prepper-plugins:otel-trace-raw-prepper' 23 | include 'data-prepper-plugins:otel-trace-group-prepper' 24 | include 'data-prepper-plugins:otel-trace-source' 25 | include 'data-prepper-plugins:peer-forwarder' 26 | include 'data-prepper-plugins:blocking-buffer' 27 | 28 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/BackOffUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink.elasticsearch; 2 | 3 | import org.elasticsearch.common.unit.TimeValue; 4 | 5 | import java.util.Iterator; 6 | 7 | public final class BackOffUtils { 8 | private final Iterator iterator; 9 | 10 | private long currTime = 0; 11 | 12 | private boolean firstAttempt = true; 13 | 14 | public BackOffUtils(final Iterator iterator) { 15 | this.iterator = iterator; 16 | } 17 | 18 | public boolean hasNext() { 19 | return firstAttempt || iterator.hasNext(); 20 | } 21 | 22 | public boolean next() throws InterruptedException { 23 | if (firstAttempt) { 24 | firstAttempt = false; 25 | return true; 26 | } 27 | if (!iterator.hasNext()) { 28 | return false; 29 | } else { 30 | final long nextTime = iterator.next().getMillis(); 31 | Thread.sleep(nextTime - currTime); 32 | currTime = nextTime; 33 | return true; 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /data-prepper-plugins/mapdb-prepper-state/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.amazon' 6 | version '0.1-beta' 7 | 8 | sourceCompatibility = 1.8 9 | 10 | repositories { 11 | mavenCentral() 12 | } 13 | 14 | dependencies { 15 | implementation project(':data-prepper-api') 16 | implementation project(':data-prepper-plugins:common') 17 | implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' 18 | implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.21' 19 | implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.30' 20 | 21 | testImplementation project(':data-prepper-plugins:common').sourceSets.test.output 22 | testImplementation group: 'junit', name: 'junit', version: '4.13.2' 23 | testImplementation "org.hamcrest:hamcrest:2.2" 24 | } 25 | 26 | jacocoTestCoverageVerification { 27 | dependsOn jacocoTestReport 28 | violationRules { 29 | rule { //in addition to core projects rule 30 | limit { 31 | minimum = 0.95 //Increase this to 0.90 32 | } 33 | } 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/sink/TestSink.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink; 2 | 3 | 4 | import com.amazon.dataprepper.model.PluginType; 5 | import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; 6 | import com.amazon.dataprepper.model.record.Record; 7 | import com.amazon.dataprepper.model.sink.Sink; 8 | 9 | import java.util.ArrayList; 10 | import java.util.Collection; 11 | import java.util.List; 12 | import java.util.stream.Collectors; 13 | 14 | @DataPrepperPlugin(name = "test-sink", type = PluginType.SINK) 15 | public class TestSink implements Sink> { 16 | private final List> collectedRecords; 17 | 18 | public TestSink() { 19 | collectedRecords = new ArrayList<>(); 20 | } 21 | 22 | @Override 23 | public void output(Collection> records) { 24 | records.stream().collect(Collectors.toCollection(() -> collectedRecords)); 25 | } 26 | 27 | @Override 28 | public void shutdown() { 29 | 30 | } 31 | 32 | public List> getCollectedRecords() { 33 | return collectedRecords; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/raw-span-error.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId":"RfU2NsEWJtR8s5PJlZDlZA==", 3 | "spanId":"eupYrPoV7wM=", 4 | "parentSpanId":"1d8vLP9tjt4=", 5 | "name":"mysql.APM", 6 | "kind":"CLIENT", 7 | "status":{}, 8 | "startTime":"2020-09-01T22:01:55.103171300Z", 9 | "endTime":"2020-09-01T22:01:55.113684900Z", 10 | "span.attributes.component":"mysql", 11 | "span.attributes.db.type":"sql", 12 | "span.attributes.db.instance":"APM", 13 | "span.attributes.db.statement":"INSERT INTO Inventory_Items (ItemId, TotalQty) VALUES (%(ItemId)s, %(Qty)s) ON DUPLICATE KEY UPDATE TotalQty = TotalQty + %(Qty)s", 14 | "span.attributes.db.statement.parameters":"{'ItemId': 'banana', 'Qty': '6'}", 15 | "span.attributes.db.user":"root", 16 | "span.attributes.net.peer.name":"mysql", 17 | "span.attributes.net.peer.port":3306, 18 | "resource.attributes.service.name":"database", 19 | "resource.attributes.service.instance.id":"140683459769616", 20 | "resource.attributes.telemetry.sdk.name":"opentelemetry", 21 | "resource.attributes.telemetry.sdk.language":"python", 22 | "resource.attributes.telemetry.sdk.version":"0.12b0", 23 | "resource.attributes.host.hostname":"07643bc19331" 24 | } -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; 2 | 3 | import com.amazon.dataprepper.metrics.PluginMetrics; 4 | import com.amazon.dataprepper.model.configuration.PluginSetting; 5 | import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; 6 | 7 | import java.util.Objects; 8 | 9 | public class PeerListProviderFactory { 10 | 11 | public PeerListProvider createProvider(final PluginSetting pluginSetting) { 12 | Objects.requireNonNull(pluginSetting); 13 | 14 | final String discoveryModeString = pluginSetting.getStringOrDefault(PeerForwarderConfig.DISCOVERY_MODE, null); 15 | Objects.requireNonNull(discoveryModeString, String.format("Missing '%s' configuration value", PeerForwarderConfig.DISCOVERY_MODE)); 16 | 17 | final DiscoveryMode discoveryMode = DiscoveryMode.valueOf(discoveryModeString.toUpperCase()); 18 | 19 | final PluginMetrics pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); 20 | 21 | return discoveryMode.create(pluginSetting, pluginMetrics); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /deployment-template/eks/ingress.yaml: -------------------------------------------------------------------------------- 1 | # Controller docs: https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/installation/ 2 | apiVersion: extensions/v1beta1 3 | kind: Ingress 4 | metadata: 5 | namespace: default 6 | name: data-prepper-alb 7 | labels: 8 | app: data-prepper 9 | annotations: 10 | # Annotation docs: https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/annotations/ 11 | alb.ingress.kubernetes.io/backend-protocol-version: GRPC 12 | alb.ingress.kubernetes.io/backend-protocol: HTTP 13 | alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:xxxxx:certificate/xxxxxxx # Replace this 14 | alb.ingress.kubernetes.io/healthcheck-path: /grpc.health.v1.Health/Check 15 | alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":21890}]' 16 | alb.ingress.kubernetes.io/scheme: internet-facing 17 | alb.ingress.kubernetes.io/success-codes: '0' 18 | alb.ingress.kubernetes.io/target-type: ip 19 | kubernetes.io/ingress.class: alb 20 | spec: 21 | rules: 22 | - http: 23 | paths: 24 | - path: /* 25 | backend: 26 | serviceName: data-prepper-headless 27 | servicePort: 21890 28 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/raw-span-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", 3 | "spanId":"mnO/qUT5ye4=", 4 | "parentSpanId": null, 5 | "name":"io.opentelemetry.auto.servlet-3.0", 6 | "kind":"CLIENT", 7 | "status": { 8 | "code": 0, 9 | "message": "Success" 10 | }, 11 | "serviceName": "Customer service", 12 | "startTime":"2020-08-20T05:40:46.041011600Z", 13 | "endTime":"2020-08-20T05:40:46.089556800Z", 14 | "durationInNanos": 48545200, 15 | "span.attributes.http@status_code":200, 16 | "span.attributes.net@peer@port":41168, 17 | "span.attributes.servlet@path":"/logs", 18 | "span.attributes.http@response_content_length":7, 19 | "span.attributes.http@user_agent":"curl/7.54.0", 20 | "span.attributes.http@flavor":"HTTP/1.1", 21 | "span.attributes.servlet@context":"", 22 | "span.attributes.http@url":"http://0.0.0.0:8087/logs", 23 | "span.attributes.net@peer@ip":"172.29.0.1", 24 | "span.attributes.http@method":"POST", 25 | "span.attributes.http@client_ip":"172.29.0.1", 26 | "resource.attributes@telemetry@sdk@language":"java", 27 | "resource.attributes@telemetry@sdk@name":"opentelemetry", 28 | "resource.attributes@telemetry@sdk@version":"0.8.0-SNAPSHOT" 29 | } -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/odfe_security.md: -------------------------------------------------------------------------------- 1 | ## Open Distro For Elasticsearch 2 | 3 | 4 | The Elasticsearch sink can send trace data to opendistro-for-elasticsearch (ODFE) cluster by administrative credentials as follows: 5 | 6 | ``` 7 | sink: 8 | - elasticsearch: 9 | ... 10 | username: "admin" 11 | password: "admin" 12 | ``` 13 | 14 | or by using user credential assigned with a role that has the below required permissions. 15 | 16 | ### Cluster permissions 17 | 18 | - `cluster_all` 19 | - `indices:admin/template/get` 20 | - `indices:admin/template/put` 21 | 22 | Note that `indices:admin/template/*` need to be in cluster permissions. 23 | 24 | ### Index permissions 25 | 26 | - `Index`: `otel-v1*`; `Index permissions`: `indices_all` 27 | - `Index`: `.opendistro-ism-config`; `Index permissions`: `indices_all` 28 | - `Index`: `*`; `Index permission`: `manage_aliases` 29 | 30 | `Field level security` and `Anonymization` should be left with default values. 31 | 32 | --------------- 33 | 34 | With administrative privilege, one can create an internal user, a role and map the user to the role by following the ODFE [instructions](https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles/). 35 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/raw-span-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", 3 | "spanId":"aupYrPoV7wM=", 4 | "parentSpanId": "mnO/qUT5ye4=", 5 | "name":"io.opentelemetry.auto.servlet-3.0", 6 | "kind":"SERVER", 7 | "status": { 8 | "code": 6, 9 | "message": "Already exists" 10 | }, 11 | "serviceName": "analytics-service", 12 | "startTime":"2020-08-20T05:40:46.042011600Z", 13 | "endTime":"2020-08-20T05:40:46.088556800Z", 14 | "durationInNanos": 46545200, 15 | "span.attributes.http@status_code":200, 16 | "span.attributes.net@peer@port":41168, 17 | "span.attributes.servlet@path":"/logs", 18 | "span.attributes.http@response_content_length":7, 19 | "span.attributes.http@user_agent":"curl/7.54.0", 20 | "span.attributes.http@flavor":"HTTP/1.1", 21 | "span.attributes.servlet@context":"", 22 | "span.attributes.http@url":"http://0.0.0.0:8087/logs", 23 | "span.attributes.net@peer@ip":"172.29.0.1", 24 | "span.attributes.http@method":"POST", 25 | "span.attributes.http@client_ip":"172.29.0.1", 26 | "resource.attributes@telemetry@sdk@language":"java", 27 | "resource.attributes@telemetry@sdk@name":"opentelemetry", 28 | "resource.attributes@telemetry@sdk@version":"0.8.0-SNAPSHOT" 29 | } -------------------------------------------------------------------------------- /release/docker/README.md: -------------------------------------------------------------------------------- 1 | ## Docker Build for Data Prepper 2 | 3 | ### Running a docker build 4 | 5 | To build an image, check out the corresponding branch for the version and follow below steps 6 | 7 | * From root project, run 8 | 9 | ``` 10 | ./gradlew clean :release:docker:docker -Prelease 11 | ``` 12 | or 13 | 14 | * if running from the current project 15 | 16 | ``` 17 | gradle docker -Prelease 18 | ``` 19 | 20 | * The image will be built and will be available for running using docker 21 | 22 | ### Running built docker image 23 | 24 | The built image will be available locally and can be used as any other docker image. 25 | The root project has some usage examples for reference included in `examples` directory. 26 | Below is an example command which can also be used to run the built image. 27 | 28 | ``` 29 | docker run \ 30 | --name data-prepper-test \ 31 | --expose 21890 \ 32 | -v /workplace/github/simple-ingest-transformation-utility-pipeline/examples/config/example-pipelines.yaml:/usr/share/data-prepper/pipelines.yaml \ 33 | -v /workplace/github/simple-ingest-transformation-utility-pipeline/examples/config/example-data-prepper-config.yaml:/usr/share/data-prepper/data-prepper-config.yaml \ 34 | data-prepper/data-prepper:1.0.0 35 | ``` -------------------------------------------------------------------------------- /data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md: -------------------------------------------------------------------------------- 1 | # Service Map Stateful Benchmarks 2 | 3 | This package contains benchmarks for the service map stateful prepper using JMH: https://openjdk.java.net/projects/code-tools/jmh/ . 4 | 5 | Integration with gradle is done with the following gradle plugin for JMH: https://github.com/melix/jmh-gradle-plugin. 6 | 7 | The plugin creates a source set for the JMH benchmarks, and provides a few gradle tasks for running and building the benchmarks. 8 | 9 | ## Running the tests via gradle task 10 | 11 | Tests can be run via the "jmh" gradle task provided by the plugin. The README for the plugin provides the various parameters that 12 | can be provided to the plugin. 13 | 14 | ## Running the tests via JAR 15 | 16 | To run the tests via JAR, you can build the benchmark jar using the gradle task "jmhJar". This jar is an executable jar 17 | that runs the benchmark tests. Example command: 18 | 19 | ``` 20 | java -jar service-map-stateful-benchmarks-0.1-beta-jmh.jar -r 600 -i 2 -p batchSize=100 -p windowDurationSeconds=180 21 | ``` 22 | 23 | The above command will run the benchmarks for 600 seconds (10 minutes) per iteration, 2 iterations. It also 24 | sets the batchSize and windowDurationSeconds benchmark parameters. 25 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/prepper/TestPrepper.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper; 2 | 3 | import com.amazon.dataprepper.model.PluginType; 4 | import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; 5 | import com.amazon.dataprepper.model.annotations.SingleThread; 6 | import com.amazon.dataprepper.model.configuration.PluginSetting; 7 | import com.amazon.dataprepper.model.prepper.Prepper; 8 | import com.amazon.dataprepper.model.record.Record; 9 | import java.util.Collection; 10 | 11 | @SingleThread 12 | @DataPrepperPlugin(name = "test_prepper", type = PluginType.PREPPER) 13 | public class TestPrepper implements Prepper, Record> { 14 | public boolean isShutdown = false; 15 | 16 | public TestPrepper(final PluginSetting pluginSetting) {} 17 | 18 | @Override 19 | public Collection> execute(Collection> records) { 20 | return records; 21 | } 22 | 23 | @Override 24 | public void prepareForShutdown() { 25 | 26 | } 27 | 28 | @Override 29 | public boolean isReadyForShutdown() { 30 | return true; 31 | } 32 | 33 | @Override 34 | public void shutdown() { 35 | isShutdown = true; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/annotations/DataPrepperPlugin.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.annotations; 2 | 3 | import com.amazon.dataprepper.model.PluginType; 4 | 5 | import java.lang.annotation.Documented; 6 | import java.lang.annotation.ElementType; 7 | import java.lang.annotation.Retention; 8 | import java.lang.annotation.Target; 9 | import java.lang.annotation.RetentionPolicy; 10 | 11 | /** 12 | * Annotates a Data Prepper Java plugin that includes Source, Sink, Buffer and Prepper. 13 | * The value returned from {@link #name()} represents the name of the plugin and is used in the pipeline configuration 14 | * and the optional {@link #type()} 15 | * 16 | * TODO 1. Pick a different name - Plugin, Component, Resource conflicts with 17 | * other most used frameworks and may confuse users 18 | * TODO 2. Add capability for ElementType.METHOD 19 | * TODO 3. Add expected RECORD_TYPE for input and expected RECORD_TYPE for output 20 | */ 21 | 22 | @Documented 23 | @Retention(RetentionPolicy.RUNTIME) 24 | @Target({ElementType.TYPE}) 25 | public @interface DataPrepperPlugin { 26 | /** 27 | * 28 | * @return Name of the plugin which should be unique for the type 29 | */ 30 | String name(); 31 | 32 | PluginType type(); 33 | } 34 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/analytics-service/src/main/java/com/example/restservice/LoggingController.java: -------------------------------------------------------------------------------- 1 | package com.example.restservice; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.elasticsearch.action.index.IndexRequest; 5 | import org.elasticsearch.action.index.IndexResponse; 6 | import org.elasticsearch.client.RequestOptions; 7 | import org.elasticsearch.client.RestHighLevelClient; 8 | import org.elasticsearch.common.xcontent.XContentType; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.web.bind.annotation.*; 11 | import com.example.restservice.Logging; 12 | import java.io.IOException; 13 | 14 | import java.util.Optional; 15 | import java.util.Random; 16 | 17 | @RestController() 18 | public class LoggingController { 19 | 20 | private static final Random RANDOM = new Random(); 21 | 22 | @PostMapping("/logs") 23 | public String save(@RequestBody Logging logging) throws IOException { 24 | //TODO send logs to another system. 25 | // 1% of the time will throw exception 26 | /*if (RANDOM.nextInt(100) == 0) { 27 | throw new IOException("Dummy Exception"); 28 | } else { 29 | return "Dummy"; 30 | }*/ 31 | return "Dummy"; 32 | } 33 | } -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId": "6d0ff634d126b6ec2c180391e67b4237", 3 | "spanId": "9daf5fc00a50b18b", 4 | "traceState": "", 5 | "parentSpanId": "5ac57618587da71c", 6 | "name": "HTTP PUT", 7 | "kind": "SPAN_KIND_CLIENT", 8 | "startTime": "2020-08-20T05:40:46.052809200Z", 9 | "endTime": "2020-08-20T05:40:46.084822500Z", 10 | "durationInNanos": 32013300, 11 | "serviceName": "analytics-service", 12 | "events": [], 13 | "links": [], 14 | "droppedAttributesCount": 0, 15 | "droppedEventsCount": 0, 16 | "droppedLinksCount": 0, 17 | "traceGroup": null, 18 | "traceGroupFields.endTime": null, 19 | "traceGroupFields.statusCode": null, 20 | "traceGroupFields.durationInNanos": null, 21 | "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", 22 | "span.attributes.http@method": "PUT", 23 | "resource.attributes.telemetry@sdk@name": "opentelemetry", 24 | "resource.attributes.telemetry@sdk@language": "java", 25 | "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", 26 | "resource.attributes.service@name": "analytics-service", 27 | "span.attributes.http@status_code": 200, 28 | "status.code": 1, 29 | "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" 30 | } -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId": "ffa576d321173ac6cef3601c8f4bde75", 3 | "spanId": "1b83d176c0b5fb46", 4 | "traceState": "", 5 | "parentSpanId": "085ac082ffcfbf8d", 6 | "name": "HTTP PUT", 7 | "kind": "SPAN_KIND_CLIENT", 8 | "startTime": "2020-08-20T05:40:43.175204700Z", 9 | "endTime": "2020-08-20T05:40:43.205117100Z", 10 | "durationInNanos": 29912400, 11 | "serviceName": "analytics-service", 12 | "events": [], 13 | "links": [], 14 | "droppedAttributesCount": 0, 15 | "droppedEventsCount": 0, 16 | "droppedLinksCount": 0, 17 | "traceGroup": null, 18 | "traceGroupFields.endTime": null, 19 | "traceGroupFields.statusCode": null, 20 | "traceGroupFields.durationInNanos": null, 21 | "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", 22 | "span.attributes.http@method": "PUT", 23 | "resource.attributes.telemetry@sdk@name": "opentelemetry", 24 | "resource.attributes.telemetry@sdk@language": "java", 25 | "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", 26 | "resource.attributes.service@name": "analytics-service", 27 | "span.attributes.http@status_code": 200, 28 | "status.code": 1, 29 | "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" 30 | } -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/prepper/PrepperFactory.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper; 2 | 3 | import com.amazon.dataprepper.model.annotations.SingleThread; 4 | import com.amazon.dataprepper.model.configuration.PluginSetting; 5 | import com.amazon.dataprepper.model.prepper.Prepper; 6 | import com.amazon.dataprepper.plugins.PluginFactory; 7 | import com.amazon.dataprepper.plugins.PluginRepository; 8 | 9 | import java.util.ArrayList; 10 | import java.util.Collections; 11 | import java.util.List; 12 | 13 | @SuppressWarnings({"rawtypes"}) 14 | public class PrepperFactory extends PluginFactory { 15 | 16 | public static List newPreppers(final PluginSetting pluginSetting) { 17 | final Class clazz = PluginRepository.getPrepperClass(pluginSetting.getName()); 18 | if (clazz.isAnnotationPresent(SingleThread.class)) { 19 | final List preppers = new ArrayList<>(); 20 | for (int i = 0; i < pluginSetting.getNumberOfProcessWorkers(); i++) { 21 | preppers.add((Prepper) newPlugin(pluginSetting, clazz)); 22 | } 23 | return preppers; 24 | } else { 25 | return Collections.singletonList((Prepper) newPlugin(pluginSetting, clazz)); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/OTelHelper.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper; 2 | 3 | import io.opentelemetry.proto.resource.v1.Resource; 4 | import io.opentelemetry.proto.trace.v1.Span; 5 | 6 | import java.util.Optional; 7 | 8 | public class OTelHelper { 9 | 10 | static final String SERVICE_NAME_KEY = "service.name"; 11 | 12 | /** 13 | * Helper method which accepts a OpenTelemetry trace resource object and return the service name object. 14 | * 15 | * @param resource 16 | * @return 17 | */ 18 | public static Optional getServiceName(final Resource resource) { 19 | return resource.getAttributesList().stream().filter( 20 | keyValue -> keyValue.getKey().equals(SERVICE_NAME_KEY) 21 | && !keyValue.getValue().getStringValue().isEmpty() 22 | ).findFirst().map(i -> i.getValue().getStringValue()); 23 | } 24 | 25 | /** 26 | * Helper method which checks if OpenTelemetry span is valid for processing. 27 | * 28 | * @param span 29 | * @return 30 | */ 31 | public static boolean checkValidSpan(final Span span) { 32 | return !span.getTraceId().isEmpty() && !span.getSpanId().isEmpty() && !span.getName().isEmpty(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId": "6d0ff634d126b6ec2c180391e67b4237", 3 | "spanId": "9daf5fc00a50b18b", 4 | "traceState": "", 5 | "parentSpanId": "5ac57618587da71c", 6 | "name": "HTTP PUT", 7 | "kind": "SPAN_KIND_CLIENT", 8 | "startTime": "2020-08-20T05:40:46.052809200Z", 9 | "endTime": "2020-08-20T05:40:46.084822500Z", 10 | "durationInNanos": 32013300, 11 | "serviceName": "analytics-service", 12 | "events": [], 13 | "links": [], 14 | "droppedAttributesCount": 0, 15 | "droppedEventsCount": 0, 16 | "droppedLinksCount": 0, 17 | "traceGroup": "/logs", 18 | "traceGroupFields.endTime": "2020-08-20T05:40:46.089556800Z", 19 | "traceGroupFields.statusCode": 1, 20 | "traceGroupFields.durationInNanos": 48545200, 21 | "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", 22 | "span.attributes.http@method": "PUT", 23 | "resource.attributes.telemetry@sdk@name": "opentelemetry", 24 | "resource.attributes.telemetry@sdk@language": "java", 25 | "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", 26 | "resource.attributes.service@name": "analytics-service", 27 | "span.attributes.http@status_code": 200, 28 | "status.code": 1, 29 | "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" 30 | } -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-raw-prepper/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | dependencies { 6 | implementation project(':data-prepper-api') 7 | implementation project(':data-prepper-plugins:common') 8 | implementation 'commons-codec:commons-codec:1.15' 9 | testImplementation project(':data-prepper-api').sourceSets.test.output 10 | implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" 11 | implementation 'com.google.protobuf:protobuf-java-util:3.17.3' 12 | implementation "com.linecorp.armeria:armeria:1.9.2" 13 | implementation "com.linecorp.armeria:armeria-grpc:1.9.2" 14 | implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" 15 | implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" 16 | testImplementation 'org.assertj:assertj-core:3.20.2' 17 | testImplementation "org.mockito:mockito-inline:3.11.2" 18 | testImplementation "org.hamcrest:hamcrest:2.2" 19 | testImplementation "org.awaitility:awaitility:4.1.0" 20 | } 21 | 22 | jacocoTestCoverageVerification { 23 | dependsOn jacocoTestReport 24 | violationRules { 25 | rule { //in addition to core projects rule 26 | limit { 27 | minimum = 0.95 //Increase this to 0.90 28 | } 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /examples/dev/trace-analytics-sample-app/resources/pipelines.yaml: -------------------------------------------------------------------------------- 1 | entry-pipeline: 2 | delay: "100" 3 | source: 4 | otel_trace_source: 5 | ssl: true 6 | sslKeyFile: /usr/share/data-prepper/demo-data-prepper.key 7 | sslKeyCertChainFile: /usr/share/data-prepper/demo-data-prepper.crt 8 | prepper: 9 | - peer_forwarder: 10 | discovery_mode: "dns" 11 | domain_name: "prepper-cluster" 12 | ssl: false 13 | sink: 14 | - pipeline: 15 | name: "raw-pipeline" 16 | - pipeline: 17 | name: "service-map-pipeline" 18 | raw-pipeline: 19 | source: 20 | pipeline: 21 | name: "entry-pipeline" 22 | prepper: 23 | - otel_trace_raw_prepper: 24 | sink: 25 | - elasticsearch: 26 | hosts: [ "https://node-0.example.com:9200" ] 27 | cert: "/usr/share/data-prepper/root-ca.pem" 28 | username: "admin" 29 | password: "admin" 30 | trace_analytics_raw: true 31 | service-map-pipeline: 32 | delay: "100" 33 | source: 34 | pipeline: 35 | name: "entry-pipeline" 36 | prepper: 37 | - service_map_stateful: 38 | sink: 39 | - elasticsearch: 40 | hosts: ["https://node-0.example.com:9200"] 41 | cert: "/usr/share/data-prepper/root-ca.pem" 42 | username: "admin" 43 | password: "admin" 44 | trace_analytics_service_map: true 45 | -------------------------------------------------------------------------------- /research/zipkin-elastic-to-otel/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'application' 4 | } 5 | 6 | ext { 7 | es_version = System.getProperty("es.version", "${versionMap.es_version}") 8 | } 9 | 10 | group 'com.amazon' 11 | version '0.1-beta' 12 | 13 | sourceCompatibility = 1.8 14 | 15 | repositories { 16 | mavenCentral() 17 | } 18 | 19 | application { 20 | mainClass = 'com.amazon.dataprepper.research.zipkin.ZipkinElasticToOtel' 21 | } 22 | 23 | run { 24 | if (System.getProperty("test") != null) { 25 | systemProperty "test", System.getProperty("test") 26 | } 27 | } 28 | 29 | dependencies { 30 | implementation project(':data-prepper-plugins:blocking-buffer') 31 | implementation project(':data-prepper-plugins:elasticsearch') 32 | implementation project(':data-prepper-plugins:otel-trace-source') 33 | implementation project(':data-prepper-plugins:otel-trace-raw-prepper') 34 | implementation "org.apache.commons:commons-lang3:3.11" 35 | implementation "com.linecorp.armeria:armeria:1.0.0" 36 | implementation "com.linecorp.armeria:armeria-grpc:1.0.0" 37 | implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" 38 | implementation "com.fasterxml.jackson.core:jackson-databind:2.12.1" 39 | implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" 40 | } 41 | -------------------------------------------------------------------------------- /data-prepper-plugins/service-map-stateful/README.md: -------------------------------------------------------------------------------- 1 | # Service-Map Stateful Prepper 2 | 3 | This is a special prepper that consumes Opentelemetry traces, stores them in a MapDB data store and evaluate relationships at fixed ```window_duration```. 4 | 5 | # Usages 6 | Example `.yaml` configuration: 7 | ``` 8 | prepper: 9 | service-map-stateful: 10 | ``` 11 | 12 | ## Configurations 13 | 14 | * window_duration(Optional) => An `int` represents the fixed time window in seconds to evaluate service-map relationships. Default is ```180```. 15 | 16 | ## Metrics 17 | Besides common metrics in [AbstractPrepper](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/prepper/AbstractPrepper.java), service-map-stateful prepper introduces the following custom metrics. 18 | 19 | ### Gauge 20 | - `spansDbSize`: measures total spans byte sizes in MapDB across the current and previous window durations. 21 | - `traceGroupDbSize`: measures total trace group byte sizes in MapDB across the current and previous trace group window durations. 22 | 23 | ## Developer Guide 24 | This plugin is compatible with Java 8. See 25 | - [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) 26 | - [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) 27 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/ShutdownHandler.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.pipeline.server; 2 | 3 | import com.amazon.dataprepper.DataPrepper; 4 | import java.io.IOException; 5 | import java.net.HttpURLConnection; 6 | import com.sun.net.httpserver.HttpExchange; 7 | import com.sun.net.httpserver.HttpHandler; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | /** 12 | * HttpHandler to handle requests to shut down the data prepper instance 13 | */ 14 | public class ShutdownHandler implements HttpHandler { 15 | 16 | private final DataPrepper dataPrepper; 17 | private final static Logger LOG = LoggerFactory.getLogger(ShutdownHandler.class); 18 | 19 | public ShutdownHandler(final DataPrepper dataPrepper) { 20 | this.dataPrepper = dataPrepper; 21 | } 22 | 23 | @Override 24 | public void handle(HttpExchange exchange) throws IOException { 25 | try { 26 | dataPrepper.shutdown(); 27 | exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, 0); 28 | } catch (Exception e) { 29 | LOG.error("Caught exception shutting down data prepper", e); 30 | exchange.sendResponseHeaders(HttpURLConnection.HTTP_INTERNAL_ERROR, 0); 31 | } finally { 32 | exchange.getResponseBody().close(); 33 | dataPrepper.shutdownDataPrepperServer(); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/PluginFactoryTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.model.source.Source; 5 | import org.junit.Test; 6 | 7 | import java.util.HashMap; 8 | 9 | import static org.junit.Assert.assertNotNull; 10 | import static org.junit.Assert.assertTrue; 11 | 12 | @SuppressWarnings("rawtypes") 13 | public class PluginFactoryTests { 14 | public static final PluginSetting NON_EXISTENT_EMPTY_CONFIGURATION = new PluginSetting("does-not-exists", new HashMap<>()); 15 | 16 | @Test 17 | public void testNoMandatoryConstructor() { 18 | final PluginSetting testPluginSettings = new PluginSetting("junit-test", new HashMap<>()); 19 | final Class clazz = PluginRepository.getSourceClass(testPluginSettings.getName()); 20 | assertNotNull(clazz); 21 | try { 22 | PluginFactory.newPlugin(testPluginSettings, clazz); 23 | } catch (PluginException e) { 24 | assertTrue("Incorrect exception or exception message was thrown", e.getMessage().startsWith( 25 | "Data Prepper plugin requires a constructor with PluginSetting parameter; Plugin " + 26 | "ConstructorLessComponent with name junit-test is missing such constructor.")); 27 | } 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/RetryConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink.elasticsearch; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | 5 | import static com.google.common.base.Preconditions.checkNotNull; 6 | 7 | public class RetryConfiguration { 8 | public static final String DLQ_FILE = "dlq_file"; 9 | 10 | private final String dlqFile; 11 | 12 | public String getDlqFile() { 13 | return dlqFile; 14 | } 15 | 16 | public static class Builder { 17 | private String dlqFile; 18 | 19 | public Builder withDlqFile(final String dlqFile) { 20 | checkNotNull(dlqFile, "dlqFile cannot be null."); 21 | this.dlqFile = dlqFile; 22 | return this; 23 | } 24 | 25 | public RetryConfiguration build() { 26 | return new RetryConfiguration(this); 27 | } 28 | } 29 | 30 | private RetryConfiguration(final Builder builder) { 31 | this.dlqFile = builder.dlqFile; 32 | } 33 | 34 | public static RetryConfiguration readRetryConfig(final PluginSetting pluginSetting) { 35 | RetryConfiguration.Builder builder = new RetryConfiguration.Builder(); 36 | final String dlqFile = (String) pluginSetting.getAttributeFromSettings(DLQ_FILE); 37 | if (dlqFile != null) { 38 | builder = builder.withDlqFile(dlqFile); 39 | } 40 | return builder.build(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; 2 | 3 | import com.amazon.dataprepper.metrics.PluginMetrics; 4 | import com.amazon.dataprepper.model.configuration.PluginSetting; 5 | 6 | import java.util.Objects; 7 | import java.util.function.BiFunction; 8 | 9 | public enum DiscoveryMode { 10 | STATIC(StaticPeerListProvider::createPeerListProvider), 11 | DNS(DnsPeerListProvider::createPeerListProvider), 12 | AWS_CLOUD_MAP(AwsCloudMapPeerListProvider::createPeerListProvider); 13 | 14 | private final BiFunction creationFunction; 15 | 16 | DiscoveryMode(final BiFunction creationFunction) { 17 | Objects.requireNonNull(creationFunction); 18 | 19 | this.creationFunction = creationFunction; 20 | } 21 | 22 | /** 23 | * Creates a new {@link PeerListProvider} for the this discovery mode. 24 | * 25 | * @param pluginSetting The plugin settings 26 | * @param pluginMetrics The plugin metrics 27 | * @return The new {@link PeerListProvider} for this discovery mode 28 | */ 29 | PeerListProvider create(PluginSetting pluginSetting, PluginMetrics pluginMetrics) { 30 | return creationFunction.apply(pluginSetting, pluginMetrics); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /data-prepper-plugins/blocking-buffer/README.md: -------------------------------------------------------------------------------- 1 | # Blocking Buffer 2 | 3 | This is a buffer based off `LinkedBlockingQueue` bounded to the specified capacity. One can read and write records with specified timeout value. 4 | 5 | ## Usages 6 | Example `.yaml` configuration 7 | ``` 8 | buffer: 9 | - bounded_blocking: 10 | ``` 11 | *Note*: *By default, Data Prepper uses only one buffer. the `bounded_blocking` buffer, so this section in the `.yaml` need not be defined unless one wants to mention a custom buffer or tune the buffer settings* 12 | 13 | ## Configuration 14 | - buffer_size => An `int` representing max number of unchecked records the buffer accepts (num of unchecked records = num of records written into the buffer + num of in-flight records not yet checked by the Checkpointing API). Default is `512`. 15 | - batch_size => An `int` representing max number of records the buffer returns on read. Default is `8`. 16 | 17 | ##Metrics 18 | This plugin inherits the common metrics defined in [AbstractBuffer](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java) 19 | 20 | ## Developer Guide 21 | This plugin is compatible with Java 14. See 22 | - [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) 23 | - [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) 24 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file; 2 | 3 | import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProvider; 4 | import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.nio.file.Files; 9 | import java.nio.file.Path; 10 | import java.util.Objects; 11 | 12 | public class FileCertificateProvider implements CertificateProvider { 13 | private final String certificateFilePath; 14 | 15 | public FileCertificateProvider(final String certificateFilePath) { 16 | this.certificateFilePath = Objects.requireNonNull(certificateFilePath); 17 | } 18 | 19 | private static final Logger LOG = LoggerFactory.getLogger(FileCertificateProvider.class); 20 | 21 | public Certificate getCertificate() { 22 | try { 23 | final Path certFilePath = Path.of(certificateFilePath); 24 | 25 | final String certAsString = Files.readString(certFilePath); 26 | 27 | return new Certificate(certAsString); 28 | } catch (final Exception ex) { 29 | LOG.error("Error encountered while reading the certificate.", ex); 30 | throw new RuntimeException(ex); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /deployment-template/eks/README.md: -------------------------------------------------------------------------------- 1 | # Deploying on Amazon EKS 2 | This directory contains an [ingress.yaml](ingress.yaml) file capable of deploying an [AWS Application Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html) 3 | so that traffic can be routed to the Data Prepper cluster. 4 | 5 | ### Prerequisites 6 | An [AWS Certificate Manager](https://docs.aws.amazon.com/acm/latest/userguide/acm-overview.html) certificate available to be used by the load balancer. Update _ingress.yaml_ with the ARN of the certificate. 7 | 8 | Other configuration options can be specified by editing the _ingress.yaml_ file; see the [annotations documentation](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/annotations/) for a complete list of options. 9 | 10 | ## Steps 11 | 1. Provision an Amazon EKS cluster: 12 | 1. Configure kubectl to connect with the cluster: 13 | 1. Apply the Data Prepper Kubernetes config files located in the [/k8s](../k8s) directory to provision Data Prepper pods 14 | 1. Install the AWS Load Balancer Controller: 15 | 1. Run `kubectl apply -f ingress.yaml` in this directory 16 | 1. Confirm an ALB with Target Group was created in the Load Balancer AWS Console 17 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/java/com/amazon/dataprepper/server/SslUtilTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.server; 2 | 3 | import com.amazon.dataprepper.pipeline.server.SslUtil; 4 | import org.junit.Test; 5 | 6 | import javax.net.ssl.SSLContext; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | public class SslUtilTest { 11 | 12 | private static final String P12_KEYSTORE = "src/test/resources/tls/test_keystore.p12"; 13 | private static final String JKS_KEYSTORE = "src/test/resources/tls/test_keystore.jks"; 14 | 15 | private static final String KEYSTORE_WITH_PASSWORDS = "src/test/resources/tls/test_keystore_with_passwords.p12"; 16 | private static final String KEYSTORE_PASSWORD = "password"; 17 | private static final String PRIVATE_KEY_PASSWORD = "key"; 18 | 19 | @Test 20 | public void testLoadP12KeyStore() { 21 | SSLContext result = SslUtil.createSslContext(P12_KEYSTORE, "", ""); 22 | assertEquals("TLS", result.getProtocol()); 23 | } 24 | 25 | @Test 26 | public void testLoadJksKeyStore() { 27 | SSLContext result = SslUtil.createSslContext(JKS_KEYSTORE, "", ""); 28 | assertEquals("TLS", result.getProtocol()); 29 | } 30 | 31 | @Test 32 | public void testLoadP12KeyStoreWithPasswords() { 33 | SSLContext result = SslUtil.createSslContext(KEYSTORE_WITH_PASSWORDS, KEYSTORE_PASSWORD, PRIVATE_KEY_PASSWORD); 34 | assertEquals("TLS", result.getProtocol()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/RandomStringSourceTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.source; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.model.record.Record; 5 | import com.amazon.dataprepper.plugins.buffer.TestBuffer; 6 | import java.util.Collections; 7 | import java.util.LinkedList; 8 | import java.util.Queue; 9 | import org.junit.Assert; 10 | import org.junit.Test; 11 | 12 | public class RandomStringSourceTests { 13 | 14 | @Test 15 | public void testPutRecord() throws InterruptedException { 16 | final RandomStringSource randomStringSource = 17 | new RandomStringSource(new PluginSetting("random", Collections.emptyMap())); 18 | final Queue> bufferQueue = new LinkedList<>(); 19 | final TestBuffer buffer = new TestBuffer(bufferQueue, 1); 20 | //Start source, and sleep for 100 millis 21 | randomStringSource.start(buffer); 22 | Thread.sleep(100); 23 | //Make sure that 1 record is in buffer 24 | Assert.assertEquals(1, buffer.size()); 25 | //Stop the source, and wait long enough that another message would be sent 26 | //if the source was running 27 | randomStringSource.stop(); 28 | Thread.sleep(500); 29 | //Make sure there is still only 1 record in buffer 30 | Assert.assertEquals(1, buffer.size()); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-raw-prepper/README.md: -------------------------------------------------------------------------------- 1 | # OTel Trace Raw Prepper 2 | 3 | This is a prepper that serializes collection of `ExportTraceServiceRequest` sent from [otel-trace-source](../dataPrepper-plugins/otel-trace-source) into collection of string records. 4 | 5 | ## Usages 6 | Example `.yaml` configuration 7 | ``` 8 | prepper: 9 | - otel_trace_raw_prepper: 10 | ``` 11 | 12 | ## Configuration 13 | 14 | * `trace_flush_interval`: An `int` represents the time interval in seconds to flush all the descendant spans without any root span. Default to 180. 15 | 16 | ## Metrics 17 | Apart from common metrics in [AbstractPrepper](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/prepper/AbstractPrepper.java), otel-trace-raw-prepper introduces the following custom metrics. 18 | 19 | ### Counter 20 | - `spanProcessingErrors`: records the number of processing exceptions for invalid spans. 21 | - `resourceSpansProcessingErrors`: records the number of processing exceptions for invalid resource spans. 22 | - `totalProcessingErrors`: records the total number of processing errors for spans and resource spans. 23 | 24 | ## Developer Guide 25 | This plugin is compatible with Java 8. See 26 | - [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) 27 | - [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) 28 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/prepper/Prepper.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.prepper; 2 | 3 | import com.amazon.dataprepper.model.record.Record; 4 | 5 | import java.util.Collection; 6 | 7 | /** 8 | * Prepper interface. These are intermediary processing units using which users can filter, 9 | * transform and enrich the records into desired format before publishing to the sink. 10 | */ 11 | public interface Prepper, OutputRecord extends Record> { 12 | 13 | /** 14 | * execute the prepper logic which could potentially modify the incoming record. The level to which the record has 15 | * been modified depends on the implementation 16 | * 17 | * @param records Input records that will be modified/processed 18 | * @return Record modified output records 19 | */ 20 | Collection execute(Collection records); 21 | 22 | /** 23 | * Indicates to the Prepper that shutdown is imminent and any data currently held by the Prepper 24 | * should be flushed downstream. 25 | */ 26 | void prepareForShutdown(); 27 | 28 | /** 29 | * Returns true if the Prepper's internal state is safe to be shutdown. 30 | * 31 | * @return shutdown readiness status 32 | */ 33 | boolean isReadyForShutdown(); 34 | 35 | /** 36 | * Final shutdown call to clean up any resources that need to be closed. 37 | */ 38 | void shutdown(); 39 | } 40 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/java/com/amazon/dataprepper/model/PluginTypeTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model; 2 | 3 | import com.amazon.dataprepper.model.buffer.Buffer; 4 | import com.amazon.dataprepper.model.prepper.Prepper; 5 | import com.amazon.dataprepper.model.sink.Sink; 6 | import com.amazon.dataprepper.model.source.Source; 7 | import org.junit.Test; 8 | 9 | import static org.hamcrest.CoreMatchers.is; 10 | import static org.hamcrest.MatcherAssert.assertThat; 11 | 12 | public class PluginTypeTests { 13 | 14 | 15 | @Test 16 | public void testPluginTypeEnumValues() { 17 | final PluginType sourcePluginType = PluginType.SOURCE; 18 | checkPluginTypeValues(sourcePluginType, "source", Source.class); 19 | 20 | final PluginType bufferPluginType = PluginType.BUFFER; 21 | checkPluginTypeValues(bufferPluginType, "buffer", Buffer.class); 22 | 23 | final PluginType prepperPluginType = PluginType.PREPPER; 24 | checkPluginTypeValues(prepperPluginType, "prepper", Prepper.class); 25 | 26 | final PluginType sinkPluginType = PluginType.SINK; 27 | checkPluginTypeValues(sinkPluginType, "sink", Sink.class); 28 | } 29 | 30 | private void checkPluginTypeValues(final PluginType pluginType, final String name, final Class pluginClass){ 31 | assertThat("Incorrect plugin type", pluginType.pluginName(), is(name)); 32 | assertThat("Incorrect plugin class type", pluginType.pluginClass(), is(pluginClass)); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/common/FutureHelper.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.pipeline.common; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.ArrayList; 7 | import java.util.LinkedList; 8 | import java.util.List; 9 | import java.util.Queue; 10 | import java.util.concurrent.ExecutionException; 11 | import java.util.concurrent.Future; 12 | 13 | public class FutureHelper { 14 | private static final Logger LOG = LoggerFactory.getLogger(FutureHelper.class); 15 | 16 | public static FutureHelperResult awaitFuturesIndefinitely(final List> futureList) { 17 | final List completedFutureResults = new ArrayList<>(futureList.size()); 18 | final List failedExceptionList = new ArrayList<>(); 19 | 20 | final Queue> futureQueue = new LinkedList<>(futureList); 21 | while(!futureQueue.isEmpty()) { 22 | final Future future = futureQueue.remove(); 23 | try{ 24 | completedFutureResults.add(future.get()); 25 | } catch (ExecutionException e) { 26 | failedExceptionList.add(e); 27 | LOG.error("FutureTask failed due to: ", e); 28 | } catch (InterruptedException e) { 29 | LOG.error("FutureTask is interrupted or timed out"); 30 | } 31 | } 32 | return new FutureHelperResult<>(completedFutureResults, failedExceptionList); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /shared-config/log4j2-rolling.properties: -------------------------------------------------------------------------------- 1 | status = error 2 | dest = err 3 | name = PropertiesConfig 4 | 5 | property.filename = log/data-prepper/data-prepper.log 6 | 7 | appender.console.type = Console 8 | appender.console.name = STDOUT 9 | appender.console.layout.type = PatternLayout 10 | appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n 11 | 12 | appender.rolling.type = RollingFile 13 | appender.rolling.name = RollingFile 14 | appender.rolling.fileName = ${filename} 15 | appender.rolling.filePattern = logs/data-prepper.log.%d{MM-dd-yy-HH}-%i.gz 16 | appender.rolling.layout.type = PatternLayout 17 | appender.rolling.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n 18 | appender.rolling.policies.type = Policies 19 | appender.rolling.policies.time.type = TimeBasedTriggeringPolicy 20 | appender.rolling.policies.time.interval = 1 21 | appender.rolling.policies.time.modulate = true 22 | appender.rolling.policies.size.type = SizeBasedTriggeringPolicy 23 | appender.rolling.policies.size.size=100MB 24 | appender.rolling.strategy.type = DefaultRolloverStrategy 25 | appender.rolling.strategy.max = 168 26 | 27 | rootLogger.level = warn 28 | rootLogger.appenderRef.stdout.ref = STDOUT 29 | rootLogger.appenderRef.file.ref = RollingFile 30 | 31 | logger.pipeline.name = com.amazon.dataprepper.pipeline 32 | logger.pipeline.level = info 33 | 34 | logger.parser.name = com.amazon.dataprepper.parser 35 | logger.parser.level = info 36 | 37 | logger.plugins.name = com.amazon.dataprepper.plugins 38 | logger.plugins.level = info 39 | -------------------------------------------------------------------------------- /examples/demo/root-ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/jCCAuagAwIBAgIBATANBgkqhkiG9w0BAQsFADCBjzETMBEGCgmSJomT8ixk 3 | ARkWA2NvbTEXMBUGCgmSJomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1w 4 | bGUgQ29tIEluYy4xITAfBgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEh 5 | MB8GA1UEAwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMB4XDTE4MDQyMjAzNDM0 6 | NloXDTI4MDQxOTAzNDM0NlowgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJ 7 | kiaJk/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEw 8 | HwYDVQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1w 9 | bGUgQ29tIEluYy4gUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC 10 | ggEBAK/u+GARP5innhpXK0c0q7s1Su1VTEaIgmZr8VWI6S8amf5cU3ktV7WT9SuV 11 | TsAm2i2A5P+Ctw7iZkfnHWlsC3HhPUcd6mvzGZ4moxnamM7r+a9otRp3owYoGStX 12 | ylVTQusAjbq9do8CMV4hcBTepCd+0w0v4h6UlXU8xjhj1xeUIz4DKbRgf36q0rv4 13 | VIX46X72rMJSETKOSxuwLkov1ZOVbfSlPaygXIxqsHVlj1iMkYRbQmaTib6XWHKf 14 | MibDaqDejOhukkCjzpptGZOPFQ8002UtTTNv1TiaKxkjMQJNwz6jfZ53ws3fh1I0 15 | RWT6WfM4oeFRFnyFRmc4uYTUgAkCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAf 16 | BgNVHSMEGDAWgBSSNQzgDx4rRfZNOfN7X6LmEpdAczAdBgNVHQ4EFgQUkjUM4A8e 17 | K0X2TTnze1+i5hKXQHMwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IB 18 | AQBoQHvwsR34hGO2m8qVR9nQ5Klo5HYPyd6ySKNcT36OZ4AQfaCGsk+SecTi35QF 19 | RHL3g2qffED4tKR0RBNGQSgiLavmHGCh3YpDupKq2xhhEeS9oBmQzxanFwWFod4T 20 | nnsG2cCejyR9WXoRzHisw0KJWeuNlwjUdJY0xnn16srm1zL/M/f0PvCyh9HU1mF1 21 | ivnOSqbDD2Z7JSGyckgKad1Omsg/rr5XYtCeyJeXUPcmpeX6erWJJNTUh6yWC/hY 22 | G/dFC4xrJhfXwz6Z0ytUygJO32bJG4Np2iGAwvvgI9EfxzEv/KP+FGrJOvQJAq4/ 23 | BU36ZAa80W/8TBnqZTkNnqZV 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfig.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.oteltracegroup; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.plugins.sink.elasticsearch.ConnectionConfiguration; 5 | import com.amazon.dataprepper.plugins.sink.elasticsearch.IndexConstants; 6 | 7 | public class OTelTraceGroupPrepperConfig { 8 | protected static final String TRACE_ID_FIELD = "traceId"; 9 | protected static final String SPAN_ID_FIELD = "spanId"; 10 | protected static final String PARENT_SPAN_ID_FIELD = "parentSpanId"; 11 | protected static final String RAW_INDEX_ALIAS = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexConstants.RAW); 12 | protected static final String STRICT_DATE_TIME = "strict_date_time"; 13 | 14 | private final ConnectionConfiguration esConnectionConfig; 15 | 16 | public ConnectionConfiguration getEsConnectionConfig() { 17 | return esConnectionConfig; 18 | } 19 | 20 | private OTelTraceGroupPrepperConfig(final ConnectionConfiguration esConnectionConfig) { 21 | this.esConnectionConfig = esConnectionConfig; 22 | } 23 | 24 | public static OTelTraceGroupPrepperConfig buildConfig(final PluginSetting pluginSetting) { 25 | final ConnectionConfiguration esConnectionConfig = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); 26 | return new OTelTraceGroupPrepperConfig(esConnectionConfig); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/resources/test-ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/jCCAuagAwIBAgIBATANBgkqhkiG9w0BAQsFADCBjzETMBEGCgmSJomT8ixk 3 | ARkWA2NvbTEXMBUGCgmSJomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1w 4 | bGUgQ29tIEluYy4xITAfBgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEh 5 | MB8GA1UEAwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMB4XDTE4MDQyMjAzNDM0 6 | NloXDTI4MDQxOTAzNDM0NlowgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJ 7 | kiaJk/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEw 8 | HwYDVQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1w 9 | bGUgQ29tIEluYy4gUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC 10 | ggEBAK/u+GARP5innhpXK0c0q7s1Su1VTEaIgmZr8VWI6S8amf5cU3ktV7WT9SuV 11 | TsAm2i2A5P+Ctw7iZkfnHWlsC3HhPUcd6mvzGZ4moxnamM7r+a9otRp3owYoGStX 12 | ylVTQusAjbq9do8CMV4hcBTepCd+0w0v4h6UlXU8xjhj1xeUIz4DKbRgf36q0rv4 13 | VIX46X72rMJSETKOSxuwLkov1ZOVbfSlPaygXIxqsHVlj1iMkYRbQmaTib6XWHKf 14 | MibDaqDejOhukkCjzpptGZOPFQ8002UtTTNv1TiaKxkjMQJNwz6jfZ53ws3fh1I0 15 | RWT6WfM4oeFRFnyFRmc4uYTUgAkCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAf 16 | BgNVHSMEGDAWgBSSNQzgDx4rRfZNOfN7X6LmEpdAczAdBgNVHQ4EFgQUkjUM4A8e 17 | K0X2TTnze1+i5hKXQHMwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IB 18 | AQBoQHvwsR34hGO2m8qVR9nQ5Klo5HYPyd6ySKNcT36OZ4AQfaCGsk+SecTi35QF 19 | RHL3g2qffED4tKR0RBNGQSgiLavmHGCh3YpDupKq2xhhEeS9oBmQzxanFwWFod4T 20 | nnsG2cCejyR9WXoRzHisw0KJWeuNlwjUdJY0xnn16srm1zL/M/f0PvCyh9HU1mF1 21 | ivnOSqbDD2Z7JSGyckgKad1Omsg/rr5XYtCeyJeXUPcmpeX6erWJJNTUh6yWC/hY 22 | G/dFC4xrJhfXwz6Z0ytUygJO32bJG4Np2iGAwvvgI9EfxzEv/KP+FGrJOvQJAq4/ 23 | BU36ZAa80W/8TBnqZTkNnqZV 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.parser.model; 2 | 3 | import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; 4 | import io.micrometer.core.instrument.MeterRegistry; 5 | import io.micrometer.core.instrument.Tag; 6 | import io.micrometer.prometheus.PrometheusConfig; 7 | import io.micrometer.prometheus.PrometheusMeterRegistry; 8 | 9 | import java.util.Arrays; 10 | 11 | import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; 12 | import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; 13 | import static java.lang.String.format; 14 | 15 | public enum MetricRegistryType { 16 | Prometheus, 17 | CloudWatch; 18 | 19 | public static MeterRegistry getDefaultMeterRegistryForType(final MetricRegistryType metricRegistryType) { 20 | MeterRegistry meterRegistry = null; 21 | switch (metricRegistryType) { 22 | case Prometheus: 23 | meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); 24 | break; 25 | case CloudWatch: 26 | meterRegistry = new CloudWatchMeterRegistryProvider().getCloudWatchMeterRegistry(); 27 | break; 28 | default: 29 | throw new IllegalArgumentException(format("Invalid metricRegistryType %s", metricRegistryType)); 30 | } 31 | meterRegistry.config().commonTags(Arrays.asList(Tag.of(SERVICE_NAME, getServiceNameForMetrics()))); 32 | return meterRegistry; 33 | } 34 | } -------------------------------------------------------------------------------- /data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.server; 2 | 3 | import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; 4 | import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.mockito.Mock; 8 | import org.mockito.runners.MockitoJUnitRunner; 9 | import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; 10 | 11 | import static org.hamcrest.CoreMatchers.notNullValue; 12 | import static org.hamcrest.MatcherAssert.assertThat; 13 | 14 | @RunWith(MockitoJUnitRunner.class) 15 | public class CloudWatchMeterRegistryProviderTest { 16 | private static final String TEST_CLOUDWATCH_PROPERTIES = "cloudwatch_test.properties"; 17 | 18 | @Mock 19 | CloudWatchAsyncClient cloudWatchAsyncClient; 20 | 21 | @Test(expected = NullPointerException.class) 22 | public void testCreateWithInvalidPropertiesFile() { 23 | new CloudWatchMeterRegistryProvider("does not exist", cloudWatchAsyncClient); 24 | } 25 | 26 | @Test 27 | public void testCreateCloudWatchMeterRegistry() { 28 | final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider = new CloudWatchMeterRegistryProvider( 29 | TEST_CLOUDWATCH_PROPERTIES, cloudWatchAsyncClient); 30 | final CloudWatchMeterRegistry cloudWatchMeterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); 31 | assertThat(cloudWatchMeterRegistry, notNullValue()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-service-map-index-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 0, 3 | "mappings": { 4 | "date_detection": false, 5 | "dynamic_templates": [ 6 | { 7 | "strings_as_keyword": { 8 | "mapping": { 9 | "ignore_above": 1024, 10 | "type": "keyword" 11 | }, 12 | "match_mapping_type": "string" 13 | } 14 | } 15 | ], 16 | "_source": { 17 | "enabled": true 18 | }, 19 | "properties": { 20 | "hashId": { 21 | "ignore_above": 1024, 22 | "type": "keyword" 23 | }, 24 | "serviceName": { 25 | "ignore_above": 1024, 26 | "type": "keyword" 27 | }, 28 | "kind": { 29 | "ignore_above": 1024, 30 | "type": "keyword" 31 | }, 32 | "destination": { 33 | "properties": { 34 | "domain": { 35 | "ignore_above": 1024, 36 | "type": "keyword" 37 | }, 38 | "resource": { 39 | "ignore_above": 1024, 40 | "type": "keyword" 41 | } 42 | } 43 | }, 44 | "target": { 45 | "properties": { 46 | "domain": { 47 | "ignore_above": 1024, 48 | "type": "keyword" 49 | }, 50 | "resource": { 51 | "ignore_above": 1024, 52 | "type": "keyword" 53 | } 54 | } 55 | }, 56 | "traceGroupName": { 57 | "ignore_above": 1024, 58 | "type": "keyword" 59 | } 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ODFETests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink.elasticsearch; 2 | 3 | import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; 4 | import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; 5 | import org.elasticsearch.client.RequestOptions; 6 | import org.elasticsearch.client.RestHighLevelClient; 7 | import org.elasticsearch.rest.RestStatus; 8 | import org.junit.Test; 9 | 10 | import java.io.IOException; 11 | import java.util.Collections; 12 | 13 | import static org.junit.Assert.assertEquals; 14 | 15 | public class ODFETests { 16 | @Test 17 | public void testODFEConnection() throws IOException { 18 | final String host = System.getProperty("odfe.host"); 19 | final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( 20 | Collections.singletonList(host)); 21 | final String user = System.getProperty("odfe.user"); 22 | final String password = System.getProperty("odfe.password"); 23 | if (user != null) { 24 | builder.withUsername(user); 25 | builder.withPassword(password); 26 | } 27 | final RestHighLevelClient client = builder.build().createClient(); 28 | final ClusterHealthRequest request = new ClusterHealthRequest().waitForYellowStatus(); 29 | final ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); 30 | assertEquals(RestStatus.OK, response.status()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /data-prepper-core/src/test/java/com/amazon/dataprepper/plugins/TestSink.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins; 2 | 3 | import com.amazon.dataprepper.model.PluginType; 4 | import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; 5 | import com.amazon.dataprepper.model.record.Record; 6 | import com.amazon.dataprepper.model.sink.Sink; 7 | 8 | import java.util.ArrayList; 9 | import java.util.Collection; 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | @DataPrepperPlugin(name = "test_sink", type = PluginType.SINK) 14 | public class TestSink implements Sink> { 15 | private final List> collectedRecords; 16 | private final boolean failSinkForTest; 17 | public boolean isShutdown = false; 18 | 19 | public TestSink() { 20 | this.failSinkForTest = false; 21 | this.collectedRecords = new ArrayList<>(); 22 | } 23 | 24 | public TestSink(boolean failSinkForTest) { 25 | this.failSinkForTest = failSinkForTest; 26 | this.collectedRecords = new ArrayList<>(); 27 | } 28 | 29 | @Override 30 | public void output(Collection> records) { 31 | if(failSinkForTest) { 32 | throw new RuntimeException("Sink is expected to fail"); 33 | } 34 | records.stream().collect(Collectors.toCollection(() -> collectedRecords)); 35 | } 36 | 37 | @Override 38 | public void shutdown() { 39 | isShutdown = true; 40 | } 41 | 42 | public List> getCollectedRecords() { 43 | return collectedRecords; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEvent.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.oteltrace.model; 2 | 3 | import io.opentelemetry.proto.trace.v1.Span; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * Java POJO of https://github.com/open-telemetry/opentelemetry-proto/blob/master/opentelemetry/proto/trace/v1/trace.proto#L169 9 | * which is compatible with elasticsearch 10 | */ 11 | public final class RawEvent { 12 | private final String time; 13 | private final String name; 14 | private final Map attributes; 15 | private final int droppedAttributesCount; 16 | 17 | public String getTime() { 18 | return time; 19 | } 20 | 21 | public String getName() { 22 | return name; 23 | } 24 | 25 | public Map getAttributes() { 26 | return attributes; 27 | } 28 | 29 | public int getDroppedAttributesCount() { 30 | return droppedAttributesCount; 31 | } 32 | 33 | private RawEvent(final String time, final String name, final Map attributes, int droppedAttributesCount) { 34 | this.time = time; 35 | this.name = name; 36 | this.attributes = attributes; 37 | this.droppedAttributesCount = droppedAttributesCount; 38 | } 39 | 40 | public static RawEvent buildRawEvent(final Span.Event event) { 41 | return new RawEvent(OTelProtoHelper.getTimeISO8601(event), event.getName(), OTelProtoHelper.getEventAttributes(event), event.getDroppedAttributesCount()); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /examples/trace-analytics-sample-app/sample-app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gradle:6.5.0-jdk8 as cache 2 | RUN mkdir -p /home/gradle/cache_home 3 | ENV GRADLE_USER_HOME /home/gradle/cache_home 4 | COPY analytics-service/build.gradle /home/gradle/src/ 5 | WORKDIR /home/gradle/src 6 | RUN gradle clean build --daemon 7 | 8 | FROM gradle:6.5.0-jdk8 AS build 9 | COPY --from=cache /home/gradle/cache_home /home/gradle/.gradle 10 | COPY analytics-service /home/gradle/src/ 11 | WORKDIR /home/gradle/src 12 | RUN gradle bootJar --daemon 13 | 14 | RUN wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.0.1/opentelemetry-javaagent-all.jar 15 | 16 | FROM alpine:3.7 17 | 18 | RUN apk update \ 19 | && apk upgrade \ 20 | # add for grpcio 21 | && apk add --no-cache g++ \ 22 | # add for grpcio 23 | && apk add --no-cache linux-headers \ 24 | && apk add --no-cache bash \ 25 | && apk add --no-cache --virtual=build-dependencies unzip \ 26 | && apk add --no-cache curl \ 27 | && apk add --no-cache openjdk8-jre \ 28 | # add python3-dev for grpcio 29 | && apk add --no-cache python3 python3-dev 30 | 31 | RUN mkdir /app 32 | COPY . /app/ 33 | COPY --from=build /home/gradle/src/build/libs/*.jar /app/spring-boot-application.jar 34 | COPY --from=build /home/gradle/src/opentelemetry-javaagent-all.jar /app/opentelemetry-javaagent-all.jar 35 | 36 | RUN pip3 install --upgrade pip 37 | RUN pip3 install -r /app/requirements.txt 38 | WORKDIR /app 39 | RUN chmod +x /app/script.sh 40 | ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.2.1/wait /app/wait 41 | RUN chmod +x /app/wait 42 | 43 | CMD /app/wait && /app/script.sh -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/BackOffUtilsTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink.elasticsearch; 2 | 3 | import org.elasticsearch.action.bulk.BackoffPolicy; 4 | import org.elasticsearch.common.unit.TimeValue; 5 | import org.junit.Test; 6 | 7 | import java.util.Iterator; 8 | 9 | import static org.junit.Assert.assertFalse; 10 | import static org.junit.Assert.assertTrue; 11 | 12 | public class BackOffUtilsTests { 13 | @Test 14 | public void testWithExpBackoff() throws InterruptedException { 15 | final long start = 50; 16 | final int testNumOfRetries = 3; 17 | final Iterator timeValueIterator = 18 | BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(start), testNumOfRetries).iterator(); 19 | final BackOffUtils backOffUtils = new BackOffUtils( 20 | BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(start), testNumOfRetries).iterator()); 21 | final long startTime = System.currentTimeMillis(); 22 | // first attempt 23 | assertTrue(backOffUtils.hasNext()); 24 | assertTrue(backOffUtils.next()); 25 | for (int i = 0; i < testNumOfRetries; i++) { 26 | assertTrue(backOffUtils.hasNext()); 27 | assertTrue(backOffUtils.next()); 28 | final long currTime = System.currentTimeMillis(); 29 | assertTrue(currTime - startTime >= timeValueIterator.next().getMillis()); 30 | } 31 | assertFalse(backOffUtils.hasNext()); 32 | assertFalse(backOffUtils.next()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Docs: https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates 2 | version: 2 3 | updates: 4 | - package-ecosystem: gradle 5 | directory: "/data-prepper-api" 6 | schedule: 7 | interval: monthly 8 | 9 | - package-ecosystem: gradle 10 | directory: "/data-prepper-core" 11 | schedule: 12 | interval: monthly 13 | 14 | - package-ecosystem: gradle 15 | directory: "/data-prepper-plugins/blocking-buffer" 16 | schedule: 17 | interval: monthly 18 | 19 | - package-ecosystem: gradle 20 | directory: "/data-prepper-plugins/common" 21 | schedule: 22 | interval: monthly 23 | 24 | - package-ecosystem: gradle 25 | directory: "/data-prepper-plugins/elasticsearch" 26 | schedule: 27 | interval: monthly 28 | 29 | - package-ecosystem: gradle 30 | directory: "/data-prepper-plugins/mapdb-prepper-state" 31 | schedule: 32 | interval: monthly 33 | 34 | - package-ecosystem: gradle 35 | directory: "/data-prepper-plugins/otel-trace-raw-prepper" 36 | schedule: 37 | interval: monthly 38 | 39 | - package-ecosystem: gradle 40 | directory: "/data-prepper-plugins/otel-trace-group-prepper" 41 | schedule: 42 | interval: monthly 43 | 44 | - package-ecosystem: gradle 45 | directory: "/data-prepper-plugins/otel-trace-source" 46 | schedule: 47 | interval: monthly 48 | 49 | - package-ecosystem: gradle 50 | directory: "/data-prepper-plugins/peer-forwarder" 51 | schedule: 52 | interval: monthly 53 | 54 | - package-ecosystem: gradle 55 | directory: "/data-prepper-plugins/service-map-stateful" 56 | schedule: 57 | interval: monthly 58 | -------------------------------------------------------------------------------- /docs/readme/error_handling.md: -------------------------------------------------------------------------------- 1 | # Error Handling 2 | 3 | In Data Prepper Pipeline, errors should be handled by implementation of plugin components and should not throw any uncaught runtime exceptions. If thrown the pipeline will stop immediately or halt eventually based on the component that throws the exception. 4 | 5 | Below are the different scenario of error, 6 | 7 | ## Single Pipeline 8 | Single pipeline is a pipeline that doesn't use the pipeline connectors. 9 | 10 | * If an exception encountered when creating a pipeline, the pipeline stops and exits using `system.exit(1)`. 11 | * If an exception encountered during pipeline `start()` method, the pipeline stops with appropriate logs. 12 | * If an exception encountered in Pipeline `preppers` or `sinks` during runtime, the pipeline stops with appropriate logs. 13 | 14 | ## Connected Pipelines 15 | 16 | Connected Pipelines is scenario where two or more pipelines connected using the Pipeline Connector. 17 | 18 | 19 | * If an exception encountered when creating any of the connected pipelines, all the connected pipelines stop and exits using `system.exit(1)`. 20 | * If an exception encountered during pipeline `start()` method in any of the connected pipelines, the pipeline that encountered the exception will shutdown and other pipelines will run but not receive or process any data as a connected pipeline is unavailable. 21 | * If an exception encountered in Pipeline `preppers` or `sinks` during runtime in any of the connected pipelines, the pipeline that encountered the exception will shutdown and other pipelines will run for a while and shutdown as a connected pipeline is unavailable. 22 | 23 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/common/PipelineThreadFactory.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.pipeline.common; 2 | 3 | import java.util.concurrent.Executors; 4 | import java.util.concurrent.ThreadFactory; 5 | import java.util.concurrent.atomic.AtomicInteger; 6 | 7 | /** 8 | * ThreadFactory with the ability to set the thread name prefix. This class is exactly similar to 9 | * {@link Executors#defaultThreadFactory()}, except for the thread naming feature. 10 | */ 11 | public class PipelineThreadFactory implements ThreadFactory { 12 | private static final AtomicInteger poolNumber = new AtomicInteger(1); 13 | private final ThreadGroup threadGroup; 14 | private final AtomicInteger threadNumber = new AtomicInteger(1); 15 | private final String namePrefix; 16 | 17 | public PipelineThreadFactory(final String namePrefix) { 18 | final SecurityManager securityManager = System.getSecurityManager(); 19 | threadGroup = (securityManager != null) ? securityManager.getThreadGroup() : 20 | Thread.currentThread().getThreadGroup(); 21 | this.namePrefix = namePrefix + "-" + poolNumber.getAndIncrement() + "-thread-"; 22 | } 23 | 24 | @Override 25 | public Thread newThread(final Runnable runnable) { 26 | Thread thread = new Thread(threadGroup, runnable, namePrefix + threadNumber.getAndIncrement(), 0); 27 | if(thread.isDaemon()) { 28 | thread.setDaemon(false); 29 | } 30 | if(thread.getPriority() != Thread.NORM_PRIORITY) { 31 | thread.setPriority(Thread.NORM_PRIORITY); 32 | } 33 | return thread; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/sink/StdOutSink.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink; 2 | 3 | import com.amazon.dataprepper.model.PluginType; 4 | import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; 5 | import com.amazon.dataprepper.model.configuration.PluginSetting; 6 | import com.amazon.dataprepper.model.record.Record; 7 | import com.amazon.dataprepper.model.sink.Sink; 8 | 9 | import java.util.Collection; 10 | import java.util.Iterator; 11 | 12 | @DataPrepperPlugin(name = "stdout", type = PluginType.SINK) 13 | public class StdOutSink implements Sink> { 14 | 15 | /** 16 | * Mandatory constructor for Data Prepper Component - This constructor is used by Data Prepper 17 | * runtime engine to construct an instance of {@link StdOutSink} using an instance of {@link PluginSetting} which 18 | * has access to pluginSetting metadata from pipeline 19 | * pluginSetting file. 20 | * 21 | * @param pluginSetting instance with metadata information from pipeline pluginSetting file. 22 | */ 23 | public StdOutSink(final PluginSetting pluginSetting) { 24 | this(); 25 | } 26 | 27 | public StdOutSink() { 28 | } 29 | 30 | @Override 31 | public void output(final Collection> records) { 32 | final Iterator> iterator = records.iterator(); 33 | while (iterator.hasNext()) { 34 | final Record record = iterator.next(); 35 | System.out.println(record.getData()); 36 | } 37 | } 38 | 39 | @Override 40 | public void shutdown() { 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/main/java/com/amazon/dataprepper/plugins/prepper/NoOpPrepper.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper; 2 | 3 | import com.amazon.dataprepper.model.PluginType; 4 | import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; 5 | import com.amazon.dataprepper.model.configuration.PluginSetting; 6 | import com.amazon.dataprepper.model.prepper.Prepper; 7 | import com.amazon.dataprepper.model.record.Record; 8 | 9 | import java.util.Collection; 10 | 11 | @DataPrepperPlugin(name = "no-op", type = PluginType.PREPPER) 12 | public class NoOpPrepper> implements Prepper { 13 | 14 | /** 15 | * Mandatory constructor for Data Prepper Component - This constructor is used by Data Prepper 16 | * runtime engine to construct an instance of {@link NoOpPrepper} using an instance of {@link PluginSetting} which 17 | * has access to pluginSetting metadata from pipeline 18 | * pluginSetting file. 19 | * 20 | * @param pluginSetting instance with metadata information from pipeline pluginSetting file. 21 | */ 22 | public NoOpPrepper(final PluginSetting pluginSetting) { 23 | //no op 24 | } 25 | 26 | public NoOpPrepper() { 27 | 28 | } 29 | 30 | @Override 31 | public Collection execute(Collection records) { 32 | return records; 33 | } 34 | 35 | @Override 36 | public void prepareForShutdown() { 37 | 38 | } 39 | 40 | @Override 41 | public boolean isReadyForShutdown() { 42 | return true; 43 | } 44 | 45 | 46 | @Override 47 | public void shutdown() { 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/source/TestSource.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.source; 2 | 3 | import com.amazon.dataprepper.model.PluginType; 4 | import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; 5 | import com.amazon.dataprepper.model.buffer.Buffer; 6 | import com.amazon.dataprepper.model.record.Record; 7 | import com.amazon.dataprepper.model.source.Source; 8 | 9 | import java.util.Iterator; 10 | import java.util.List; 11 | import java.util.concurrent.TimeoutException; 12 | import java.util.stream.Collectors; 13 | import java.util.stream.Stream; 14 | 15 | @DataPrepperPlugin(name = "test-source", type = PluginType.SOURCE) 16 | public class TestSource implements Source> { 17 | public static final List> TEST_DATA = Stream.of("THIS", "IS", "TEST", "DATA") 18 | .map(Record::new).collect(Collectors.toList()); 19 | private static final int WRITE_TIMEOUT = 5_000; 20 | private boolean isStopRequested; 21 | 22 | public TestSource() { 23 | isStopRequested = false; 24 | } 25 | 26 | @Override 27 | public void start(Buffer> buffer) { 28 | final Iterator> iterator = TEST_DATA.iterator(); 29 | while (iterator.hasNext() && !isStopRequested) { 30 | try { 31 | buffer.write(iterator.next(), WRITE_TIMEOUT); 32 | } catch (TimeoutException e) { 33 | throw new RuntimeException("Timed out writing to buffer"); 34 | } 35 | } 36 | } 37 | 38 | @Override 39 | public void stop() { 40 | isStopRequested = true; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.pipeline.server; 2 | 3 | import com.sun.net.httpserver.HttpExchange; 4 | import com.sun.net.httpserver.HttpHandler; 5 | import io.micrometer.prometheus.PrometheusMeterRegistry; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.IOException; 10 | import java.net.HttpURLConnection; 11 | 12 | /** 13 | * HttpHandler to handle requests for Prometheus metrics 14 | */ 15 | public class PrometheusMetricsHandler implements HttpHandler { 16 | 17 | private PrometheusMeterRegistry prometheusMeterRegistry; 18 | private final Logger LOG = LoggerFactory.getLogger(PrometheusMetricsHandler.class); 19 | 20 | public PrometheusMetricsHandler(final PrometheusMeterRegistry prometheusMeterRegistry) { 21 | this.prometheusMeterRegistry = prometheusMeterRegistry; 22 | } 23 | 24 | @Override 25 | public void handle(HttpExchange exchange) throws IOException { 26 | try { 27 | byte response[] = prometheusMeterRegistry.scrape().getBytes("UTF-8"); 28 | exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=UTF-8"); 29 | exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, response.length); 30 | exchange.getResponseBody().write(response); 31 | } catch (Exception e) { 32 | LOG.error("Encountered exception scraping prometheus meter registry", e); 33 | exchange.sendResponseHeaders(HttpURLConnection.HTTP_INTERNAL_ERROR, 0); 34 | } finally { 35 | exchange.getResponseBody().close(); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "traceId": "ffa576d321173ac6cef3601c8f4bde75", 3 | "spanId": "cb1c0734d14940fd", 4 | "traceState": "", 5 | "parentSpanId": "", 6 | "name": "/logs", 7 | "kind": "SPAN_KIND_SERVER", 8 | "startTime": "2020-08-20T05:40:43.168010200Z", 9 | "endTime": "2020-08-20T05:40:43.217170200Z", 10 | "durationInNanos": 49160000, 11 | "serviceName": "analytics-service", 12 | "events": [], 13 | "links": [], 14 | "droppedAttributesCount": 0, 15 | "droppedEventsCount": 0, 16 | "droppedLinksCount": 0, 17 | "traceGroup": "/logs", 18 | "traceGroupFields.endTime": "2020-08-20T05:40:43.217170200Z", 19 | "traceGroupFields.statusCode": 1, 20 | "traceGroupFields.durationInNanos": 49160000, 21 | "span.attributes.http@url": "http://0.0.0.0:8087/logs", 22 | "span.attributes.net@peer@ip": "172.29.0.1", 23 | "resource.attributes.telemetry@sdk@language": "java", 24 | "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", 25 | "resource.attributes.service@name": "analytics-service", 26 | "span.attributes.servlet@path": "/logs", 27 | "status.code": 1, 28 | "instrumentationLibrary.name": "io.opentelemetry.auto.servlet-3.0", 29 | "span.attributes.http@response_content_length": 7, 30 | "span.attributes.http@user_agent": "curl/7.54.0", 31 | "span.attributes.http@method": "POST", 32 | "span.attributes.net@peer@port": 41164, 33 | "resource.attributes.telemetry@sdk@name": "opentelemetry", 34 | "span.attributes.servlet@context": "", 35 | "span.attributes.http@flavor": "HTTP/1.1", 36 | "span.attributes.http@status_code": 200, 37 | "span.attributes.http@client_ip": "172.29.0.1" 38 | } -------------------------------------------------------------------------------- /data-prepper-benchmarks/service-map-stateful-benchmarks/results/summary.md: -------------------------------------------------------------------------------- 1 | # Benchmarking Results 2 | 3 | ## Setup 4 | These tests were run on AWS EC2 r5 generation instances. For benchmarking with CPU counts of 2, 4, 8, and 16, 5 | instance types of r5.large, r5.xlarge, r5.2xlarge, and r5.4xlarge were used respectively. 6 | 7 | For the benchmarking, iterations were run for 10 minutes, with the default window duration of 3 minutes. The batch size 8 | used, in terms of Records, was 1024. Throughput is represented in terms of Records processed per second, rather than 9 | batches processed per second. 10 | 11 | ## Results 12 | 13 | | CPU Count | Processor Count | Throughput | 14 | |-----------|-----------------|------------| 15 | | 2 | 1 | 5546 | 16 | | 2 | 2 | 6562 | 17 | | 2 | 4 | 6060 | 18 | | 2 | 8 | 5330 | 19 | | 2 | 16 | 4877 | 20 | | 4 | 1 | 6149 | 21 | | 4 | 2 | 9372 | 22 | | 4 | 4 | 10482 | 23 | | 4 | 8 | 9924 | 24 | | 4 | 16 | 8513 | 25 | | 8 | 1 | 5838 | 26 | | 8 | 2 | 10803 | 27 | | 8 | 4 | 18114 | 28 | | 8 | 8 | 15708 | 29 | | 8 | 16 | 16719 | 30 | | 16 | 1 | 5986 | 31 | | 16 | 2 | 10773 | 32 | | 16 | 4 | 20157 | 33 | | 16 | 8 | 21387 | 34 | | 16 | 16 | 34537 | -------------------------------------------------------------------------------- /data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/SslUtil.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.pipeline.server; 2 | 3 | import javax.net.ssl.KeyManagerFactory; 4 | import javax.net.ssl.SSLContext; 5 | import javax.net.ssl.TrustManagerFactory; 6 | import java.nio.file.Files; 7 | import java.nio.file.Path; 8 | import java.security.KeyStore; 9 | 10 | public class SslUtil { 11 | 12 | public static SSLContext createSslContext(final String keyStoreFilePath, 13 | final String keyStorePassword, 14 | final String privateKeyPassword) { 15 | final SSLContext sslContext; 16 | 17 | try { 18 | final KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); 19 | keyStore.load(Files.newInputStream(Path.of(keyStoreFilePath)), keyStorePassword.toCharArray()); 20 | 21 | final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); 22 | keyManagerFactory.init(keyStore, privateKeyPassword.toCharArray()); 23 | 24 | final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); 25 | trustManagerFactory.init(keyStore); 26 | 27 | sslContext = SSLContext.getInstance("TLS"); 28 | sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), null); 29 | } catch (Exception e) { 30 | throw new IllegalStateException("Problem loading keystore to create SSLContext", e); 31 | } 32 | 33 | return sslContext; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfigTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.oteltracegroup; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.plugins.sink.elasticsearch.ConnectionConfiguration; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.mockito.Mock; 8 | import org.mockito.MockedStatic; 9 | import org.mockito.Mockito; 10 | import org.mockito.junit.MockitoJUnitRunner; 11 | 12 | import java.util.HashMap; 13 | 14 | import static org.junit.Assert.assertEquals; 15 | import static org.mockito.ArgumentMatchers.any; 16 | 17 | @RunWith(MockitoJUnitRunner.class) 18 | public class OTelTraceGroupPrepperConfigTests { 19 | 20 | @Mock 21 | private ConnectionConfiguration connectionConfigurationMock; 22 | 23 | @Test 24 | public void testInitialize() { 25 | try (MockedStatic connectionConfigurationMockedStatic = Mockito.mockStatic(ConnectionConfiguration.class)) { 26 | connectionConfigurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(any(PluginSetting.class))) 27 | .thenReturn(connectionConfigurationMock); 28 | PluginSetting testPluginSetting = new PluginSetting("otel_trace_group_prepper", new HashMap<>()); 29 | OTelTraceGroupPrepperConfig otelTraceGroupPrepperConfig = OTelTraceGroupPrepperConfig.buildConfig(testPluginSetting); 30 | assertEquals(connectionConfigurationMock, otelTraceGroupPrepperConfig.getEsConnectionConfig()); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/Buffer.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.buffer; 2 | 3 | import com.amazon.dataprepper.model.record.Record; 4 | import com.amazon.dataprepper.model.CheckpointState; 5 | 6 | import java.util.Collection; 7 | import java.util.Map; 8 | import java.util.concurrent.TimeoutException; 9 | 10 | /** 11 | * Buffer queues the records between TI components and acts as a layer between source and prepper/sink. Buffer can 12 | * be in-memory, disk based or other a standalone implementation. 13 | *

14 | */ 15 | public interface Buffer> { 16 | /** 17 | * writes the record to the buffer 18 | * 19 | * @param record the Record to add 20 | * @param timeoutInMillis how long to wait before giving up 21 | */ 22 | void write(T record, int timeoutInMillis) throws TimeoutException; 23 | 24 | /** 25 | * Retrieves and removes the batch of records from the head of the queue. The batch size is defined/determined by 26 | * the configuration attribute "batch_size" or the @param timeoutInMillis 27 | * @param timeoutInMillis how long to wait before giving up 28 | * @return The earliest batch of records in the buffer which are still not read and its corresponding checkpoint state. 29 | */ 30 | Map.Entry, CheckpointState> read(int timeoutInMillis); 31 | 32 | /** 33 | * Check summary of records processed by data-prepper downstreams(preppers, sinks, pipelines). 34 | * 35 | * @param checkpointState the summary object of checkpoint variables 36 | */ 37 | void checkpoint(CheckpointState checkpointState); 38 | 39 | boolean isEmpty(); 40 | } 41 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/RetryConfigurationTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink.elasticsearch; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import org.junit.Test; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | import static org.junit.Assert.assertNull; 11 | 12 | public class RetryConfigurationTests { 13 | @Test 14 | public void testDefaultConfigurationIsNotNull() { 15 | final RetryConfiguration retryConfiguration = new RetryConfiguration.Builder().build(); 16 | assertNull(retryConfiguration.getDlqFile()); 17 | } 18 | 19 | @Test 20 | public void testReadRetryConfigNoDLQFilePath() { 21 | final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(null)); 22 | assertNull(retryConfiguration.getDlqFile()); 23 | } 24 | 25 | @Test 26 | public void testReadRetryConfigWithDLQFilePath() { 27 | final String fakeDlqFilePath = "foo.txt"; 28 | final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(fakeDlqFilePath)); 29 | assertEquals(fakeDlqFilePath, retryConfiguration.getDlqFile()); 30 | } 31 | 32 | private PluginSetting generatePluginSetting(final String dlqFilePath) { 33 | final Map metadata = new HashMap<>(); 34 | if (dlqFilePath != null) { 35 | metadata.put(RetryConfiguration.DLQ_FILE, dlqFilePath); 36 | } 37 | 38 | return new PluginSetting("elasticsearch", metadata); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/MetricsTestUtil.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.metrics; 2 | 3 | import java.util.List; 4 | import java.util.stream.Collectors; 5 | import java.util.stream.StreamSupport; 6 | import io.micrometer.core.instrument.Measurement; 7 | import io.micrometer.core.instrument.MeterRegistry; 8 | import io.micrometer.core.instrument.Metrics; 9 | import io.micrometer.core.instrument.Statistic; 10 | import io.micrometer.core.instrument.simple.SimpleMeterRegistry; 11 | 12 | public class MetricsTestUtil { 13 | 14 | public static void initMetrics() { 15 | Metrics.globalRegistry.getRegistries().forEach(meterRegistry -> Metrics.globalRegistry.remove(meterRegistry)); 16 | Metrics.globalRegistry.getMeters().forEach(meter -> Metrics.globalRegistry.remove(meter)); 17 | Metrics.addRegistry(new SimpleMeterRegistry()); 18 | } 19 | 20 | public static List getMeasurementList(final String meterName) { 21 | return StreamSupport.stream(getRegistry().find(meterName).meter().measure().spliterator(), false) 22 | .collect(Collectors.toList()); 23 | } 24 | 25 | public static Measurement getMeasurementFromList(final List measurements, final Statistic statistic) { 26 | return measurements.stream().filter(measurement -> measurement.getStatistic() == statistic).findAny().get(); 27 | } 28 | 29 | private static MeterRegistry getRegistry() { 30 | return Metrics.globalRegistry.getRegistries().iterator().next(); 31 | } 32 | 33 | public static boolean isBetween(double value, double low, double high) { 34 | return value > low && value < high; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /release/archives/macos/data-prepper-tar-install-with-jdk-x64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). 6 | # You may not use this file except in compliance with the License. 7 | # A copy of the License is located at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | 16 | if [[ $# -ne 2 ]] 17 | then 18 | echo 19 | echo "Error: Paths to pipeline and data-prepper configuration files are required. Example:" 20 | echo "./data-prepper-tar-install.sh config/example-pipelines.yaml config/example-data-prepper-config.yaml" 21 | echo 22 | exit 1 23 | fi 24 | 25 | PIPELINES_FILE_LOCATION=$1 26 | CONFIG_FILE_LOCATION=$2 27 | DATA_PREPPER_HOME=$(cd "$(dirname "$0")"; pwd) 28 | EXECUTABLE_JAR=$(ls -1 $DATA_PREPPER_HOME/bin/*.jar 2>/dev/null) 29 | OPENJDK=$(ls -1 $DATA_PREPPER_HOME/openjdk/ 2>/dev/null) 30 | 31 | if [[ -z "$EXECUTABLE_JAR" ]] 32 | then 33 | echo "Jar file is missing from directory $DATA_PREPPER_HOME/bin" 34 | exit 1 35 | fi 36 | 37 | export JAVA_HOME=$DATA_PREPPER_HOME/openjdk/$OPENJDK/Contents/Home 38 | export PATH=$JAVA_HOME/bin:$PATH 39 | DATA_PREPPER_JAVA_OPTS="-Dlog4j.configurationFile=$DATA_PREPPER_HOME/config/log4j2-rolling.properties" 40 | $JAVA_HOME/bin/java $JAVA_OPTS $DATA_PREPPER_JAVA_OPTS -jar $EXECUTABLE_JAR $PIPELINES_FILE_LOCATION $CONFIG_FILE_LOCATION -------------------------------------------------------------------------------- /release/archives/linux/data-prepper-tar-install-with-jdk-x64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"). 6 | # You may not use this file except in compliance with the License. 7 | # A copy of the License is located at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed 12 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | # express or implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | if [[ $# -ne 2 ]] 16 | then 17 | echo 18 | echo "Error: Paths to pipeline and data-prepper configuration files are required. Example:" 19 | echo "./data-prepper-tar-install.sh config/example-pipelines.yaml config/example-data-prepper-config.yaml" 20 | echo 21 | exit 1 22 | fi 23 | 24 | PIPELINES_FILE_LOCATION=$1 25 | CONFIG_FILE_LOCATION=$2 26 | DATA_PREPPER_HOME=$(dirname $(realpath $0)) 27 | EXECUTABLE_JAR=$(ls -1 $DATA_PREPPER_HOME/bin/*.jar 2>/dev/null) 28 | OPENJDK=$(ls -1 $DATA_PREPPER_HOME/openjdk/ 2>/dev/null) 29 | 30 | if [[ -z "$EXECUTABLE_JAR" ]] 31 | then 32 | echo "Jar file is missing from directory $DATA_PREPPER_HOME/bin" 33 | exit 1 34 | fi 35 | 36 | export JAVA_HOME=$DATA_PREPPER_HOME/openjdk/$OPENJDK 37 | echo "JAVA_HOME is set to $JAVA_HOME" 38 | export PATH=$JAVA_HOME/bin:$PATH 39 | 40 | DATA_PREPPER_JAVA_OPTS="-Dlog4j.configurationFile=$DATA_PREPPER_HOME/config/log4j2-rolling.properties" 41 | java $JAVA_OPTS $DATA_PREPPER_JAVA_OPTS -jar $EXECUTABLE_JAR $PIPELINES_FILE_LOCATION $CONFIG_FILE_LOCATION -------------------------------------------------------------------------------- /data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.certificate.file; 2 | 3 | import com.amazon.dataprepper.plugins.certificate.CertificateProvider; 4 | import com.amazon.dataprepper.plugins.certificate.model.Certificate; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.nio.file.Files; 9 | import java.nio.file.Path; 10 | import java.util.Objects; 11 | 12 | public class FileCertificateProvider implements CertificateProvider { 13 | private final String certificateFilePath; 14 | private final String privateKeyFilePath; 15 | 16 | public FileCertificateProvider(final String certificateFilePath, 17 | final String privateKeyFilePath) { 18 | this.certificateFilePath = Objects.requireNonNull(certificateFilePath); 19 | this.privateKeyFilePath = Objects.requireNonNull(privateKeyFilePath); 20 | } 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(FileCertificateProvider.class); 23 | 24 | public Certificate getCertificate() { 25 | try { 26 | final Path certFilePath = Path.of(certificateFilePath); 27 | final Path pkFilePath = Path.of(privateKeyFilePath); 28 | 29 | final String certAsString = Files.readString(certFilePath); 30 | final String privateKeyAsString = Files.readString(pkFilePath); 31 | 32 | return new Certificate(certAsString, privateKeyAsString); 33 | } catch (final Exception ex) { 34 | LOG.error("Error encountered while reading the certificate.", ex); 35 | throw new RuntimeException(ex); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkConfigurationTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink.elasticsearch; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import org.junit.Test; 5 | 6 | import java.util.Collections; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | import static org.junit.Assert.assertNotNull; 12 | 13 | public class ElasticsearchSinkConfigurationTests { 14 | private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); 15 | private static final String PLUGIN_NAME = "elasticsearch"; 16 | private static final String PIPELINE_NAME = "integTestPipeline"; 17 | 18 | @Test 19 | public void testReadESConfig() { 20 | final ElasticsearchSinkConfiguration elasticsearchSinkConfiguration = ElasticsearchSinkConfiguration.readESConfig( 21 | generatePluginSetting()); 22 | assertNotNull(elasticsearchSinkConfiguration.getConnectionConfiguration()); 23 | assertNotNull(elasticsearchSinkConfiguration.getIndexConfiguration()); 24 | assertNotNull(elasticsearchSinkConfiguration.getRetryConfiguration()); 25 | } 26 | 27 | private PluginSetting generatePluginSetting() { 28 | final Map metadata = new HashMap<>(); 29 | metadata.put(IndexConfiguration.TRACE_ANALYTICS_RAW_FLAG, true); 30 | metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); 31 | 32 | final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); 33 | pluginSetting.setPipelineName(PIPELINE_NAME); 34 | return pluginSetting; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /data-prepper-api/src/main/java/com/amazon/dataprepper/model/sink/AbstractSink.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.model.sink; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.metrics.MetricNames; 5 | import com.amazon.dataprepper.metrics.PluginMetrics; 6 | import com.amazon.dataprepper.model.record.Record; 7 | import java.util.Collection; 8 | import io.micrometer.core.instrument.Counter; 9 | import io.micrometer.core.instrument.Timer; 10 | 11 | /** 12 | * This class implements the Sink interface and records boilerplate metrics 13 | */ 14 | public abstract class AbstractSink> implements Sink { 15 | protected final PluginMetrics pluginMetrics; 16 | private final Counter recordsInCounter; 17 | private final Timer timeElapsedTimer; 18 | 19 | public AbstractSink(final PluginSetting pluginSetting) { 20 | this.pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); 21 | recordsInCounter = pluginMetrics.counter(MetricNames.RECORDS_IN); 22 | timeElapsedTimer = pluginMetrics.timer(MetricNames.TIME_ELAPSED); 23 | } 24 | 25 | /** 26 | * Records metrics for ingress and time elapsed, while calling 27 | * doOutput to perform the actual output logic 28 | * @param records the records to write to the sink. 29 | */ 30 | @Override 31 | public void output(Collection records) { 32 | recordsInCounter.increment(records.size()*1.0); 33 | timeElapsedTimer.record(() -> doOutput(records)); 34 | } 35 | 36 | /** 37 | * This method should implement the output logic 38 | * @param records Records to be output 39 | */ 40 | public abstract void doOutput(Collection records); 41 | } 42 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/sink/SinkFactoryTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.sink; 2 | 3 | import com.amazon.dataprepper.model.configuration.PluginSetting; 4 | import com.amazon.dataprepper.model.sink.Sink; 5 | import com.amazon.dataprepper.plugins.PluginException; 6 | import org.junit.Test; 7 | 8 | import java.util.HashMap; 9 | 10 | import static com.amazon.dataprepper.plugins.PluginFactoryTests.NON_EXISTENT_EMPTY_CONFIGURATION; 11 | import static org.hamcrest.CoreMatchers.equalTo; 12 | import static org.hamcrest.CoreMatchers.is; 13 | import static org.hamcrest.CoreMatchers.notNullValue; 14 | import static org.hamcrest.MatcherAssert.assertThat; 15 | import static org.junit.Assert.assertThrows; 16 | 17 | @SuppressWarnings("rawtypes") 18 | public class SinkFactoryTests { 19 | 20 | /** 21 | * Tests if SinkFactory is able to retrieve default Source plugins by name 22 | */ 23 | @Test 24 | public void testNewSinkClassByNameThatExists() { 25 | final PluginSetting stdOutSinkConfiguration = new PluginSetting("stdout", new HashMap<>()); 26 | final Sink actualSink = SinkFactory.newSink(stdOutSinkConfiguration); 27 | final Sink expectedSink = new StdOutSink(); 28 | assertThat(actualSink, notNullValue()); 29 | assertThat(actualSink.getClass().getSimpleName(), is(equalTo(expectedSink.getClass().getSimpleName()))); 30 | } 31 | 32 | /** 33 | * Tests if SinkFactory fails with correct Exception when queried for a non-existent plugin 34 | */ 35 | @Test 36 | public void testNonExistentSinkRetrieval() { 37 | assertThrows(PluginException.class, () -> SinkFactory.newSink(NON_EXISTENT_EMPTY_CONFIGURATION)); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /data-prepper-plugins/common/src/test/java/com/amazon/dataprepper/plugins/buffer/BufferFactoryTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.buffer; 2 | 3 | import com.amazon.dataprepper.model.buffer.Buffer; 4 | import com.amazon.dataprepper.model.configuration.PluginSetting; 5 | import com.amazon.dataprepper.plugins.PluginException; 6 | import org.junit.Test; 7 | 8 | import java.util.HashMap; 9 | 10 | import static com.amazon.dataprepper.plugins.PluginFactoryTests.NON_EXISTENT_EMPTY_CONFIGURATION; 11 | import static org.hamcrest.CoreMatchers.equalTo; 12 | import static org.hamcrest.CoreMatchers.is; 13 | import static org.hamcrest.CoreMatchers.notNullValue; 14 | import static org.hamcrest.MatcherAssert.assertThat; 15 | import static org.junit.Assert.assertThrows; 16 | 17 | @SuppressWarnings("rawtypes") 18 | public class BufferFactoryTests { 19 | /** 20 | * Tests if BufferFactory is able to retrieve default Source plugins by name 21 | */ 22 | @Test 23 | public void testNewBufferClassByNameThatExists() { 24 | final PluginSetting pluginSetting = new PluginSetting("test_buffer", new HashMap<>()); 25 | final Buffer testBuffer = BufferFactory.newBuffer(pluginSetting); 26 | final Buffer expectedBuffer = new TestBuffer(pluginSetting); 27 | assertThat(testBuffer, notNullValue()); 28 | assertThat(testBuffer.getClass().getSimpleName(), is(equalTo(expectedBuffer.getClass().getSimpleName()))); 29 | } 30 | 31 | /** 32 | * Tests if BufferFactory fails with correct Exception when queried for a non-existent plugin 33 | */ 34 | @Test 35 | public void testNonExistentSinkRetrieval() { 36 | assertThrows(PluginException.class, () -> BufferFactory.newBuffer(NON_EXISTENT_EMPTY_CONFIGURATION)); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | workflow_dispatch: 11 | 12 | jobs: 13 | integration_tests: 14 | strategy: 15 | matrix: 16 | java: [14] 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - name: Set up JDK ${{ matrix.java }} 22 | uses: actions/setup-java@v1 23 | with: 24 | java-version: ${{ matrix.java }} 25 | - name: Checkout Data-Prepper 26 | uses: actions/checkout@v2 27 | - name: Grant execute permission for gradlew 28 | run: chmod +x gradlew 29 | - name: Run ODFE docker 30 | run: | 31 | export version=1.12.0 32 | docker pull amazon/opendistro-for-elasticsearch:$version 33 | docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version 34 | sleep 90 35 | - name: Run ODFE tests 36 | run: | 37 | ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin 38 | ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin -------------------------------------------------------------------------------- /.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Data Prepper elasticsearch sink integration tests with ODFE >= 1.13.0 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | workflow_dispatch: 11 | 12 | jobs: 13 | integration_tests: 14 | strategy: 15 | matrix: 16 | java: [14] 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - name: Set up JDK ${{ matrix.java }} 22 | uses: actions/setup-java@v1 23 | with: 24 | java-version: ${{ matrix.java }} 25 | - name: Checkout Data-Prepper 26 | uses: actions/checkout@v2 27 | - name: Grant execute permission for gradlew 28 | run: chmod +x gradlew 29 | - name: Run ODFE docker 30 | run: | 31 | export version=1.13.2 32 | docker pull amazon/opendistro-for-elasticsearch:$version 33 | docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version 34 | sleep 90 35 | - name: Run ODFE tests 36 | run: | 37 | ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin 38 | ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin 39 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file; 2 | 3 | import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.mockito.junit.MockitoJUnitRunner; 7 | 8 | import java.io.IOException; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | 12 | import static org.hamcrest.CoreMatchers.is; 13 | import static org.hamcrest.MatcherAssert.assertThat; 14 | 15 | @RunWith(MockitoJUnitRunner.class) 16 | public class FileCertificateProviderTest { 17 | 18 | private FileCertificateProvider fileCertificateProvider; 19 | 20 | @Test 21 | public void getCertificateValidPathSuccess() throws IOException { 22 | final String certificateFilePath = FileCertificateProviderTest.class.getClassLoader().getResource("test-crt.crt").getPath(); 23 | 24 | fileCertificateProvider = new FileCertificateProvider(certificateFilePath); 25 | 26 | final Certificate certificate = fileCertificateProvider.getCertificate(); 27 | 28 | final Path certFilePath = Path.of(certificateFilePath); 29 | final String certAsString = Files.readString(certFilePath); 30 | 31 | assertThat(certificate.getCertificate(), is(certAsString)); 32 | } 33 | 34 | @Test(expected = RuntimeException.class) 35 | public void getCertificateInvalidPathSuccess() { 36 | final String certificateFilePath = "path_does_not_exit/test_cert.crt"; 37 | 38 | fileCertificateProvider = new FileCertificateProvider(certificateFilePath); 39 | 40 | fileCertificateProvider.getCertificate(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/src/test/resources/test-crt.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEpDCCAowCCQDSDW5G0pixKjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls 3 | b2NhbGhvc3QwHhcNMjAxMjE1MjIyMDM0WhcNMjEwMTE0MjIyMDM0WjAUMRIwEAYD 4 | VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQD4 5 | g1JdRcOoNEfsBFZmfc1Rk1vRONWPZ9eKjs5tjCoLFTmrpVOBtJDlvdhGTQdxH62f 6 | CAThJVfK4dWnLS+n9Zc+EYQTKKBZViResaZtdZmWDpXxy99uhBbqH0v4lbKPMxcd 7 | 1iGVIzdZS+IyaJ9oFZB5Tzcaq2Hai5XHEJyXD+lpt+wIQssjHZqeIjr/1eCOaPe/ 8 | 4jDFKst71zeitifA2Y1tv/voysWUAzfiZMmFSuqGfa8UFqIvNNLLH0HHBpxQ1sqW 9 | sS8XcxnVmRvxVeqZ4VjYdK0SxIuqim7SHa/leGxpu1DchJurS9kAco5Be4JLGrh2 10 | WeLKvBW/1cQsiIAyggkLRmPZWiol+2wsMcWbWTDwGKK2p7Ny8j0GMoSfrqKgL3Li 11 | yXMsLdP9welDwStcJLo4XS69Z5HA48K7d68PV1+/m3MBjsCxPn3Uf0LCFcIq8+/c 12 | CFjp0Y+rz1naPdhAvzQmPD276QLCYJlAoU1iEb3fQkAOOduShHJddq9X/hHHHHc7 13 | ArUJcyF1EIaOFO5XenskGFoZOc2qBcrJ1NQPO5yd1xQj5NACW2EjEwtSciMJLJXN 14 | yNPffEY2WwWc7gmpD1gJpaFKj63UwyYRnySYO/pFCgwdUGrs6NIRp1TS7/NrvhID 15 | Mbz34bMSot4eAJqZFLdloJVCjcyc/sAhNW8zBr7O9wIDAQABMA0GCSqGSIb3DQEB 16 | CwUAA4ICAQA+pB11FWyKz0ezfXbqDbYlR5OuGxwFFkhLTSj1+zXYA6aIDVqdEyG2 17 | FfHCMYEbWUUJJ8Bk+qWw6U/6pu1l+j07kcEHKMXBJifrWnWsj/qVOrSyjnALJa7h 18 | Jsz44eLjgEFmtdvMWOMam98/Tb2DddUn8jd/7AAzz9VhwA4xKW6dPj2ZFyO75UHp 19 | uccxypF9uVcNvNRnNSvq3qZHDwRzBGhTSHp6juFJrSSDi16N3SATJO22KYZV9gpM 20 | veRJowYuK0urWgaiOmcbDxMWP8moDGmLIwbk+IqT1vt1lHPA/9G+QiGRPkyEnbpz 21 | 4TyBkns/BLT+Wmb2F+m1iytoXqVgmq2YzQ7OHzar8zjJFNLEGlPVrEVgDWyWNQZW 22 | pvN/BEaZChNPnYgoxah7DIx19ug2bFPxyBXiJR6q0eDecAmwf7b4jA4+BuEOj+lc 23 | 0pH+1GO4URAzoUdZoXYs2EJjPaIUA6LHXBzo42MKFYl80aU6vnzrMztftnouRrZK 24 | Uzd+s862wTug4uJqYRSYdO0O8tCPMw+rjMciRI7LZ83b4r1hJDhC9WYJSPd5K7Q/ 25 | IkmNfeSTPkHNcma04NauWOm+Wx5n0V9YaaTSHYD5ge+Bk1e+DRrlcmav79W7tbkD 26 | KayBf/TxC7ZwRRL85kLDJCmDG1Tda2bjdLz+3vtVVgw9Jxv7wEjXZA== 27 | -----END CERTIFICATE----- 28 | -------------------------------------------------------------------------------- /examples/demo/demo-data-prepper.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDBuSTI3oz9ZJRi 3 | VQCdTfbQlG9TWCXwuIjjJcVwvgVjhIuj5VLEOWUpC8nrbe4/3ttGLuqOBViLlw2a 4 | elN7ZANgFy33ONF7kDh4xFMp4rZdFKxCOsmUpbSD7hRf8mVRFv30bx9//bVjLP8q 5 | i2QKuHc8WOevMsdNENFI6BgiI52EoVPyYJIIMaE2ripGqkQVn04Tk+rWz3zQ+NyO 6 | cExow98+pxMIjAanMUELTpMjQCWq2HtV2nnqcfqqNCaeig7NSDRvQtPL84Fjy7y4 7 | B1pS558hBmfOyZcGHo/gxLxQmvb1UKy4bcL7Kd8H3aNrLZ8zsr+he3cM9a7SZbii 8 | lALdcv0NAgMBAAECggEAd0rwmPVa8usN6OOZRkTtWRIe377nVi38tYvb4opRPTAw 9 | 4VNlPUpuvfDxfZWrYg91Den0LFKkbQkuqlLrt48l+PUtLBEEBsgPzmZvI9rlG7qQ 10 | PRSAb5X+7EMertrQgzfaWLxZNHLiUNkzpFpD18oyOqowuZVWX2Gczv2FZvhxlmrB 11 | fCajh2k4kYfFfedA8xpmUr1ZXRFsVCioTZswovN7CjNqPdNgjTho62mt62XdVTn1 12 | KpGwCquJLHfttG71qkRhhAaOqgPv+egbxAxbRx5dy2PHnKB3jl68LLXDAog/deEs 13 | Px39CoGiNT6v0TDPmxKY3Btns0Mf97GW1JxKJz/GIQKBgQD9ZimNv03tVE9Hfg9a 14 | TwgBzDmwQeAc+70g6hdtxTMp0CWWkgx/Pdb9tN9tfCTEA7/TeRYV2YwxMRgAChPK 15 | 8NfPrB8ctq6IlsjANtiJ0OqCARFsuMc2oA0ecosA9Dt5IRz/9r+mjpaDvCWv2s7D 16 | D6yK7WlcY9mIOAdYPeVMvwTXhQKBgQDDtiz1SeSvn4Jboi2zNspkJ4K/7gXdj73b 17 | tmQVT8mVZKuxOCOFqC8e69/KcnOu3sOWowFVkbCmQswYmMiwvm2d2APfk1wC1+i4 18 | cGkBspzvl/N9dyBCU2QwaHW0Op/jva3HIWm6dCh0D6W5+r7BEWE3AGfENW0/4E6q 19 | v8qv7j8R6QKBgG7Id2pNNNiJf4+WAUhOPQXsL1xrsHt7Y5g8P7nx+QhSw9wsReJr 20 | Os74gKXxHHHF4dummI086xmmiQObZEPZP8C/U2rOQ0fcBDBkyg6Wk6/6n4+UySOM 21 | na+uDC+Cl23BmFgfnHJI7GzzmNZlJvgIpjqmyOVnkfcH584gYVtIN+mxAoGAddFM 22 | 3Vlfly5CTqGiO1oHmyWB+8LxuwDWfdceBwpiQ1I4XdZ64/Jsc4Px1DD0XVhzjFKS 23 | 8g1jt6NMtJ39axEocjZhlo1ifb+2tJnPTRtWiJIEHgrB28vR66Zm/VmxeWttmwwN 24 | VxaoZ74nuX8uqJGq4W6YCTFUlHiXQwl0nlahl0kCgYEAjkKqOzYDr/1/HalLdyct 25 | AnkTGqPRA+ml888VMlgmGss1EpFvxHTxWNpSfBoF9jRY2eO0N2yMwzvRqFjiA4WV 26 | bLKUTD6E3t12XEZPIpZA5UamWvZDaN3pXkONMZN+4+F1r+j9MB/626dPqBUDpq3V 27 | G+yZxerQdvFvNby4HJxkXSM= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /data-prepper-plugins/peer-forwarder/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.amazon' 6 | version '0.1.0' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation project(':data-prepper-api') 14 | testImplementation project(':data-prepper-api').sourceSets.test.output 15 | implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" 16 | implementation "com.linecorp.armeria:armeria:1.9.2" 17 | implementation "com.linecorp.armeria:armeria-grpc:1.9.2" 18 | implementation(platform('software.amazon.awssdk:bom:2.17.15')) 19 | implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" 20 | implementation "com.amazonaws:aws-java-sdk-acm:1.12.43" 21 | implementation 'software.amazon.awssdk:servicediscovery' 22 | implementation "commons-io:commons-io:2.11.0" 23 | implementation "org.apache.commons:commons-lang3:3.12.0" 24 | implementation "commons-validator:commons-validator:1.7" 25 | testImplementation(platform('org.junit:junit-bom:5.7.2')) 26 | testImplementation 'org.junit.jupiter:junit-jupiter' 27 | testImplementation 'org.junit.vintage:junit-vintage-engine' 28 | testImplementation "org.hamcrest:hamcrest:2.2" 29 | testImplementation "org.mockito:mockito-inline:3.11.2" 30 | testImplementation "org.mockito:mockito-core:3.11.2" 31 | testImplementation 'org.mockito:mockito-junit-jupiter:3.11.2' 32 | testImplementation "commons-io:commons-io:2.10.0" 33 | testImplementation 'org.awaitility:awaitility:4.1.0' 34 | } 35 | 36 | test { 37 | useJUnitPlatform() 38 | } 39 | 40 | jacocoTestCoverageVerification { 41 | dependsOn jacocoTestReport 42 | violationRules { 43 | rule { 44 | limit { 45 | minimum = 0.90 46 | } 47 | } 48 | } 49 | } --------------------------------------------------------------------------------