├── java ├── amazon-kinesis-producer-sample │ ├── .gitignore │ ├── README.md │ ├── src │ │ └── software │ │ │ └── amazon │ │ │ └── kinesis │ │ │ └── producer │ │ │ └── sample │ │ │ └── Utils.java │ └── pom.xml └── amazon-kinesis-producer │ ├── src │ ├── test │ │ ├── resources │ │ │ ├── test-data │ │ │ │ └── test.txt │ │ │ └── logback-test.xml │ │ └── java │ │ │ └── software │ │ │ └── amazon │ │ │ └── kinesis │ │ │ └── producer │ │ │ ├── BinaryToHexConverterTest.java │ │ │ ├── GlueSchemaRegistrySerializerInstanceTest.java │ │ │ └── FileAgeManagerTest.java │ └── main │ │ ├── java │ │ └── software │ │ │ └── amazon │ │ │ └── kinesis │ │ │ └── producer │ │ │ ├── ProcessFailureBehavior.java │ │ │ ├── BinaryToHexConverter.java │ │ │ ├── DaemonException.java │ │ │ ├── IrrecoverableError.java │ │ │ ├── FutureTimedOutException.java │ │ │ ├── UnexpectedMessageException.java │ │ │ ├── UserRecordFailedException.java │ │ │ ├── IKinesisProducer.java │ │ │ ├── KinesisProducerException.java │ │ │ ├── GlueSchemaRegistrySerializerInstance.java │ │ │ ├── FileAgeManager.java │ │ │ ├── Attempt.java │ │ │ ├── UserRecordResult.java │ │ │ ├── Metric.java │ │ │ └── UserRecord.java │ │ └── resources │ │ └── cacerts │ │ ├── 8cb5ee0f.0 │ │ ├── b0e59380.0 │ │ ├── de6d66f3.0 │ │ ├── 1d3472b9.0 │ │ ├── dd8e9d41.0 │ │ ├── 7f3d5d1d.0 │ │ ├── c089bbbd.0 │ │ ├── 116bf586.0 │ │ ├── ce5e74ef.0 │ │ ├── 2c543cd1.0 │ │ ├── 062cdee6.0 │ │ ├── cbeee9e2.0 │ │ ├── 5ad8a5d6.0 │ │ ├── 653b494a.0 │ │ ├── 480720ec.0 │ │ ├── 7d0b38bd.0 │ │ ├── 607986c7.0 │ │ ├── 9d04f354.0 │ │ ├── 76cb8f92.0 │ │ ├── 3513523f.0 │ │ ├── b1159c4c.0 │ │ ├── 4a6481c9.0 │ │ ├── 244b5494.0 │ │ ├── cbf06781.0 │ │ ├── 4bfab552.0 │ │ ├── 09789157.0 │ │ ├── e2799e36.0 │ │ ├── f081611a.0 │ │ ├── f387163d.0 │ │ ├── c0ff1f52.0 │ │ ├── 2e4eed3c.0 │ │ ├── ba89ed3b.0 │ │ ├── c01cdfa2.0 │ │ ├── 6d41d539.0 │ │ ├── ad088e1d.0 │ │ ├── 8867006a.0 │ │ ├── 75d1b2ed.0 │ │ └── b204d74a.0 │ └── .gitignore ├── .github ├── dependabot.yml └── PULL_REQUEST_TEMPLATE.md ├── CODE_OF_CONDUCT.md ├── .gitignore ├── aws ├── utils │ ├── timed_death.h │ ├── segfault_signal.h │ ├── segfault_signal.cc │ ├── signal_handler.h │ ├── backtrace │ │ ├── backtrace.h │ │ ├── null_backtrace.cc │ │ └── bsd_backtrace.cc │ ├── writer_methods.h │ ├── timed_death.cc │ ├── spin_lock.cc │ ├── writer_methods.cc │ ├── concurrent_linked_queue.h │ ├── logging.h │ ├── test │ │ ├── concurrent_hash_map_test.cc │ │ ├── token_bucket_test.cc │ │ └── spin_lock_test.cc │ ├── executor.h │ ├── concurrent_hash_map.h │ ├── time_sensitive_queue.h │ ├── time_sensitive.h │ └── token_bucket.h ├── kinesis │ ├── test │ │ └── test.cc │ ├── core │ │ ├── put_records_request.h │ │ ├── kinesis_record.h │ │ ├── test │ │ │ ├── test_utils.h │ │ │ └── put_records_request_test.cc │ │ ├── user_record.cc │ │ ├── attempt.h │ │ ├── serializable_container.h │ │ ├── user_record.h │ │ ├── collector.h │ │ ├── put_records_context.h │ │ ├── ipc_manager.cc │ │ ├── aggregator.h │ │ └── kinesis_producer.h │ └── protobuf │ │ ├── config.proto │ │ └── messages.proto ├── metrics │ ├── metrics_header.h │ ├── metrics_index.h │ ├── metric.h │ ├── metrics_finder.h │ ├── metrics_index.cc │ └── metrics_constants.h ├── auth │ ├── mutable_static_creds_provider.h │ └── mutable_static_creds_provider.cc └── mutex.h ├── aggregation-format.md └── CONTRIBUTING.md /java/amazon-kinesis-producer-sample/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | *.iml 3 | .idea/ -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/test/resources/test-data/test.txt: -------------------------------------------------------------------------------- 1 | This is a test 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "maven" 4 | directory: "/java/amazon-kinesis-producer" 5 | schedule: 6 | interval: "weekly" 7 | 8 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/ProcessFailureBehavior.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | public enum ProcessFailureBehavior { 4 | Shutdown, AutoRestart 5 | } 6 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. 7 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | 3 | # Eclipse 4 | .cache* 5 | .classpath 6 | .project 7 | .scala_dependencies 8 | .settings 9 | .target 10 | .worksheet 11 | 12 | # IntelliJ 13 | .idea 14 | *.iml 15 | 16 | # ENSIME 17 | .ensime 18 | .ensime_lucene 19 | .ensime_cache 20 | 21 | # Mac 22 | .DS_Store 23 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /b2 2 | /bin 3 | /third_party 4 | /java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/**/* 5 | 6 | # VS Code 7 | .vscode/ 8 | 9 | # Autogenerated protobuf 10 | /aws/kinesis/protobuf/config.pb.cc 11 | /aws/kinesis/protobuf/config.pb.h 12 | /aws/kinesis/protobuf/messages.pb.cc 13 | /aws/kinesis/protobuf/messages.pb.h -------------------------------------------------------------------------------- /aws/utils/timed_death.h: -------------------------------------------------------------------------------- 1 | #ifndef TIMED_DEATH_H 2 | #define TIMED_DEATH_H 3 | 4 | #include 5 | 6 | 7 | namespace aws { 8 | namespace utils { 9 | class TimedDeath 10 | { 11 | private: 12 | TimedDeath() = default; 13 | public: 14 | 15 | static void initialize(time_t min_wait); 16 | static void die_if_time(); 17 | 18 | }; 19 | 20 | } 21 | } 22 | 23 | #endif // TIMEDDEATH_H 24 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/test/java/software/amazon/kinesis/producer/BinaryToHexConverterTest.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertEquals; 6 | 7 | public class BinaryToHexConverterTest { 8 | 9 | @Test 10 | public void testConvert() { 11 | byte[] bytes = {0, 1, 2, 3, 124, 125, 126, 127}; 12 | assertEquals("000102037C7D7E7F", BinaryToHexConverter.convert(bytes)); 13 | } 14 | 15 | } -------------------------------------------------------------------------------- /aws/utils/segfault_signal.h: -------------------------------------------------------------------------------- 1 | #ifndef SEGFAULT_SIGNAL_H 2 | #define SEGFAULT_SIGNAL_H 3 | 4 | #ifdef SEGFAULT_SIGNAL_ENABLED 5 | 6 | #define CHECK_SEGFAULT_STATUS() ::aws::utils::trigger_segfault_if_needed(); 7 | #define INSTALL_SEGFAULT_TRIGGER() ::aws::utils::install_user_handler(); 8 | 9 | #include 10 | 11 | namespace aws { 12 | namespace utils { 13 | 14 | void trigger_segfault_if_needed(); 15 | void install_user_handler(); 16 | 17 | } 18 | } 19 | #else 20 | #define CHECK_SEGFAULT_STATUS() 21 | #define INSTALL_SEGFAULT_TRIGGER() 22 | #endif 23 | 24 | #endif // SEGFAULT_SIGNAL_H 25 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/8cb5ee0f.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5 3 | MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g 4 | Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG 5 | A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg 6 | Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl 7 | ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j 8 | QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr 9 | ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr 10 | BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM 11 | YyRIHN8wfdVoOw== 12 | -----END CERTIFICATE----- 13 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/b0e59380.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIB4TCCAYegAwIBAgIRKjikHJYKBN5CsiilC+g0mAIwCgYIKoZIzj0EAwIwUDEk 3 | MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI0MRMwEQYDVQQKEwpH 4 | bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX 5 | DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD 6 | QSAtIFI0MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu 7 | MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuMZ5049sJQ6fLjkZHAOkrprlOQcJ 8 | FspjsbmG+IpXwVfOQvpzofdlQv8ewQCybnMO/8ch5RikqtlxP6jUuc6MHaNCMEAw 9 | DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFFSwe61F 10 | uOJAf/sKbvu+M8k8o4TVMAoGCCqGSM49BAMCA0gAMEUCIQDckqGgE6bPA7DmxCGX 11 | kPoUVy0D7O48027KqGx2vKLeuwIgJ6iFJzWbVsaj8kfSt24bAgAXqmemFZHe+pTs 12 | ewv4n4Q= 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/de6d66f3.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIB8jCCAXigAwIBAgITBmyf18G7EEwpQ+Vxe3ssyBrBDjAKBggqhkjOPQQDAzA5 3 | MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g 4 | Um9vdCBDQSA0MB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG 5 | A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg 6 | Q0EgNDB2MBAGByqGSM49AgEGBSuBBAAiA2IABNKrijdPo1MN/sGKe0uoe0ZLY7Bi 7 | 9i0b2whxIdIA6GO9mif78DluXeo9pcmBqqNbIJhFXRbb/egQbeOc4OO9X4Ri83Bk 8 | M6DLJC9wuoihKqB1+IGuYgbEgds5bimwHvouXKNCMEAwDwYDVR0TAQH/BAUwAwEB 9 | /zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFNPsxzplbszh2naaVvuc84ZtV+WB 10 | MAoGCCqGSM49BAMDA2gAMGUCMDqLIfG9fhGt0O9Yli/W651+kI0rz2ZVwyzjKKlw 11 | CkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1AE47xDqUEpHJWEadIRNyp4iciuRMStuW 12 | 1KyLa2tJElMzrdfkviT8tQp21KW8EA== 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /aws/utils/segfault_signal.cc: -------------------------------------------------------------------------------- 1 | #include "segfault_signal.h" 2 | #include 3 | #include 4 | #include 5 | 6 | #ifdef SEGFAULT_SIGNAL_ENABLED 7 | namespace { 8 | 9 | std::atomic segfault_signaled_(false); 10 | 11 | void user_signal_handler(int, siginfo_t *info, void *) { 12 | segfault_signaled_.store(true); 13 | } 14 | 15 | volatile int* p = (int*)0; 16 | 17 | } 18 | 19 | namespace aws { 20 | namespace utils { 21 | void trigger_segfault_if_needed() { 22 | if (segfault_signaled_.load()) { 23 | LOG(info) << "Triggering segfault due to signal."; 24 | *p = 10; 25 | } 26 | } 27 | 28 | void install_user_handler() { 29 | struct sigaction action; 30 | action.sa_sigaction = &user_signal_handler; 31 | action.sa_flags |= SA_SIGINFO; 32 | 33 | sigaction(SIGUSR1, &action, NULL); 34 | } 35 | 36 | } 37 | } 38 | #endif 39 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/1d3472b9.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEk 3 | MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpH 4 | bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX 5 | DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD 6 | QSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu 7 | MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6SFkc 8 | 8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8ke 9 | hOvRnkmSh5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD 10 | VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYI 11 | KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg 12 | 515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7yFz9SO8NdCKoCOJuxUnO 13 | xwy8p2Fp8fc74SrL+SvzZpA3 14 | -----END CERTIFICATE----- 15 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/dd8e9d41.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw 3 | CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu 4 | ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe 5 | Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw 6 | EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x 7 | IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF 8 | K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG 9 | fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO 10 | Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd 11 | BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx 12 | AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ 13 | oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 14 | sycX 15 | -----END CERTIFICATE----- 16 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/7f3d5d1d.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw 3 | CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu 4 | ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg 5 | RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV 6 | UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu 7 | Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq 8 | hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf 9 | Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q 10 | RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ 11 | BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD 12 | AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY 13 | JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv 14 | 6pZjamVFkpUBtA== 15 | -----END CERTIFICATE----- 16 | -------------------------------------------------------------------------------- /aws/kinesis/test/test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #define BOOST_TEST_MAIN 17 | #include 18 | 19 | #include 20 | 21 | struct AwsSdkInitFixture { 22 | AwsSdkInitFixture() { 23 | Aws::SDKOptions sdkOptions; 24 | Aws::InitAPI(sdkOptions); 25 | } 26 | }; 27 | 28 | BOOST_GLOBAL_FIXTURE(AwsSdkInitFixture); -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/BinaryToHexConverter.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | public class BinaryToHexConverter { 4 | 5 | private static final BinaryToHexConverter INSTANCE = new BinaryToHexConverter(); 6 | private static final char[] HEX_CODE = "0123456789ABCDEF".toCharArray(); 7 | 8 | /** 9 | * Converts an array of bytes into a hex string. 10 | * 11 | * @param data An array of bytes 12 | * @return A string containing a lexical representation of xsd:hexBinary 13 | * @throws IllegalArgumentException if {@code data} is null. 14 | */ 15 | public static String convert(byte[] data) { 16 | return INSTANCE.convertToHex(data); 17 | } 18 | 19 | private String convertToHex(byte[] data) { 20 | StringBuilder r = new StringBuilder(data.length * 2); 21 | for (byte b : data) { 22 | r.append(HEX_CODE[(b >> 4) & 0xF]); 23 | r.append(HEX_CODE[(b & 0xF)]); 24 | } 25 | return r.toString(); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /aws/utils/signal_handler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AMAZON_KINESIS_PRODUCER_INTERNAL_SIGNAL_HANDLER_H 17 | #define AMAZON_KINESIS_PRODUCER_INTERNAL_SIGNAL_HANDLER_H 18 | 19 | namespace aws { 20 | namespace utils { 21 | void setup_stack_trace(const char *exe); 22 | } 23 | } 24 | 25 | #endif //AMAZON_KINESIS_PRODUCER_INTERNAL_SIGNAL_HANDLER_H 26 | -------------------------------------------------------------------------------- /aws/utils/backtrace/backtrace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef VARIED_BACKTRACE_H 17 | #define VARIED_BACKTRACE_H 18 | 19 | namespace aws { 20 | namespace utils { 21 | namespace backtrace { 22 | 23 | void initialize(const char* exe); 24 | void stack_trace_for_signal(int skip = 0, bool signaled = true); 25 | 26 | } 27 | } 28 | } 29 | 30 | #endif // VARIED_BACKTRACE_H 31 | -------------------------------------------------------------------------------- /aws/utils/writer_methods.h: -------------------------------------------------------------------------------- 1 | #ifndef WRITER_METHODS_H 2 | #define WRITER_METHODS_H 3 | 4 | #include 5 | #include 6 | 7 | namespace aws { 8 | namespace utils { 9 | namespace writer { 10 | 11 | void write_number_checked(int number, const char *error_message, size_t error_message_size); 12 | void write_signal_descriptiong(int signal); 13 | void write_pointer(void *pointer); 14 | void write_number(uint64_t number); 15 | void write_message(const char* message, size_t message_size); 16 | 17 | } 18 | } 19 | } 20 | 21 | 22 | #define WRITE_NUM_CHECKED(number, err_msg) ::aws::utils::writer::write_number_checked(number, err_msg, sizeof(err_msg) - 1); 23 | #define WRITE_CODE(number) WRITE_NUM_CHECKED(number, "Code <= 0"); 24 | #define WRITE_POINTER(pointer) ::aws::utils::writer::write_pointer(pointer); 25 | #define WRITE_NUMBER(number) ::aws::utils::writer::write_number(number); 26 | #define WRITE_MESSAGE(message) ::aws::utils::writer::write_message(message, sizeof(message) - 1); 27 | 28 | #endif // WRITER_METHODS_H 29 | -------------------------------------------------------------------------------- /aws/metrics/metrics_header.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_METRICS_METRICS_HEADER_H 17 | #define AWS_METRICS_METRICS_HEADER_H 18 | 19 | #include 20 | 21 | namespace aws { 22 | namespace metrics { 23 | 24 | using Clock = std::chrono::steady_clock; 25 | using TimePoint = Clock::time_point; 26 | 27 | } // namespace metrics 28 | } // namespace aws 29 | 30 | #endif //AWS_METRICS_METRICS_HEADER_H 31 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/DaemonException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | public class DaemonException extends RuntimeException { 19 | private static final long serialVersionUID = -1161618354800585162L; 20 | 21 | public DaemonException(String message) { 22 | super(message); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/c089bbbd.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICiDCCAg2gAwIBAgIQNfwmXNmET8k9Jj1Xm67XVjAKBggqhkjOPQQDAzCBhDEL 3 | MAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjE4MDYGA1UECxMvKGMp 4 | IDIwMDcgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAi 5 | BgNVBAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMjAeFw0wNzExMDUwMDAw 6 | MDBaFw0zODAxMTgyMzU5NTlaMIGEMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhh 7 | d3RlLCBJbmMuMTgwNgYDVQQLEy8oYykgMjAwNyB0aGF3dGUsIEluYy4gLSBGb3Ig 8 | YXV0aG9yaXplZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9v 9 | dCBDQSAtIEcyMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEotWcgnuVnfFSeIf+iha/ 10 | BebfowJPDQfGAFG6DAJSLSKkQjnE/o/qycG+1E3/n3qe4rF8mq2nhglzh9HnmuN6 11 | papu+7qzcMBniKI11KOasf2twu8x+qi58/sIxpHR+ymVo0IwQDAPBgNVHRMBAf8E 12 | BTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUmtgAMADna3+FGO6Lts6K 13 | DPgR4bswCgYIKoZIzj0EAwMDaQAwZgIxAN344FdHW6fmCsO99YCKlzUNG4k8VIZ3 14 | KMqh9HneteY4sPBlcIx/AlTCv//YoT7ZzwIxAMSNlPzcU9LcnXgWHxUzI1NS41ox 15 | XZ3Krr0TKUQNJ1uo52icEvdYPy5yAlejj6EULg== 16 | -----END CERTIFICATE----- 17 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/116bf586.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICrjCCAjWgAwIBAgIQPLL0SAoA4v7rJDteYD7DazAKBggqhkjOPQQDAzCBmDEL 3 | MAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsTMChj 4 | KSAyMDA3IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTE2 5 | MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 6 | eSAtIEcyMB4XDTA3MTEwNTAwMDAwMFoXDTM4MDExODIzNTk1OVowgZgxCzAJBgNV 7 | BAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykgMjAw 8 | NyBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0BgNV 9 | BAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH 10 | MjB2MBAGByqGSM49AgEGBSuBBAAiA2IABBWx6P0DFUPlrOuHNxFi79KDNlJ9RVcL 11 | So17VDs6bl8VAsBQps8lL33KSLjHUGMcKiEIfJo22Av+0SbFWDEwKCXzXV2juLal 12 | tJLtbCyf691DiaI8S0iRHVDsJt/WYC69IaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO 13 | BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFBVfNVdRVfslsq0DafwBo/q+EVXVMAoG 14 | CCqGSM49BAMDA2cAMGQCMGSWWaboCd6LuvpaiIjwH5HTRqjySkwCY/tsXzjbLkGT 15 | qQ7mndwxHLKgpxgceeHHNgIwOlavmnRs9vuD4DPTCF+hnMJbn0bWtsuRBmOiBucz 16 | rD6ogRLQy7rQkgu2npaqBA+K 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /aws/utils/backtrace/null_backtrace.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "backtrace.h" 17 | #include 18 | #include 19 | #include 20 | 21 | #ifdef NULL_STACKTRACE 22 | 23 | namespace aws { 24 | namespace utils { 25 | namespace backtrace { 26 | 27 | void initialize(const char* exe) { 28 | 29 | } 30 | 31 | void stack_trace_for_signal(int skip, bool /*signaled*/) { 32 | WRITE_MESSAGE("Backtrace not supported."); 33 | } 34 | 35 | } 36 | } 37 | } 38 | 39 | #endif // NULL_STACKTRACE 40 | -------------------------------------------------------------------------------- /aws/utils/timed_death.cc: -------------------------------------------------------------------------------- 1 | #include "timed_death.h" 2 | 3 | #include 4 | 5 | #include 6 | 7 | namespace { 8 | 9 | time_t die_after = 0; 10 | time_t base_time = 0; 11 | time_t random_offset = 0; 12 | time_t init_time = 0; 13 | 14 | volatile time_t* current = 0; 15 | 16 | } 17 | 18 | namespace aws { 19 | namespace utils { 20 | 21 | void TimedDeath::initialize(time_t min_wait) 22 | { 23 | 24 | std::random_device rd; 25 | std::mt19937 mt(rd()); 26 | std::uniform_int_distribution uniform(1, min_wait); 27 | random_offset = uniform(mt); 28 | time_t now = time(nullptr); 29 | die_after = now + min_wait + random_offset; 30 | 31 | LOG(info) << "now: " << now << " + random: " << random_offset << " = die_after: " << die_after << " ( " << (die_after - now) << " seconds )"; 32 | init_time = now; 33 | } 34 | 35 | void TimedDeath::die_if_time() 36 | { 37 | time_t now = time(nullptr); 38 | if (now > die_after) { 39 | time_t taken = now - init_time; 40 | LOG(info) << "Time to die: " << now << " > " << die_after << " taken: " << taken << " seconds."; 41 | *current = now; 42 | } 43 | } 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /aws/utils/spin_lock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "spin_lock.h" 17 | 18 | using namespace aws::utils; 19 | #ifdef DEBUG 20 | namespace { 21 | thread_local TicketSpinLock::DebugStats debug_stats; 22 | } 23 | 24 | void TicketSpinLock::add_acquired() { 25 | debug_stats.acquired_count++; 26 | } 27 | 28 | void TicketSpinLock::add_acquired_lock() { 29 | debug_stats.acquired_with_lock++; 30 | } 31 | 32 | void TicketSpinLock::add_spin() { 33 | debug_stats.total_spins++; 34 | } 35 | 36 | TicketSpinLock::DebugStats TicketSpinLock::get_debug_stats() { 37 | return debug_stats; 38 | } 39 | #endif 40 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/IrrecoverableError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | public class IrrecoverableError extends RuntimeException { 19 | private static final long serialVersionUID = 2657526423645068574L; 20 | 21 | public IrrecoverableError(String message) { 22 | super(message); 23 | } 24 | 25 | public IrrecoverableError(Throwable t) { 26 | super(t); 27 | } 28 | 29 | public IrrecoverableError(String message, Throwable t) { 30 | super(message, t); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/ce5e74ef.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF 3 | ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 4 | b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL 5 | MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv 6 | b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj 7 | ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM 8 | 9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw 9 | IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 10 | VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L 11 | 93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm 12 | jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC 13 | AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA 14 | A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI 15 | U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs 16 | N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv 17 | o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU 18 | 5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy 19 | rqXRfboQnoZsG4q5WTP468SQvvG5 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/FutureTimedOutException.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | /* 4 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 5 | * Licensed under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance 7 | * with the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /** 19 | * The exception thrown when a UserRecord reaches the 20 | * {@link KinesisProducerConfiguration#setUserRecordTimeoutInMillis(long)} timeout without being put into Kinesis. 21 | */ 22 | public class FutureTimedOutException extends KinesisProducerException { 23 | private static final long serialVersionUID = 3168271192277927600L; 24 | 25 | public FutureTimedOutException(String message, UserRecord userRecord) { 26 | super(message, userRecord); 27 | } 28 | } 29 | 30 | 31 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/UnexpectedMessageException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | /** 19 | * The exception thrown when an unexpected message is read from the native layer. 20 | */ 21 | public class UnexpectedMessageException extends KinesisProducerException { 22 | private static final long serialVersionUID = 3168271192277927600L; 23 | 24 | public UnexpectedMessageException(String message) { 25 | super(message); 26 | } 27 | 28 | public UnexpectedMessageException(String message, UserRecord userRecord) { 29 | super(message, userRecord); 30 | } 31 | 32 | } 33 | 34 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/2c543cd1.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT 3 | MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i 4 | YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG 5 | EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg 6 | R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 7 | 9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq 8 | fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv 9 | iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU 10 | 1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ 11 | bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW 12 | MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA 13 | ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l 14 | uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn 15 | Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS 16 | tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF 17 | PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un 18 | hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV 19 | 5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/062cdee6.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G 3 | A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp 4 | Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 5 | MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG 6 | A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI 7 | hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 8 | RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT 9 | gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm 10 | KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd 11 | QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ 12 | XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw 13 | DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o 14 | LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU 15 | RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp 16 | jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK 17 | 6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX 18 | mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs 19 | Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH 20 | WD9f 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/cbeee9e2.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDZjCCAk6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBEMQswCQYDVQQGEwJVUzEW 3 | MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEdMBsGA1UEAxMUR2VvVHJ1c3QgR2xvYmFs 4 | IENBIDIwHhcNMDQwMzA0MDUwMDAwWhcNMTkwMzA0MDUwMDAwWjBEMQswCQYDVQQG 5 | EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEdMBsGA1UEAxMUR2VvVHJ1c3Qg 6 | R2xvYmFsIENBIDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDvPE1A 7 | PRDfO1MA4Wf+lGAVPoWI8YkNkMgoI5kF6CsgncbzYEbYwbLVjDHZ3CB5JIG/NTL8 8 | Y2nbsSpr7iFY8gjpeMtvy/wWUsiRxP89c96xPqfCfWbB9X5SJBri1WeR0IIQ13hL 9 | TytCOb1kLUCgsBDTOEhGiKEMuzozKmKY+wCdE1l/bztyqu6mD4b5BWHqZ38MN5aL 10 | 5mkWRxHCJ1kDs6ZgwiFAVvqgx306E+PsV8ez1q6diYD3Aecs9pYrEw15LNnA5IZ7 11 | S4wMcoKK+xfNAGw6EzywhIdLFnopsk/bHdQL82Y3vdj2V7teJHq4PIu5+pIaGoSe 12 | 2HSPqht/XvT+RSIhAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE 13 | FHE4NvICMVNHK266ZUapEBVYIAUJMB8GA1UdIwQYMBaAFHE4NvICMVNHK266ZUap 14 | EBVYIAUJMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQUFAAOCAQEAA/e1K6td 15 | EPx7srJerJsOflN4WT5CBP51o62sgU7XAotexC3IUnbHLB/8gTKY0UvGkpMzNTEv 16 | /NgdRN3ggX+d6YvhZJFiCzkIjKx0nVnZellSlxG5FntvRdOW2TF9AjYPnDtuzywN 17 | A0ZF66D0f0hExghAzN4bcLUprbqLOzRldRtxIR0sFAqwlpW41uryZfspuk/qkZN0 18 | abby/+Ea0AzRdoXLiiW9l14sbxWZJue2Kf8i7MkCx1YAzUm5s2x7UwQa4qjJqhIF 19 | I8LO57sEAszAR6LkxCkvW0VXiVHuPOtSCP8HNR6fNWpHSlaY0VqFH4z1Ir+rzoPz 20 | 4iIprn2DQKi6bA== 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /aws/utils/writer_methods.cc: -------------------------------------------------------------------------------- 1 | #include "writer_methods.h" 2 | 3 | #include 4 | 5 | #define DISPLAY_SIZE 21 6 | 7 | namespace aws { 8 | namespace utils { 9 | namespace writer { 10 | 11 | 12 | void write_number(uint64_t number) { 13 | char display[DISPLAY_SIZE]; 14 | for (int i = 0; i < DISPLAY_SIZE; ++i) { 15 | display[i] = ' '; 16 | } 17 | size_t pos = DISPLAY_SIZE - 1; 18 | do { 19 | uint64_t tail = number % 10; 20 | display[pos--] = (char) (tail + '0'); 21 | number = number / 10; 22 | } while (number > 0 && pos > 0); 23 | size_t offset = pos + 1; 24 | char* head = display + offset; 25 | size_t len = DISPLAY_SIZE - offset; 26 | write(STDERR_FILENO, head, len); 27 | } 28 | 29 | void write_number_checked(int number, const char *error_message, size_t error_message_size) { 30 | if (number <= 0) { 31 | write(STDERR_FILENO, error_message, error_message_size); 32 | } else { 33 | write_number((uint64_t) number); 34 | } 35 | } 36 | 37 | 38 | 39 | 40 | void write_pointer(void *pointer) { 41 | if (pointer == NULL) { 42 | WRITE_MESSAGE("NULL") 43 | } else { 44 | write_number((uint64_t) pointer); 45 | } 46 | } 47 | 48 | void write_message(const char* message, size_t size) { 49 | write(STDERR_FILENO, message, size); 50 | } 51 | 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/5ad8a5d6.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG 3 | A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv 4 | b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw 5 | MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i 6 | YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT 7 | aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ 8 | jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp 9 | xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp 10 | 1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG 11 | snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ 12 | U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 13 | 9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E 14 | BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B 15 | AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz 16 | yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE 17 | 38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP 18 | AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad 19 | DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME 20 | HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/653b494a.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ 3 | RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD 4 | VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX 5 | DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y 6 | ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy 7 | VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr 8 | mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr 9 | IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK 10 | mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu 11 | XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy 12 | dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye 13 | jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1 14 | BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3 15 | DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92 16 | 9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx 17 | jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0 18 | Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz 19 | ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS 20 | R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/480720ec.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDfDCCAmSgAwIBAgIQGKy1av1pthU6Y2yv2vrEoTANBgkqhkiG9w0BAQUFADBY 3 | MQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjExMC8GA1UEAxMo 4 | R2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEx 5 | MjcwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMFgxCzAJBgNVBAYTAlVTMRYwFAYDVQQK 6 | Ew1HZW9UcnVzdCBJbmMuMTEwLwYDVQQDEyhHZW9UcnVzdCBQcmltYXJ5IENlcnRp 7 | ZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC 8 | AQEAvrgVe//UfH1nrYNke8hCUy3f9oQIIGHWAVlqnEQRr+92/ZV+zmEwu3qDXwK9 9 | AWbK7hWNb6EwnL2hhZ6UOvNWiAAxz9juapYC2e0DjPt1befquFUWBRaa9OBesYjA 10 | ZIVcFU2Ix7e64HXprQU9nceJSOC7KMgD4TCTZF5SwFlwIjVXiIrxlQqD17wxcwE0 11 | 7e9GceBrAqg1cmuXm2bgyxx5X9gaBGgeRwLmnWDiNpcB3841kt++Z8dtd1k7j53W 12 | kBWUvEI0EME5+bEnPn7WinXFsq+W06Lem+SYvn3h6YGttm/81w7a4DSwDRp35+MI 13 | mO9Y+pyEtzavwt+s0vQQBnBxNQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4G 14 | A1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQULNVQQZcVi/CPNmFbSvtr2ZnJM5IwDQYJ 15 | KoZIhvcNAQEFBQADggEBAFpwfyzdtzRP9YZRqSa+S7iq8XEN3GHHoOo0Hnp3DwQ1 16 | 6CePbJC/kRYkRj5KTs4rFtULUh38H2eiAkUxT87z+gOneZ1TatnaYzr4gNfTmeGl 17 | 4b7UVXGYNTq+k+qurUKykG/g/CFNNWMziUnWm07Kx+dOCQD32sfvmWKZd7aVIl6K 18 | oKv0uHiYyjgZmclynnjNS6yvGaBzEi38wkG6gZHaFloxt/m0cYASSJlyc1pZU8Fj 19 | UjPtp8nSOQJw+uCxQmYpqptR7TBUIhRf2asdweSU8Pj1K/fqynhG1riR/aYNKxoU 20 | AT6A8EKglQdebc3MS6RFjasS6LPeWuWgfOgPIh1a6Vk= 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/7d0b38bd.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDhDCCAwqgAwIBAgIQL4D+I4wOIg9IZxIokYesszAKBggqhkjOPQQDAzCByjEL 3 | MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW 4 | ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2ln 5 | biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp 6 | U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y 7 | aXR5IC0gRzQwHhcNMDcxMTA1MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCByjELMAkG 8 | A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZWZXJp 9 | U2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2lnbiwg 10 | SW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJpU2ln 11 | biBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5 12 | IC0gRzQwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASnVnp8Utpkmw4tXNherJI9/gHm 13 | GUo9FANL+mAnINmDiWn6VMaaGF5VKmTeBvaNSjutEDxlPZCIBIngMGGzrl0Bp3ve 14 | fLK+ymVhAIau2o970ImtTR1ZmkGxvEeA3J5iw/mjgbIwga8wDwYDVR0TAQH/BAUw 15 | AwEB/zAOBgNVHQ8BAf8EBAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJ 16 | aW1hZ2UvZ2lmMCEwHzAHBgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYj 17 | aHR0cDovL2xvZ28udmVyaXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFLMW 18 | kf3upm7ktS5Jj4d4gYDs5bG1MAoGCCqGSM49BAMDA2gAMGUCMGYhDBgmYFo4e1ZC 19 | 4Kf8NoRRkSAsdk1DPcQdhCPQrNZ8NQbOzWm9kA3bbEhCHQ6qQgIxAJw9SDkjOVga 20 | FRJZap7v1VmyHVIsmXHNxynfGyphe3HR3vPA5Q06Sqotp9iGKt0uEA== 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/607986c7.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh 3 | MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 4 | d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH 5 | MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT 6 | MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j 7 | b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG 8 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI 9 | 2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx 10 | 1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ 11 | q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz 12 | tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ 13 | vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP 14 | BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV 15 | 5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY 16 | 1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4 17 | NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG 18 | Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 19 | 8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe 20 | pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl 21 | MrY= 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/9d04f354.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl 3 | MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 4 | d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv 5 | b3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQG 6 | EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl 7 | cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwggEi 8 | MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSA 9 | n61UQbVH35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4Htecc 10 | biJVMWWXvdMX0h5i89vqbFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9Hp 11 | EgjAALAcKxHad3A2m67OeYfcgnDmCXRwVWmvo2ifv922ebPynXApVfSr/5Vh88lA 12 | bx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OPYLfykqGxvYmJHzDNw6Yu 13 | YjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+RnlTGNAgMB 14 | AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQW 15 | BBTOw0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPI 16 | QW5pJ6d1Ee88hjZv0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I 17 | 0jJmwYrA8y8678Dj1JGG0VDjA9tzd29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4Gni 18 | lmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAWhsI6yLETcDbYz+70CjTVW0z9 19 | B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv 20 | ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo 21 | IhNzbM8m9Yop5w== 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/76cb8f92.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDoTCCAomgAwIBAgILBAAAAAABD4WqLUgwDQYJKoZIhvcNAQEFBQAwOzEYMBYG 3 | A1UEChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2Jh 4 | bCBSb290MB4XDTA2MTIxNTA4MDAwMFoXDTIxMTIxNTA4MDAwMFowOzEYMBYGA1UE 5 | ChMPQ3liZXJ0cnVzdCwgSW5jMR8wHQYDVQQDExZDeWJlcnRydXN0IEdsb2JhbCBS 6 | b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+Mi8vRRQZhP/8NN5 7 | 7CPytxrHjoXxEnOmGaoQ25yiZXRadz5RfVb23CO21O1fWLE3TdVJDm71aofW0ozS 8 | J8bi/zafmGWgE07GKmSb1ZASzxQG9Dvj1Ci+6A74q05IlG2OlTEQXO2iLb3VOm2y 9 | HLtgwEZLAfVJrn5GitB0jaEMAs7u/OePuGtm839EAL9mJRQr3RAwHQeWP032a7iP 10 | t3sMpTjr3kfb1V05/Iin89cqdPHoWqI7n1C6poxFNcJQZZXcY4Lv3b93TZxiyWNz 11 | FtApD0mpSPCzqrdsxacwOUBdrsTiXSZT8M4cIwhhqJQZugRiQOwfOHB3EgZxpzAY 12 | XSUnpQIDAQABo4GlMIGiMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/ 13 | MB0GA1UdDgQWBBS2CHsNesysIEyGVjJez6tuhS1wVzA/BgNVHR8EODA2MDSgMqAw 14 | hi5odHRwOi8vd3d3Mi5wdWJsaWMtdHJ1c3QuY29tL2NybC9jdC9jdHJvb3QuY3Js 15 | MB8GA1UdIwQYMBaAFLYIew16zKwgTIZWMl7Pq26FLXBXMA0GCSqGSIb3DQEBBQUA 16 | A4IBAQBW7wojoFROlZfJ+InaRcHUowAl9B8Tq7ejhVhpwjCt2BWKLePJzYFa+HMj 17 | Wqd8BfP9IjsO0QbE2zZMcwSO5bAi5MXzLqXZI+O4Tkogp24CJJ8iYGd7ix1yCcUx 18 | XOl5n4BHPa2hCwcUPUf/A2kaDAtE52Mlp3+yybh2hO0j9n0Hq0V+09+zv+mKts2o 19 | omcrUtW3ZfA5TGOgkXmTUg9U3YO7n9GPp1Nzw8v/MOx8BLjYRB+TX3EJIrduPuoc 20 | A06dGiBh+4E37F78CkWr1+cXVdCg6mCbpvbjjFspwgZgFJ0tl0ypkxWdYcQBX0jW 21 | WL1WMRJOEcgh4LMRkWXbtKaIOM5V 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/3513523f.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh 3 | MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 4 | d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD 5 | QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT 6 | MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j 7 | b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG 8 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB 9 | CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97 10 | nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt 11 | 43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P 12 | T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4 13 | gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO 14 | BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR 15 | TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw 16 | DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr 17 | hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg 18 | 06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF 19 | PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls 20 | YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk 21 | CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4= 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/b1159c4c.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl 3 | MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 4 | d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv 5 | b3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzExMTEwMDAwMDAwWjBlMQswCQYDVQQG 6 | EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl 7 | cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwggEi 8 | MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7c 9 | JpSIqvTO9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYP 10 | mDI2dsze3Tyoou9q+yHyUmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+ 11 | wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4 12 | VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpyoeb6pNnVFzF1roV9Iq4/ 13 | AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whfGHdPAgMB 14 | AAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW 15 | BBRF66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYun 16 | pyGd823IDzANBgkqhkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRC 17 | dWKuh+vy1dneVrOfzM4UKLkNl2BcEkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTf 18 | fwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38FnSbNd67IJKusm7Xi+fT8r87cm 19 | NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx 20 | H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe 21 | +o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g== 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/4a6481c9.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDujCCAqKgAwIBAgILBAAAAAABD4Ym5g0wDQYJKoZIhvcNAQEFBQAwTDEgMB4G 3 | A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjIxEzARBgNVBAoTCkdsb2JhbFNp 4 | Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDYxMjE1MDgwMDAwWhcNMjExMjE1 5 | MDgwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEG 6 | A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI 7 | hvcNAQEBBQADggEPADCCAQoCggEBAKbPJA6+Lm8omUVCxKs+IVSbC9N/hHD6ErPL 8 | v4dfxn+G07IwXNb9rfF73OX4YJYJkhD10FPe+3t+c4isUoh7SqbKSaZeqKeMWhG8 9 | eoLrvozps6yWJQeXSpkqBy+0Hne/ig+1AnwblrjFuTosvNYSuetZfeLQBoZfXklq 10 | tTleiDTsvHgMCJiEbKjNS7SgfQx5TfC4LcshytVsW33hoCmEofnTlEnLJGKRILzd 11 | C9XZzPnqJworc5HGnRusyMvo4KD0L5CLTfuwNhv2GXqF4G3yYROIXJ/gkwpRl4pa 12 | zq+r1feqCapgvdzZX99yqWATXgAByUr6P6TqBwMhAo6CygPCm48CAwEAAaOBnDCB 13 | mTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUm+IH 14 | V2ccHsBqBt5ZtJot39wZhi4wNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5n 15 | bG9iYWxzaWduLm5ldC9yb290LXIyLmNybDAfBgNVHSMEGDAWgBSb4gdXZxwewGoG 16 | 3lm0mi3f3BmGLjANBgkqhkiG9w0BAQUFAAOCAQEAmYFThxxol4aR7OBKuEQLq4Gs 17 | J0/WwbgcQ3izDJr86iw8bmEbTUsp9Z8FHSbBuOmDAGJFtqkIk7mpM0sYmsL4h4hO 18 | 291xNBrBVNpGP+DTKqttVCL1OmLNIG+6KYnX3ZHu01yiPqFbQfXf5WRDLenVOavS 19 | ot+3i9DAgBkcRcAtjOj4LaR0VknFBbVPFd5uRHg5h6h+u/N5GJG79G+dwfCMNYxd 20 | AfvDbbnvRG15RjF+Cv6pgsH/76tuIMRQyV+dTZsXjAzlAcmgQWpzU/qlULRuJQ/7 21 | TBj0/VLZjmmx6BEP3ojY+x1J96relc8geMJgEtslQIxq/H5COEBkEveegeGTLg== 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/244b5494.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs 3 | MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 4 | d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j 5 | ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL 6 | MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 7 | LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug 8 | RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm 9 | +9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW 10 | PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM 11 | xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB 12 | Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3 13 | hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg 14 | EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF 15 | MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA 16 | FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec 17 | nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z 18 | eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF 19 | hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2 20 | Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe 21 | vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep 22 | +OkuE6N36B9K 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/cbf06781.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx 3 | EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT 4 | EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp 5 | ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz 6 | NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH 7 | EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE 8 | AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw 9 | DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD 10 | E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH 11 | /PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy 12 | DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh 13 | GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR 14 | tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA 15 | AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE 16 | FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX 17 | WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu 18 | 9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr 19 | gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo 20 | 2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO 21 | LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI 22 | 4uJEvlz36hz1 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /aws/utils/backtrace/bsd_backtrace.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "backtrace.h" 17 | #include 18 | #include 19 | #include 20 | 21 | #ifdef BSD_BACKTRACE 22 | namespace aws { 23 | namespace utils { 24 | namespace backtrace { 25 | 26 | void initialize(const char* exe) { 27 | 28 | } 29 | 30 | void stack_trace_for_signal(int skip, bool /*signaled*/) { 31 | void *buffer[100]; 32 | int symbol_count = ::backtrace(buffer, sizeof(buffer)); 33 | WRITE_MESSAGE("SYMBOLS:(") 34 | WRITE_NUM_CHECKED(symbol_count, "NEG") 35 | WRITE_MESSAGE("/") 36 | WRITE_NUMBER(sizeof(buffer)) 37 | WRITE_MESSAGE(")\n") 38 | if (symbol_count > 0) { 39 | backtrace_symbols_fd(buffer, symbol_count, STDERR_FILENO); 40 | } 41 | WRITE_MESSAGE("\n") 42 | } 43 | 44 | } 45 | } 46 | } 47 | #endif // BSD_STACKTRACE 48 | 49 | 50 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/4bfab552.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx 3 | EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT 4 | HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs 5 | ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw 6 | MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 7 | b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj 8 | aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp 9 | Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC 10 | ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg 11 | nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1 12 | HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N 13 | Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN 14 | dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0 15 | HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO 16 | BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G 17 | CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU 18 | sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3 19 | 4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg 20 | 8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K 21 | pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1 22 | mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/UserRecordFailedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | /** 19 | * The exception thrown when the max attempts to put a UserRecord into Kinesis is exhausted and the result is not 20 | * successful. 21 | */ 22 | public class UserRecordFailedException extends KinesisProducerException { 23 | private static final long serialVersionUID = 3168271192277927600L; 24 | 25 | private UserRecordResult result; 26 | 27 | public UserRecordFailedException(UserRecordResult result, UserRecord userRecord) { 28 | super(userRecord); 29 | this.result = result; 30 | } 31 | 32 | public UserRecordFailedException(UserRecordResult result) { 33 | this(result, null); 34 | } 35 | 36 | public UserRecordResult getResult() { 37 | return result; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/09789157.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx 3 | EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT 4 | HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs 5 | ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 6 | MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD 7 | VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy 8 | ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy 9 | dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI 10 | hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p 11 | OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 12 | 8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K 13 | Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe 14 | hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk 15 | 6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw 16 | DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q 17 | AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI 18 | bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB 19 | ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z 20 | qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd 21 | iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn 22 | 0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN 23 | sSi6 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/e2799e36.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/jCCAuagAwIBAgIQFaxulBmyeUtB9iepwxgPHzANBgkqhkiG9w0BAQsFADCB 3 | mDELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsT 4 | MChjKSAyMDA4IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s 5 | eTE2MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhv 6 | cml0eSAtIEczMB4XDTA4MDQwMjAwMDAwMFoXDTM3MTIwMTIzNTk1OVowgZgxCzAJ 7 | BgNVBAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykg 8 | MjAwOCBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0 9 | BgNVBAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg 10 | LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANziXmJYHTNXOTIz 11 | +uvLh4yn1ErdBojqZI4xmKU4kB6Yzy5jK/BGvESyiaHAKAxJcCGVn2TAppMSAmUm 12 | hsalifD614SgcK9PGpc/BkTVyetyEH3kMSj7HGHmKAdEc5IiaacDiGydY8hS2pgn 13 | 5whMcD60yRLBxWeDXTPzAxHsatBT4tG6NmCUgLthY2xbF37fQJQeqw3CIShwiP/W 14 | JmxsYAQlTlV+fe+/lEjetx3dcI0FX4ilm/LC7urRQEFtYjgdVgbFA0dRIBn8exAL 15 | DmKudlW/X3e+PkkBUz2YJQN2JFodtNuJ6nnltrM7P7pMKEF/BqxqjsHQ9gUdfeZC 16 | huOl1UcCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw 17 | HQYDVR0OBBYEFMR5yo6hTgMdHNxr2zFblD4/MH8tMA0GCSqGSIb3DQEBCwUAA4IB 18 | AQAtxRPPVoB7eni9n64smefv2t+UXglpp+duaIy9cr5HqQ6XErhK8WTTOd8lNNTB 19 | zU6B8A8ExCSzNJbGpqow32hhc9f5joWJ7w5elShKKiePEI4ufIbEAp7aDHdlDkQN 20 | kv39sxY2+hENHYwOB4lqKVb3cvTdFZx3NWZXqxNT2I7BQMXXExZacse3aQHEerGD 21 | AWh9jUGhlBjBJVz88P6DAod8DQ3PLghcSkANPuyBYeYk28rgDi0Hsj5W3I31QYUH 22 | SJsMC8tJP33st/3LjWeJGqvtux6jAAgIFyqCXDFdRootD4abdNlF+9RAsXqqaC2G 23 | spki4cErx5z481+oghLrGREt 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/f081611a.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh 3 | MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE 4 | YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 5 | MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo 6 | ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg 7 | MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN 8 | ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA 9 | PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w 10 | wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi 11 | EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY 12 | avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ 13 | YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE 14 | sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h 15 | /t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 16 | IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj 17 | YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD 18 | ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy 19 | OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P 20 | TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ 21 | HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER 22 | dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf 23 | ReYNnyicsbkqWletNw+vHX/bvZ8= 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/f387163d.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEDzCCAvegAwIBAgIBADANBgkqhkiG9w0BAQUFADBoMQswCQYDVQQGEwJVUzEl 3 | MCMGA1UEChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMp 4 | U3RhcmZpZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQw 5 | NjI5MTczOTE2WhcNMzQwNjI5MTczOTE2WjBoMQswCQYDVQQGEwJVUzElMCMGA1UE 6 | ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMpU3RhcmZp 7 | ZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggEgMA0GCSqGSIb3 8 | DQEBAQUAA4IBDQAwggEIAoIBAQC3Msj+6XGmBIWtDBFk385N78gDGIc/oav7PKaf 9 | 8MOh2tTYbitTkPskpD6E8J7oX+zlJ0T1KKY/e97gKvDIr1MvnsoFAZMej2YcOadN 10 | +lq2cwQlZut3f+dZxkqZJRRU6ybH838Z1TBwj6+wRir/resp7defqgSHo9T5iaU0 11 | X9tDkYI22WY8sbi5gv2cOj4QyDvvBmVmepsZGD3/cVE8MC5fvj13c7JdBmzDI1aa 12 | K4UmkhynArPkPw2vCHmCuDY96pzTNbO8acr1zJ3o/WSNF4Azbl5KXZnJHoe0nRrA 13 | 1W4TNSNe35tfPe/W93bC6j67eA0cQmdrBNj41tpvi/JEoAGrAgEDo4HFMIHCMB0G 14 | A1UdDgQWBBS/X7fRzt0fhvRbVazc1xDCDqmI5zCBkgYDVR0jBIGKMIGHgBS/X7fR 15 | zt0fhvRbVazc1xDCDqmI56FspGowaDELMAkGA1UEBhMCVVMxJTAjBgNVBAoTHFN0 16 | YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAsTKVN0YXJmaWVsZCBD 17 | bGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8w 18 | DQYJKoZIhvcNAQEFBQADggEBAAWdP4id0ckaVaGsafPzWdqbAYcaT1epoXkJKtv3 19 | L7IezMdeatiDh6GX70k1PncGQVhiv45YuApnP+yz3SFmH8lU+nLMPUxA2IGvd56D 20 | eruix/U0F47ZEUD0/CwqTRV/p2JdLiXTAAsgGh1o+Re49L2L7ShZ3U0WixeDyLJl 21 | xy16paq8U4Zt3VekyvggQQto8PT7dL5WXXp59fkdheMtlb71cZBDzI0fmgAKhynp 22 | VSJYACPq4xJDKVtHCN2MQWplBqjlIapBtJUhlbl90TSrE9atvNziPTnNvT51cKEY 23 | WQPJIrSPnNVeKtelttQKbfi3QBFGmh95DmK/D5fs4C8fF5Q= 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/c0ff1f52.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw 3 | CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl 4 | cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu 5 | LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT 6 | aWduIENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp 7 | dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD 8 | VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT 9 | aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ 10 | bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu 11 | IENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg 12 | LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMu6nFL8eB8aHm8b 13 | N3O9+MlrlBIwT/A2R/XQkQr1F8ilYcEWQE37imGQ5XYgwREGfassbqb1EUGO+i2t 14 | KmFZpGcmTNDovFJbcCAEWNF6yaRpvIMXZK0Fi7zQWM6NjPXr8EJJC52XJ2cybuGu 15 | kxUccLwgTS8Y3pKI6GyFVxEa6X7jJhFUokWWVYPKMIno3Nij7SqAP395ZVc+FSBm 16 | CC+Vk7+qRy+oRpfwEuL+wgorUeZ25rdGt+INpsyow0xZVYnm6FNcHOqd8GIWC6fJ 17 | Xwzw3sJ2zq/3avL6QaaiMxTJ5Xpj055iN9WFZZ4O5lMkdBteHRJTW8cs54NJOxWu 18 | imi5V5cCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAERSWwauSCPc/L8my/uRan2Te 19 | 2yFPhpk0djZX3dAVL8WtfxUfN2JzPtTnX84XA9s1+ivbrmAJXx5fj267Cz3qWhMe 20 | DGBvtcC1IyIuBwvLqXTLR7sdwdela8wv0kL9Sd2nic9TutoAWii/gt/4uhMdUIaC 21 | /Y4wjylGsB49Ndo4YhYYSq3mtlFs3q9i6wHQHiT+eo8SGhJouPtmmRQURVyu565p 22 | F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt 23 | TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ== 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/2e4eed3c.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB 3 | qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf 4 | Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw 5 | MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV 6 | BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMDYxMTE3MDAwMDAwWhcNMzYw 7 | NzE2MjM1OTU5WjCBqTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5j 8 | LjEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYG 9 | A1UECxMvKGMpIDIwMDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNl 10 | IG9ubHkxHzAdBgNVBAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwggEiMA0GCSqG 11 | SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsoPD7gFnUnMekz52hWXMJEEUMDSxuaPFs 12 | W0hoSVk3/AszGcJ3f8wQLZU0HObrTQmnHNK4yZc2AreJ1CRfBsDMRJSUjQJib+ta 13 | 3RGNKJpchJAQeg29dGYvajig4tVUROsdB58Hum/u6f1OCyn1PoSgAfGcq/gcfomk 14 | 6KHYcWUNo1F77rzSImANuVud37r8UVsLr5iy6S7pBOhih94ryNdOwUxkHt3Ph1i6 15 | Sk/KaAcdHJ1KxtUvkcx8cXIcxcBn6zL9yZJclNqFwJu/U30rCfSMnZEfl2pSy94J 16 | NqR32HuHUETVPm4pafs5SSYeCaWAe0At6+gnhcn+Yf1+5nyXHdWdAgMBAAGjQjBA 17 | MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBR7W0XP 18 | r87Lev0xkhpqtvNG61dIUDANBgkqhkiG9w0BAQUFAAOCAQEAeRHAS7ORtvzw6WfU 19 | DW5FvlXok9LOAz/t2iWwHVfLHjp2oEzsUHboZHIMpKnxuIvW1oeEuzLlQRHAd9mz 20 | YJ3rG9XRbkREqaYB7FViHXe4XI5ISXycO1cRrK1zN44veFyQaEfZYGDm/Ac9IiAX 21 | xPcW6cTYcvnIc3zfFi8VqT79aie2oetaupgf1eNNZAqdE8hhuvU5HIe6uL17In/2 22 | /qxAeeWsEG89jxt5dovEN7MhGITlNgDrYyCZuen+MwS7QcjBAvlEYyCegc5C09Y/ 23 | LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7 24 | jVaMaA== 25 | -----END CERTIFICATE----- 26 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/ba89ed3b.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEKjCCAxKgAwIBAgIQYAGXt0an6rS0mtZLL/eQ+zANBgkqhkiG9w0BAQsFADCB 3 | rjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf 4 | Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw 5 | MDggdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAiBgNV 6 | BAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMzAeFw0wODA0MDIwMDAwMDBa 7 | Fw0zNzEyMDEyMzU5NTlaMIGuMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhhd3Rl 8 | LCBJbmMuMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9uIFNlcnZpY2VzIERpdmlzaW9u 9 | MTgwNgYDVQQLEy8oYykgMjAwOCB0aGF3dGUsIEluYy4gLSBGb3IgYXV0aG9yaXpl 10 | ZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9vdCBDQSAtIEcz 11 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsr8nLPvb2FvdeHsbnndm 12 | gcs+vHyu86YnmjSjaDFxODNi5PNxZnmxqWWjpYvVj2AtP0LMqmsywCPLLEHd5N/8 13 | YZzic7IilRFDGF/Eth9XbAoFWCLINkw6fKXRz4aviKdEAhN0cXMKQlkC+BsUa0Lf 14 | b1+6a4KinVvnSr0eAXLbS3ToO39/fR8EtCab4LRarEc9VbjXsCZSKAExQGbY2SS9 15 | 9irY7CFJXJv2eul/VTV+lmuNk5Mny5K76qxAwJ/C+IDPXfRa3M50hqY+bAtTyr2S 16 | zhkGcuYMXDhpxwTWvGzOW/b3aJzcJRVIiKHpqfiYnODz1TEoYRFsZ5aNOZnLwkUk 17 | OQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNV 18 | HQ4EFgQUrWyqlGCc7eT/+j4KdCtjA/e2Wb8wDQYJKoZIhvcNAQELBQADggEBABpA 19 | 2JVlrAmSicY59BDlqQ5mU1143vokkbvnRFHfxhY0Cu9qRFHqKweKA3rD6z8KLFIW 20 | oCtDuSWQP3CpMyVtRRooOyfPqsMpQhvfO0zAMzRbQYi/aytlryjvsvXDqmbOe1bu 21 | t8jLZ8HJnBoYuMTDSQPxYA5QzUbF83d597YV4Djbxy8ooAw/dyZ02SUS2jHaGh7c 22 | KUGRIjxpp7sC8rZcJwOJ9Abqm+RyguOhCcHpABnTPtRwa7pxpqpYrvS76Wy274fM 23 | m7v/OeZWYdMKp8RcTGB7BXcmer/YB1IsYvdwY9k5vG8cwnncdimvzsUsZAReiDZu 24 | MdRAGmI0Nj81Aa6sY6A= 25 | -----END CERTIFICATE----- 26 | -------------------------------------------------------------------------------- /aws/metrics/metrics_index.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_METRICS_METRICS_INDEX_H_ 17 | #define AWS_METRICS_METRICS_INDEX_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | namespace aws { 31 | namespace metrics { 32 | 33 | class MetricsIndex : boost::noncopyable { 34 | public: 35 | std::shared_ptr get_metric(const MetricsFinder& metrics_finder); 36 | 37 | std::vector> get_all(); 38 | 39 | private: 40 | using Mutex = aws::shared_mutex; 41 | using ReadLock = aws::shared_lock; 42 | using WriteLock = aws::unique_lock; 43 | 44 | std::unordered_map> metrics_; 45 | Mutex mutex_; 46 | }; 47 | 48 | } //namespace metrics 49 | } //namespace aws 50 | 51 | #endif //AWS_METRICS_METRICS_INDEX_H_ 52 | -------------------------------------------------------------------------------- /aws/utils/concurrent_linked_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_UTILS_CONCURRENT_LINKED_QUEUE_H_ 17 | #define AWS_UTILS_CONCURRENT_LINKED_QUEUE_H_ 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | namespace aws { 25 | namespace utils { 26 | 27 | template 28 | class ConcurrentLinkedQueue { 29 | public: 30 | template 31 | void put(U&& data) { 32 | Lock lk(mutex_); 33 | q_.push_back(std::forward(data)); 34 | } 35 | 36 | bool try_take(T& t) { 37 | Lock lk(mutex_); 38 | if (q_.empty()) { 39 | return false; 40 | } else { 41 | t = std::move(q_.front()); 42 | q_.pop_front(); 43 | return true; 44 | } 45 | } 46 | 47 | private: 48 | using Mutex = aws::utils::TicketSpinLock; 49 | using Lock = aws::lock_guard; 50 | 51 | Mutex mutex_; 52 | std::list q_; 53 | }; 54 | 55 | } //namespace utils 56 | } //namespace aws 57 | 58 | #endif //AWS_UTILS_CONCURRENT_LINKED_QUEUE_H_ 59 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/IKinesisProducer.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.util.List; 5 | import java.util.concurrent.ExecutionException; 6 | import com.google.common.util.concurrent.ListenableFuture; 7 | import com.amazonaws.services.schemaregistry.common.Schema; 8 | 9 | 10 | public interface IKinesisProducer { 11 | ListenableFuture addUserRecord(String stream, String partitionKey, ByteBuffer data); 12 | 13 | ListenableFuture addUserRecord(UserRecord userRecord); 14 | 15 | ListenableFuture addUserRecord(String stream, String partitionKey, String explicitHashKey, ByteBuffer data); 16 | 17 | ListenableFuture addUserRecord(String stream, String partitionKey, String explicitHashKey, ByteBuffer data, Schema schema); 18 | 19 | int getOutstandingRecordsCount(); 20 | 21 | default long getOldestRecordTimeInMillis() { 22 | throw new UnsupportedOperationException("This method is not supported in this IKinesisProducer type"); 23 | } 24 | 25 | List getMetrics(String metricName, int windowSeconds) throws InterruptedException, ExecutionException; 26 | 27 | List getMetrics(String metricName) throws InterruptedException, ExecutionException; 28 | 29 | List getMetrics() throws InterruptedException, ExecutionException; 30 | 31 | List getMetrics(int windowSeconds) throws InterruptedException, ExecutionException; 32 | 33 | void destroy(); 34 | 35 | void flush(String stream); 36 | 37 | void flush(); 38 | 39 | void flushSync(); 40 | } 41 | -------------------------------------------------------------------------------- /aws/utils/logging.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_UTILS_LOGGING_H_ 17 | #define AWS_UTILS_LOGGING_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | 25 | #define LOG(sev) BOOST_LOG_SEV(aws::utils::_logger, \ 26 | boost::log::trivial::severity_level::sev) \ 27 | << "[" << boost::filesystem::path(__FILE__).filename().string() \ 28 | << ":" << __LINE__ << "] " 29 | 30 | namespace aws { 31 | namespace utils { 32 | 33 | static boost::log::sources::severity_logger_mt< 34 | boost::log::trivial::severity_level> _logger; 35 | 36 | boost::log::trivial::severity_level set_log_level(const std::string& min_level); 37 | 38 | void setup_logging(const std::string& min_level = "info"); 39 | void setup_logging(boost::log::trivial::severity_level level); 40 | 41 | void setup_aws_logging(Aws::Utils::Logging::LogLevel log_level); 42 | void teardown_aws_logging(); 43 | 44 | } //namespace utils 45 | } //namespace aws 46 | 47 | #endif //AWS_UTILS_LOGGING_H_ 48 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/test/java/software/amazon/kinesis/producer/GlueSchemaRegistrySerializerInstanceTest.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; 4 | import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializer; 5 | import org.junit.Test; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | import static org.junit.Assert.assertNotNull; 9 | 10 | public class GlueSchemaRegistrySerializerInstanceTest { 11 | 12 | private static final String REGION = "us-west-1"; 13 | private GlueSchemaRegistrySerializerInstance glueSchemaRegistrySerializerInstance = new GlueSchemaRegistrySerializerInstance(); 14 | 15 | @Test 16 | public void testGet_Returns_SingletonInstance() { 17 | KinesisProducerConfiguration configuration = new KinesisProducerConfiguration(); 18 | configuration.setRegion(REGION); 19 | 20 | GlueSchemaRegistrySerializer serializer1 = 21 | glueSchemaRegistrySerializerInstance.get(configuration); 22 | 23 | assertNotNull(serializer1); 24 | GlueSchemaRegistrySerializer serializer2 = 25 | glueSchemaRegistrySerializerInstance.get(configuration); 26 | 27 | assertEquals(serializer1.hashCode(), serializer2.hashCode()); 28 | } 29 | 30 | @Test 31 | public void testGet_Returns_WhenGlueConfigurationIsExplicitlyConfigured() { 32 | KinesisProducerConfiguration configuration = new KinesisProducerConfiguration(); 33 | configuration.setGlueSchemaRegistryConfiguration(new GlueSchemaRegistryConfiguration(REGION)); 34 | 35 | GlueSchemaRegistrySerializer serializer = 36 | glueSchemaRegistrySerializerInstance.get(configuration); 37 | 38 | assertNotNull(serializer); 39 | } 40 | } -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/c01cdfa2.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEuTCCA6GgAwIBAgIQQBrEZCGzEyEDDrvkEhrFHTANBgkqhkiG9w0BAQsFADCB 3 | vTELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL 4 | ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwOCBWZXJp 5 | U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MTgwNgYDVQQDEy9W 6 | ZXJpU2lnbiBVbml2ZXJzYWwgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAe 7 | Fw0wODA0MDIwMDAwMDBaFw0zNzEyMDEyMzU5NTlaMIG9MQswCQYDVQQGEwJVUzEX 8 | MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlTaWduIFRydXN0 9 | IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAyMDA4IFZlcmlTaWduLCBJbmMuIC0gRm9y 10 | IGF1dGhvcml6ZWQgdXNlIG9ubHkxODA2BgNVBAMTL1ZlcmlTaWduIFVuaXZlcnNh 11 | bCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEF 12 | AAOCAQ8AMIIBCgKCAQEAx2E3XrEBNNti1xWb/1hajCMj1mCOkdeQmIN65lgZOIzF 13 | 9uVkhbSicfvtvbnazU0AtMgtc6XHaXGVHzk8skQHnOgO+k1KxCHfKWGPMiJhgsWH 14 | H26MfF8WIFFE0XBPV+rjHOPMee5Y2A7Cs0WTwCznmhcrewA3ekEzeOEz4vMQGn+H 15 | LL729fdC4uW/h2KJXwBL38Xd5HVEMkE6HnFuacsLdUYI0crSK5XQz/u5QGtkjFdN 16 | /BMReYTtXlT2NJ8IAfMQJQYXStrxHXpma5hgZqTZ79IugvHw7wnqRMkVauIDbjPT 17 | rJ9VAMf2CGqUuV/c4DPxhGD5WycRtPwW8rtWaoAljQIDAQABo4GyMIGvMA8GA1Ud 18 | EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMG0GCCsGAQUFBwEMBGEwX6FdoFsw 19 | WTBXMFUWCWltYWdlL2dpZjAhMB8wBwYFKw4DAhoEFI/l0xqGrI2Oa8PPgGrUSBgs 20 | exkuMCUWI2h0dHA6Ly9sb2dvLnZlcmlzaWduLmNvbS92c2xvZ28uZ2lmMB0GA1Ud 21 | DgQWBBS2d/ppSEefUxLVwuoHMnYH0ZcHGTANBgkqhkiG9w0BAQsFAAOCAQEASvj4 22 | sAPmLGd75JR3Y8xuTPl9Dg3cyLk1uXBPY/ok+myDjEedO2Pzmvl2MpWRsXe8rJq+ 23 | seQxIcaBlVZaDrHC1LGmWazxY8u4TB1ZkErvkBYoH1quEPuBUDgMbMzxPcP1Y+Oz 24 | 4yHJJDnp/RVmRvQbEdBNc6N9Rvk97ahfYtTxP/jgdFcrGJ2BtMQo2pSXpXDrrB2+ 25 | BxHw1dvd5Yzw1TKwg+ZX4o+/vqGqvz0dtdQ46tewXDpPaj+PwGZsY6rp2aQW9IHR 26 | lRQOfc2VNNnSj3BzgXucfr2YYdhFh5iQxeuGMMY1v/D/w1WIg0vvBZIGcfK4mJO3 27 | 7M2CYfE45k+XmCpajQ== 28 | -----END CERTIFICATE----- 29 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/6d41d539.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwF 3 | ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 4 | b24gUm9vdCBDQSAyMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTEL 5 | MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv 6 | b3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK2Wny2cSkxK 7 | gXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4kHbZ 8 | W0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg 9 | 1dKmSYXpN+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K 10 | 8nu+NQWpEjTj82R0Yiw9AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r 11 | 2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvdfLC6HM783k81ds8P+HgfajZRRidhW+me 12 | z/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAExkv8LV/SasrlX6avvDXbR 13 | 8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSSbtqDT6Zj 14 | mUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz 15 | 7Mt0Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6 16 | +XUyo05f7O0oYtlNc/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI 17 | 0u1ufm8/0i2BWSlmy5A5lREedCf+3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB 18 | Af8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSwDPBMMPQFWAJI/TPlUq9LhONm 19 | UjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oAA7CXDpO8Wqj2 20 | LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY 21 | +gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kS 22 | k5Nrp+gvU5LEYFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl 23 | 7uxMMne0nxrpS10gxdr9HIcWxkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygm 24 | btmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQgj9sAq+uEjonljYE1x2igGOpm/Hl 25 | urR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbWaQbLU8uz/mtBzUF+ 26 | fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoVYh63 27 | n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE 28 | 76KlXIx3KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H 29 | 9jVlpNMKVv/1F2Rs76giJUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT 30 | 4PsJYGw= 31 | -----END CERTIFICATE----- 32 | -------------------------------------------------------------------------------- /aws/auth/mutable_static_creds_provider.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_AUTH_MUTABLE_STATIC_CREDS_PROVIDER_H_ 17 | #define AWS_AUTH_MUTABLE_STATIC_CREDS_PROVIDER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace aws { 26 | namespace auth { 27 | 28 | struct VersionedCredentials { 29 | std::uint64_t version_; 30 | Aws::Auth::AWSCredentials creds_; 31 | 32 | VersionedCredentials() : version_(0), creds_("", "", "") {} 33 | VersionedCredentials(std::uint64_t version, const std::string& akid, const std::string& sk, const std::string& token); 34 | }; 35 | 36 | // Like basic static creds, but with an atomic set operation 37 | class MutableStaticCredentialsProvider 38 | : public Aws::Auth::AWSCredentialsProvider { 39 | public: 40 | MutableStaticCredentialsProvider(const std::string& akid, const std::string& sk, std::string token = ""); 41 | 42 | void set_credentials(const std::string& akid, const std::string& sk, std::string token = ""); 43 | 44 | Aws::Auth::AWSCredentials GetAWSCredentials() override; 45 | 46 | private: 47 | std::mutex update_mutex_; 48 | std::shared_ptr creds_; 49 | std::atomic version_; 50 | 51 | }; 52 | 53 | } //namespace auth 54 | } //namespace aws 55 | 56 | #endif //AWS_AUTH_MUTABLE_STATIC_CREDS_PROVIDER_H_ 57 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/ad088e1d.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFaDCCA1CgAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJVUzEW 3 | MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEeMBwGA1UEAxMVR2VvVHJ1c3QgVW5pdmVy 4 | c2FsIENBMB4XDTA0MDMwNDA1MDAwMFoXDTI5MDMwNDA1MDAwMFowRTELMAkGA1UE 5 | BhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xHjAcBgNVBAMTFUdlb1RydXN0 6 | IFVuaXZlcnNhbCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKYV 7 | VaCjxuAfjJ0hUNfBvitbtaSeodlyWL0AG0y/YckUHUWCq8YdgNY96xCcOq9tJPi8 8 | cQGeBvV8Xx7BDlXKg5pZMK4ZyzBIle0iN430SppyZj6tlcDgFgDgEB8rMQ7XlFTT 9 | QjOgNB0eRXbdT8oYN+yFFXoZCPzVx5zw8qkuEKmS5j1YPakWaDwvdSEYfyh3peFh 10 | F7em6fgemdtzbvQKoiFs7tqqhZJmr/Z6a4LauiIINQ/PQvE1+mrufislzDoR5G2v 11 | c7J2Ha3QsnhnGqQ5HFELZ1aD/ThdDc7d8Lsrlh/eezJS/R27tQahsiFepdaVaH/w 12 | mZ7cRQg+59IJDTWU3YBOU5fXtQlEIGQWFwMCTFMNaN7VqnJNk22CDtucvc+081xd 13 | VHppCZbW2xHBjXWotM85yM48vCR85mLK4b19p71XZQvk/iXttmkQ3CgaRr0BHdCX 14 | teGYO8A3ZNY9lO4L4fUorgtWv3GLIylBjobFS1J72HGrH4oVpjuDWtdYAVHGTEHZ 15 | f9hBZ3KiKN9gg6meyHv8U3NyWfWTehd2Ds735VzZC1U0oqpbtWpU5xPKV+yXbfRe 16 | Bi9Fi1jUIxaS5BZuKGNZMN9QAZxjiRqf2xeUgnA3wySemkfWWspOqGmJch+RbNt+ 17 | nhutxx9z3SxPGWX9f5NAEC7S8O08ni4oPmkmM8V7AgMBAAGjYzBhMA8GA1UdEwEB 18 | /wQFMAMBAf8wHQYDVR0OBBYEFNq7LqqwDLiIJlF0XG0D08DYj3rWMB8GA1UdIwQY 19 | MBaAFNq7LqqwDLiIJlF0XG0D08DYj3rWMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG 20 | 9w0BAQUFAAOCAgEAMXjmx7XfuJRAyXHEqDXsRh3ChfMoWIawC/yOsjmPRFWrZIRc 21 | aanQmjg8+uUfNeVE44B5lGiku8SfPeE0zTBGi1QrlaXv9z+ZhP015s8xxtxqv6fX 22 | IwjhmF7DWgh2qaavdy+3YL1ERmrvl/9zlcGO6JP7/TG37FcREUWbMPEaiDnBTzyn 23 | ANXH/KttgCJwpQzgXQQpAvvLoJHRfNbDflDVnVi+QTjruXU8FdmbyUqDWcDaU/0z 24 | uzYYm4UPFd3uLax2k7nZAY1IEKj79TiG8dsKxr2EoyNB3tZ3b4XUhRxQ4K5RirqN 25 | Pnbiucon8l+f725ZDQbYKxek0nxru18UGkiPGkzns0ccjkxFKyDuSN/n3QmOGKja 26 | QI2SJhFTYXNd673nxE0pN2HrrDktZy4W1vUAg4WhzH92xH3kt0tm7wNFYGm2DFKW 27 | koRepqO1pD4r2czYG0eq8kTaT/kD6PAUyz/zg97QwVTjt+gKN02LIFkDMBmhLMi9 28 | ER/frslKxfMnZmaGrGiR/9nmUxwPi1xpZQomyB40w11Re9epnAahNt3ViZS82eQt 29 | DF4JbAiXfKM9fJP/P6EUp8+1Xevb2xzEdt+Iub1FBZUbrvxGakyvSOPOrg/Sfuvm 30 | bJxPgWp6ZKy7PtXny3YuxadIwVyQD8vIP/rmMuGNG2+k5o7Y+SlIis5z/iw= 31 | -----END CERTIFICATE----- 32 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/8867006a.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFbDCCA1SgAwIBAgIBATANBgkqhkiG9w0BAQUFADBHMQswCQYDVQQGEwJVUzEW 3 | MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1c3QgVW5pdmVy 4 | c2FsIENBIDIwHhcNMDQwMzA0MDUwMDAwWhcNMjkwMzA0MDUwMDAwWjBHMQswCQYD 5 | VQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1 6 | c3QgVW5pdmVyc2FsIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC 7 | AQCzVFLByT7y2dyxUxpZKeexw0Uo5dfR7cXFS6GqdHtXr0om/Nj1XqduGdt0DE81 8 | WzILAePb63p3NeqqWuDW6KFXlPCQo3RWlEQwAx5cTiuFJnSCegx2oG9NzkEtoBUG 9 | FF+3Qs17j1hhNNwqCPkuwwGmIkQcTAeC5lvO0Ep8BNMZcyfwqph/Lq9O64ceJHdq 10 | XbboW0W63MOhBW9Wjo8QJqVJwy7XQYci4E+GymC16qFjwAGXEHm9ADwSbSsVsaxL 11 | se4YuU6W3Nx2/zu+z18DwPw76L5GG//aQMJS9/7jOvdqdzXQ2o3rXhhqMcceujwb 12 | KNZrVMaqW9eiLBsZzKIC9ptZvTdrhrVtgrrY6slWvKk2WP0+GfPtDCapkzj4T8Fd 13 | IgbQl+rhrcZV4IErKIM6+vR7IVEAvlI4zs1meaj0gVbi0IMJR1FbUGrP20gaXT73 14 | y/Zl92zxlfgCOzJWgjl6W70viRu/obTo/3+NjN8D8WBOWBFM66M/ECuDmgFz2ZRt 15 | hAAnZqzwcEAJQpKtT5MNYQlRJNiS1QuUYbKHsu3/mjX/hVTK7URDrBs8FmtISgoc 16 | QIgfksILAAX/8sgCSqSqqcyZlpwvWOB94b67B9xfBHJcMTTD7F8t4D1kkCLm0ey4 17 | Lt1ZrtmhN79UNdxzMk+MBB4zsslG8dhcyFVQyWi9qLo2CQIDAQABo2MwYTAPBgNV 18 | HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAfBgNV 19 | HSMEGDAWgBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAOBgNVHQ8BAf8EBAMCAYYwDQYJ 20 | KoZIhvcNAQEFBQADggIBAGbBxiPz2eAubl/oz66wsCVNK/g7WJtAJDday6sWSf+z 21 | dXkzoS9tcBc0kf5nfo/sm+VegqlVHy/c1FEHEv6sFj4sNcZj/NwQ6w2jqtB8zNHQ 22 | L1EuxBRa3ugZ4T7GzKQp5y6EqgYweHZUcyiYWTjgAA1i00J9IZ+uPTqM1fp3DRgr 23 | Fg5fNuH8KrUwJM/gYwx7WBr+mbpCErGR9Hxo4sjoryzqyX6uuyo9DRXcNJW2GHSo 24 | ag/HtPQTxORb7QrSpJdMKu0vbBKJPfEncKpqA1Ihn0CoZ1Dy81of398j9tx4TuaY 25 | T1U6U+Pv8vSfx3zYWK8pIpe44L2RLrB27FcRz+8pRPPphXpgY+RdM4kX2TGq2tbz 26 | GDVyz4crL2MjhF2EjD9XoIj8mZEoJmmZ1I+XRL6O1UixpCgp8RW04eWe3fiPpm8m 27 | 1wk8OhwRDqZsN/etRIcsKMfYdIKz0G9KV7s1KSegi+ghp4dkNl3M2Basx7InQJJV 28 | OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH 29 | 6aLcr34YEoP9VhdBLtUpgn2Z9DH2canPLAEnpQW5qrJITirvn5NSUZU8UnOOVkwX 30 | QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS 31 | -----END CERTIFICATE----- 32 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/75d1b2ed.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi 3 | MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 4 | d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg 5 | RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV 6 | UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu 7 | Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG 8 | SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y 9 | ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If 10 | xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV 11 | ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO 12 | DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ 13 | jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ 14 | CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi 15 | EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM 16 | fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY 17 | uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK 18 | chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t 19 | 9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB 20 | hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD 21 | ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 22 | SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd 23 | +SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc 24 | fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa 25 | sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N 26 | cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N 27 | 0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie 28 | 4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI 29 | r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 30 | /YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm 31 | gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ 32 | -----END CERTIFICATE----- 33 | -------------------------------------------------------------------------------- /aws/utils/test/concurrent_hash_map_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | BOOST_AUTO_TEST_SUITE(ConcurrentHashMap) 28 | 29 | BOOST_AUTO_TEST_CASE(ConcurrentInsert) { 30 | std::atomic counter(0); 31 | 32 | aws::utils::ConcurrentHashMap map([&](auto& k) { 33 | counter++; 34 | return new std::string("world"); 35 | }); 36 | 37 | std::chrono::high_resolution_clock::time_point start = 38 | std::chrono::high_resolution_clock::now() + std::chrono::milliseconds(50); 39 | 40 | std::vector results; 41 | aws::mutex mutex; 42 | 43 | const int num_threads = 16; 44 | std::vector threads; 45 | for (int i = 0; i < num_threads; i++) { 46 | threads.emplace_back([&] { 47 | while (std::chrono::high_resolution_clock::now() < start) { 48 | aws::this_thread::yield(); 49 | } 50 | auto s = map["hello"]; 51 | aws::lock_guard lk(mutex); 52 | results.push_back(s); 53 | }); 54 | } 55 | 56 | for (auto& t : threads) { 57 | t.join(); 58 | } 59 | 60 | BOOST_CHECK_EQUAL(counter, 1); 61 | BOOST_CHECK_EQUAL(results.size(), num_threads); 62 | for (auto s : results) { 63 | BOOST_CHECK_EQUAL(s, "world"); 64 | } 65 | } 66 | 67 | BOOST_AUTO_TEST_SUITE_END() 68 | -------------------------------------------------------------------------------- /aws/mutex.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_MUTEX_H_ 17 | #define AWS_MUTEX_H_ 18 | 19 | #include 20 | 21 | #if BOOST_OS_WINDOWS == 0 22 | #include 23 | #include 24 | #include 25 | #include 26 | #endif 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | namespace aws { 38 | 39 | #if BOOST_OS_WINDOWS 40 | namespace threading_namespace = boost; 41 | #else 42 | namespace threading_namespace = std; 43 | #endif 44 | 45 | namespace this_thread = threading_namespace::this_thread; 46 | 47 | using shared_mutex = boost::shared_mutex; 48 | 49 | using thread = threading_namespace::thread; 50 | using condition_variable = threading_namespace::condition_variable; 51 | using mutex = threading_namespace::mutex; 52 | using recursive_mutex = threading_namespace::recursive_mutex; 53 | 54 | template 55 | using unique_lock = threading_namespace::unique_lock; 56 | 57 | template 58 | using shared_lock = threading_namespace::shared_lock; 59 | 60 | template 61 | using lock_guard = threading_namespace::lock_guard; 62 | 63 | static const threading_namespace::defer_lock_t defer_lock{}; 64 | 65 | } //namespace aws 66 | 67 | #endif //AWS_MUTEX_H_ 68 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/KinesisProducerException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | import lombok.Getter; 19 | /** 20 | * This is the wrapper exception thrown and returned to callbacks in onFailure. If 21 | * {@link KinesisProducerConfiguration#setReturnUserRecordOnFailure(boolean)} is set to true, this will contain 22 | * the userRecord associated with the future returned from a {@link KinesisProducer#addUserRecord(UserRecord)} 23 | * call or any of its overloaded methods. 24 | */ 25 | public class KinesisProducerException extends Exception { 26 | private static final long serialVersionUID = 3168271192277927600L; 27 | 28 | /** 29 | * If {@link KinesisProducerConfiguration#setReturnUserRecordOnFailure(boolean)} is set to true, 30 | * this will contain UserRecord associated with this future. If it is false, this will be null. 31 | */ 32 | @Getter 33 | private final UserRecord userRecord; 34 | 35 | public KinesisProducerException(Throwable cause, UserRecord userRecord) { 36 | super(cause); 37 | this.userRecord = userRecord; 38 | } 39 | 40 | public KinesisProducerException(String message, UserRecord userRecord) { 41 | super(message); 42 | this.userRecord = userRecord; 43 | } 44 | 45 | public KinesisProducerException(String message) { 46 | super(message); 47 | this.userRecord = null; 48 | } 49 | 50 | public KinesisProducerException(UserRecord userRecord) { 51 | this.userRecord = userRecord; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /aws/utils/executor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_UTILS_EXECUTOR_H_ 17 | #define AWS_UTILS_EXECUTOR_H_ 18 | 19 | #include 20 | #include 21 | 22 | namespace aws { 23 | namespace utils { 24 | 25 | using Clock = std::chrono::steady_clock; 26 | using TimePoint = Clock::time_point; 27 | 28 | class ScheduledCallback { 29 | public: 30 | virtual ~ScheduledCallback() = default; 31 | 32 | virtual void cancel() = 0; 33 | 34 | virtual bool completed() = 0; 35 | 36 | virtual void reschedule(TimePoint at) = 0; 37 | 38 | virtual TimePoint expiration() = 0; 39 | 40 | virtual void reschedule(std::chrono::milliseconds from_now) { 41 | reschedule(Clock::now() + from_now); 42 | } 43 | 44 | virtual std::chrono::milliseconds time_left() { 45 | auto dur = 46 | std::chrono::duration_cast( 47 | expiration() - Clock::now()); 48 | if (dur.count() > 0) { 49 | return dur; 50 | } else { 51 | return std::chrono::milliseconds(0); 52 | } 53 | } 54 | }; 55 | 56 | class Executor { 57 | public: 58 | using Func = std::function; 59 | 60 | virtual void submit(Func f) = 0; 61 | 62 | virtual std::shared_ptr 63 | schedule(Func f, TimePoint at) = 0; 64 | 65 | virtual std::shared_ptr 66 | schedule(Func f, std::chrono::milliseconds from_now) { 67 | return schedule(std::move(f), Clock::now() + from_now); 68 | } 69 | 70 | virtual size_t num_threads() const noexcept = 0; 71 | 72 | virtual void join() = 0; 73 | }; 74 | 75 | } //namespace utils 76 | } //namespace aws 77 | 78 | #endif //AWS_UTILS_EXECUTOR_H_ 79 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer-sample/README.md: -------------------------------------------------------------------------------- 1 | # KPL Java Sample Application 2 | 3 | ## Setup 4 | 5 | You will need the following: 6 | 7 | + A stream to put into (with any number of shards). There should be no previous data in the stream, creating a new stream is recommended. 8 | + AWS credentials, preferably an IAM role if using EC2 9 | + JDK 10 | + Maven (```brew install maven```, ```sudo apt-get install maven```, [Amazon Linux](https://gist.github.com/sebsto/19b99f1fa1f32cae5d00)) 11 | 12 | Since we'll be running a consumer as well as a producer, the credentials/IAM role should contain permissions for DynamoDB and CloudWatch, in addition to Kinesis. See the [official guide](http://docs.aws.amazon.com/kinesis/latest/dev/learning-kinesis-module-one-iam.html) for details. 13 | 14 | If running locally, set environment variables for the AWS credentials: 15 | 16 | ``` 17 | export AWS_ACCESS_KEY_ID=... 18 | export AWS_SECRET_KEY=... 19 | ``` 20 | 21 | If running in EC2, the KPL will automatically retrieve credentials from any associated IAM role. 22 | 23 | You can also pass credentials to the KPL programmatically during initialization. 24 | 25 | ## Run the Sample Application 26 | 27 | The sample application is a Maven project. It takes dependencies on the KPL and KCL and contains both a producer and consumer. 28 | 29 | Refer to the documentation within the code files for more details about what it's actually doing. 30 | 31 | The sample producer takes optional positional parameters for the stream name, region and duration of the test in seconds. The default stream name is ``test`` and default region is ``us-west-1``. 32 | 33 | The sample consumer takes optional positional parameters for the stream name and region. 34 | 35 | Build the sample application: 36 | 37 | ``` 38 | mvn clean package 39 | ``` 40 | 41 | Then run the producer to put some data into a stream called ``kpltest`` in ``us-west-2`` for ``100 seconds``: 42 | 43 | ``` 44 | mvn exec:java -Dexec.mainClass="software.amazon.kinesis.producer.sample.SampleProducer" -Dexec.args="kpltest us-west-2 100" 45 | ``` 46 | 47 | Finally run the consumer to retrieve the data from a stream called ``kpltest`` in ``us-west-2``: 48 | 49 | ``` 50 | mvn exec:java -Dexec.mainClass="software.amazon.kinesis.producer.sample.SampleConsumer" -Dexec.args="kpltest us-west-2" 51 | ``` 52 | -------------------------------------------------------------------------------- /aws/kinesis/core/put_records_request.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_PUT_RECORDS_REQUEST_H_ 17 | #define AWS_KINESIS_CORE_PUT_RECORDS_REQUEST_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace aws { 24 | namespace kinesis { 25 | namespace core { 26 | 27 | class PutRecordsRequest : public SerializableContainer { 28 | public: 29 | PutRecordsRequest() : total_size_(0) {} 30 | 31 | size_t accurate_size() override { 32 | return total_size_; 33 | } 34 | 35 | std::string serialize() override { 36 | throw std::runtime_error( 37 | "Serialize not implemented for PutRecordsRequest. Use the SDK."); 38 | } 39 | 40 | const std::string& stream() { 41 | if (items_.empty()) { 42 | throw std::runtime_error( 43 | "Cannot get stream name of an empty PutRecordsRequest"); 44 | } 45 | return items_.at(0)->items().at(0)->stream(); 46 | } 47 | 48 | protected: 49 | void after_add(const std::shared_ptr& ur) override { 50 | // This is how the backend counts towards the current 5MB limit 51 | total_size_ += ur->partition_key().length() + ur->accurate_size(); 52 | } 53 | 54 | void after_remove(const std::shared_ptr& ur) override { 55 | total_size_ -= ur->partition_key().length() + ur->accurate_size(); 56 | } 57 | 58 | void after_clear() override { 59 | total_size_ = 0; 60 | } 61 | 62 | private: 63 | size_t total_size_; 64 | }; 65 | 66 | } //namespace core 67 | } //namespace kinesis 68 | } //namespace aws 69 | 70 | #endif //AWS_KINESIS_CORE_PUT_RECORDS_REQUEST_H_ 71 | -------------------------------------------------------------------------------- /aws/metrics/metric.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_METRICS_METRIC_H_ 17 | #define AWS_METRICS_METRIC_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | namespace aws { 27 | namespace metrics { 28 | 29 | class Metric { 30 | public: 31 | using Dimension = std::pair; 32 | 33 | Metric(std::shared_ptr parent, Dimension d) 34 | : parent_(std::move(parent)), 35 | dimension_(std::move(d)), 36 | accumulator_(std::make_shared()) { 37 | if (parent_) { 38 | all_dimensions_.insert(all_dimensions_.end(), 39 | parent_->all_dimensions().cbegin(), 40 | parent_->all_dimensions().cend()); 41 | } 42 | all_dimensions_.push_back(dimension_); 43 | } 44 | 45 | Accumulator& accumulator() const noexcept { 46 | return *accumulator_; 47 | } 48 | 49 | const Dimension& dimension() const noexcept { 50 | return dimension_; 51 | } 52 | 53 | const std::vector& all_dimensions() const noexcept { 54 | return all_dimensions_; 55 | } 56 | 57 | const std::shared_ptr& parent() const noexcept { 58 | return parent_; 59 | } 60 | 61 | void put(double val) { 62 | accumulator_->put(val); 63 | if (parent_) { 64 | parent_->put(val); 65 | } 66 | } 67 | 68 | private: 69 | std::shared_ptr parent_; 70 | Dimension dimension_; 71 | std::vector all_dimensions_; 72 | std::shared_ptr accumulator_; 73 | }; 74 | 75 | } //namespace metrics 76 | } //namespace aws 77 | 78 | #endif //AWS_METRICS_METRIC_H_ 79 | -------------------------------------------------------------------------------- /aggregation-format.md: -------------------------------------------------------------------------------- 1 | # KPL Aggregated Record Format 2 | 3 | ## Intro 4 | 5 | The Amazon Kinesis Producer Library (KPL) aggregates multiple logical user records into a single Amazon Kinesis record for efficient puts. 6 | 7 | We use Google protocol buffers (protobuf) to create a binary file format for this. The Amazon Kinesis Client Library (KCL) implements deaggregation based on this format on the consumer side. 8 | 9 | This document contains the format used. Developers may use this information to produce aggregated records from their own code that will be compatible with the KCL's deaggregation logic. 10 | 11 | ## Format 12 | 13 | All of the user data is contained in a protobuf message. To this, we add a magic number and a checksum. The overall format is as follows: 14 | 15 | ``` 16 | 0 4 N N+15 17 | +---+---+---+---+==================+---+...+---+ 18 | | MAGIC NUMBER | PROTOBUF MESSAGE | MD5 | 19 | +---+---+---+---+==================+---+...+---+ 20 | 21 | ``` 22 | 23 | The magic number contains the 4 bytes `0xF3 0x89 0x9A 0xC2`. 24 | 25 | The protobuf message is as follows: 26 | 27 | ``` 28 | message AggregatedRecord { 29 | repeated string partition_key_table = 1; 30 | repeated string explicit_hash_key_table = 2; 31 | repeated Record records = 3; 32 | } 33 | ``` 34 | 35 | The sub-messages are as follows: 36 | 37 | ``` 38 | message Tag { 39 | required string key = 1; 40 | optional string value = 2; 41 | } 42 | 43 | message Record { 44 | required uint64 partition_key_index = 1; 45 | optional uint64 explicit_hash_key_index = 2; 46 | required bytes data = 3; 47 | repeated Tag tags = 4; 48 | } 49 | ``` 50 | 51 | Note: we use the proto2 language (not proto3). 52 | 53 | The protobuf message allows more efficient partition and explicit hash key packing by allowing multiple records to point to the same key in a table. This feature is optional; implementations can simply store the keys of every record as a separate entry in the tables, even if two or more of them are the same. 54 | 55 | The key tables are zero-indexed; they are simply arrays, and the key indices are indices into those arrays. 56 | 57 | Tags are not yet implemented in the KPL and KCL APIs. 58 | 59 | Lastly, the 16-byte MD5 checksum is computed over the bytes of the serialized protobuf message. 60 | -------------------------------------------------------------------------------- /aws/metrics/metrics_finder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_METRICS_METRICS_FINDER_H_ 17 | #define AWS_METRICS_METRICS_FINDER_H_ 18 | 19 | namespace aws { 20 | namespace metrics { 21 | 22 | class MetricsFinder { 23 | public: 24 | MetricsFinder() = default; 25 | MetricsFinder(const MetricsFinder&) = default; 26 | MetricsFinder(MetricsFinder&&) = default; 27 | MetricsFinder& operator =(const MetricsFinder&) = default; 28 | MetricsFinder& operator =(MetricsFinder&&) = default; 29 | 30 | MetricsFinder& push_dimension(std::string k, std::string v) noexcept { 31 | delims_.emplace_back(canon_.size(), k.length()); 32 | canon_ += (char) 0; 33 | canon_ += k; 34 | canon_ += (char) 1; 35 | canon_ += v; 36 | return *this; 37 | } 38 | 39 | MetricsFinder& pop_dimension() noexcept { 40 | if (!delims_.empty()) { 41 | canon_.erase(delims_.back().first); 42 | delims_.pop_back(); 43 | } 44 | return *this; 45 | } 46 | 47 | std::pair last_dimension() const noexcept { 48 | if (empty()) { 49 | throw std::runtime_error( 50 | "Cannot call last_dimension() on a MetricsFinder that's empty"); 51 | } 52 | 53 | auto& d = delims_.back(); 54 | return std::make_pair(canon_.substr(d.first + 1, d.second), 55 | canon_.substr(d.first + 1 + d.second + 1)); 56 | } 57 | 58 | operator const std::string&() const noexcept { 59 | return canon_; 60 | } 61 | 62 | bool empty() const noexcept { 63 | return delims_.empty(); 64 | } 65 | 66 | private: 67 | std::string canon_; 68 | std::vector> delims_; 69 | }; 70 | 71 | } //namespace metrics 72 | } //namespace aws 73 | 74 | #endif //AWS_METRICS_METRICS_FINDER_H_ 75 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer-sample/src/software/amazon/kinesis/producer/sample/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer.sample; 17 | 18 | import java.io.UnsupportedEncodingException; 19 | import java.math.BigInteger; 20 | import java.nio.ByteBuffer; 21 | import java.util.Random; 22 | 23 | public class Utils { 24 | private static final Random RANDOM = new Random(); 25 | 26 | /** 27 | * @return A random unsigned 128-bit int converted to a decimal string. 28 | */ 29 | public static String randomExplicitHashKey() { 30 | return new BigInteger(128, RANDOM).toString(10); 31 | } 32 | 33 | /** 34 | * Generates a blob containing a UTF-8 string. The string begins with the 35 | * sequence number in decimal notation, followed by a space, followed by 36 | * padding. 37 | * 38 | * @param sequenceNumber 39 | * The sequence number to place at the beginning of the record 40 | * data. 41 | * @param totalLen 42 | * Total length of the data. After the sequence number, padding 43 | * is added until this length is reached. 44 | * @return ByteBuffer containing the blob 45 | */ 46 | public static ByteBuffer generateData(long sequenceNumber, int totalLen) { 47 | StringBuilder sb = new StringBuilder(); 48 | sb.append(Long.toString(sequenceNumber)); 49 | sb.append(" "); 50 | while (sb.length() < totalLen) { 51 | sb.append("a"); 52 | } 53 | try { 54 | return ByteBuffer.wrap(sb.toString().getBytes("UTF-8")); 55 | } catch (UnsupportedEncodingException e) { 56 | throw new RuntimeException(e); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /aws/kinesis/protobuf/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package aws.kinesis.protobuf; 3 | 4 | option java_package = "software.amazon.kinesis.producer.protobuf"; 5 | 6 | message AdditionalDimension { 7 | required string key = 1; 8 | required string value = 2; 9 | required string granularity = 3; 10 | } 11 | 12 | message Configuration { 13 | repeated AdditionalDimension additional_metric_dims = 128; 14 | 15 | optional bool aggregation_enabled = 1 [default = true]; 16 | optional uint64 aggregation_max_count = 2 [default = 4294967295]; 17 | optional uint64 aggregation_max_size = 3 [default = 51200]; 18 | optional string cloudwatch_endpoint = 4 [default = ""]; 19 | optional uint64 cloudwatch_port = 5 [default = 443]; 20 | optional uint64 collection_max_count = 6 [default = 500]; 21 | optional uint64 collection_max_size = 7 [default = 5242880]; 22 | optional uint64 connect_timeout = 8 [default = 6000]; 23 | optional bool enable_core_dumps = 9 [default = false]; 24 | optional bool fail_if_throttled = 10 [default = false]; 25 | optional string kinesis_endpoint = 11 [default = ""]; 26 | optional uint64 kinesis_port = 12 [default = 443]; 27 | optional string log_level = 13 [default = "info"]; 28 | optional uint64 max_connections = 14 [default = 24]; 29 | optional string metrics_granularity = 15 [default = "shard"]; 30 | optional string metrics_level = 16 [default = "detailed"]; 31 | optional string metrics_namespace = 17 [default = "KinesisProducerLibrary"]; 32 | optional uint64 metrics_upload_delay = 18 [default = 60000]; 33 | optional uint64 min_connections = 19 [default = 1]; 34 | optional uint64 rate_limit = 20 [default = 150]; 35 | optional uint64 record_max_buffered_time = 21 [default = 100]; 36 | optional uint64 record_ttl = 22 [default = 30000]; 37 | optional string region = 23 [default = ""]; 38 | optional uint64 request_timeout = 24 [default = 6000]; 39 | optional bool verify_certificate = 25 [default = true]; 40 | optional string proxy_host = 26 [default = ""]; 41 | optional uint64 proxy_port = 27 [default = 443]; 42 | optional string proxy_user_name = 28 [default = ""]; 43 | optional string proxy_password = 29 [default = ""]; 44 | enum ThreadConfig { 45 | PER_REQUEST = 0; 46 | POOLED = 1; 47 | } 48 | optional ThreadConfig thread_config = 30 [default = PER_REQUEST]; 49 | optional uint32 thread_pool_size = 31 [default = 64]; 50 | } 51 | -------------------------------------------------------------------------------- /aws/utils/concurrent_hash_map.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_UTILS_CONCURRENT_HASH_MAP_H_ 17 | #define AWS_UTILS_CONCURRENT_HASH_MAP_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | 26 | namespace aws { 27 | namespace utils { 28 | 29 | template 30 | class ConcurrentHashMap : boost::noncopyable { 31 | public: 32 | using Factory = std::function; 33 | 34 | ConcurrentHashMap(Factory&& factory) 35 | : factory_(std::forward(factory)) {} 36 | 37 | ~ConcurrentHashMap() { 38 | foreach([](auto& k, auto v) { delete v; }); 39 | } 40 | 41 | V& get(const K& key) { 42 | auto it = map_.end(); 43 | 44 | { 45 | ReadLock lock(mutex_); 46 | it = map_.find(key); 47 | } 48 | 49 | if (it == map_.end()) { 50 | WriteLock lock(mutex_); 51 | // Double check that someone didn't get there first 52 | it = map_.find(key); 53 | if (it == map_.end()) { 54 | it = map_.insert(std::make_pair(key, factory_(key))).first; 55 | } 56 | } 57 | 58 | return *it->second; 59 | } 60 | 61 | V& operator [](const K& key) { 62 | return get(key); 63 | } 64 | 65 | void foreach(const std::function& f) { 66 | ReadLock lock(mutex_); 67 | for (auto it = map_.begin(); it != map_.end(); ++it) { 68 | if (it->second != nullptr) { 69 | f(it->first, it->second); 70 | } 71 | } 72 | } 73 | 74 | private: 75 | using Mutex = aws::shared_mutex; 76 | using ReadLock = aws::shared_lock; 77 | using WriteLock = aws::unique_lock; 78 | 79 | Factory factory_; 80 | Mutex mutex_; 81 | std::unordered_map map_; 82 | }; 83 | 84 | } //namespace utils 85 | } //namespace aws 86 | 87 | #endif //AWS_UTILS_CONCURRENT_HASH_MAP_H_ 88 | -------------------------------------------------------------------------------- /aws/metrics/metrics_index.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | namespace aws { 19 | namespace metrics { 20 | 21 | std::shared_ptr 22 | MetricsIndex::get_metric(const MetricsFinder& metrics_finder) { 23 | assert(!metrics_finder.empty()); 24 | 25 | { 26 | ReadLock lk(mutex_); 27 | auto it = metrics_.find(metrics_finder); 28 | if (it != metrics_.end()) { 29 | return it->second; 30 | } 31 | } 32 | 33 | WriteLock lk(mutex_); 34 | 35 | // check whether someone else beat us to it 36 | auto it = metrics_.find(metrics_finder); 37 | if (it != metrics_.end()) { 38 | return it->second; 39 | } 40 | 41 | std::vector keys_to_add; 42 | std::vector> dims; 43 | std::shared_ptr last_node; 44 | MetricsFinder mf(metrics_finder); 45 | 46 | while (!mf.empty() && it == metrics_.end()) { 47 | keys_to_add.push_back(mf); 48 | dims.push_back(mf.last_dimension()); 49 | mf.pop_dimension(); 50 | it = metrics_.find(mf); 51 | } 52 | 53 | if (it != metrics_.end()) { 54 | last_node = it->second; 55 | } 56 | 57 | assert(dims.size() == keys_to_add.size()); 58 | 59 | for (ssize_t i = dims.size() - 1; i >= 0; i--) { 60 | auto m = std::make_shared(std::move(last_node), std::move(dims[i])); 61 | last_node = m; 62 | metrics_.emplace(std::piecewise_construct, 63 | std::forward_as_tuple(std::move(keys_to_add[i])), 64 | std::forward_as_tuple(std::move(m))); 65 | } 66 | 67 | return last_node; 68 | } 69 | 70 | std::vector> MetricsIndex::get_all() { 71 | std::vector> v; 72 | ReadLock lk(mutex_); 73 | for (auto& p : metrics_) { 74 | v.push_back(p.second); 75 | } 76 | return v; 77 | } 78 | 79 | } //namespace metrics 80 | } //namespace aws 81 | -------------------------------------------------------------------------------- /aws/kinesis/core/kinesis_record.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_KINESIS_RECORD_H_ 17 | #define AWS_KINESIS_CORE_KINESIS_RECORD_H_ 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace aws { 26 | namespace kinesis { 27 | namespace core { 28 | 29 | namespace detail { 30 | 31 | class KeySet { 32 | public: 33 | std::pair add(const std::string& s); 34 | bool empty() const; 35 | void clear(); 36 | std::pair remove_one(const std::string& d); 37 | const std::string& first() const; 38 | 39 | private: 40 | std::vector keys_; 41 | std::unordered_map lookup_; 42 | std::unordered_map counts_; 43 | }; 44 | 45 | } // namespace detail 46 | 47 | class KinesisRecord : public SerializableContainer { 48 | public: 49 | static constexpr const char* kMagic = "\xF3\x89\x9A\xC2"; 50 | 51 | KinesisRecord(); 52 | 53 | size_t accurate_size() override; 54 | size_t estimated_size() override; 55 | 56 | std::string serialize() override; 57 | 58 | std::string partition_key() const; 59 | std::string explicit_hash_key() const; 60 | 61 | protected: 62 | void after_add(const std::shared_ptr& ur) override; 63 | void after_remove(const std::shared_ptr& ur) override; 64 | void after_clear() override; 65 | 66 | private: 67 | static const size_t kFixedOverhead = 4 + 16; 68 | 69 | aws::kinesis::protobuf::AggregatedRecord aggregated_record_; 70 | detail::KeySet explicit_hash_keys_; 71 | detail::KeySet partition_keys_; 72 | size_t estimated_size_; 73 | size_t cached_accurate_size_; 74 | bool cached_accurate_size_valid_; 75 | }; 76 | 77 | } //namespace core 78 | } //namespace kinesis 79 | } //namespace aws 80 | 81 | #endif //AWS_KINESIS_CORE_KINESIS_RECORD_H_ 82 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/test/java/software/amazon/kinesis/producer/FileAgeManagerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package software.amazon.kinesis.producer; 16 | 17 | import java.io.File; 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | import java.util.concurrent.ScheduledExecutorService; 21 | 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.mockito.Mock; 25 | import org.mockito.junit.MockitoJUnitRunner; 26 | 27 | import static org.mockito.ArgumentMatchers.anyLong; 28 | import static org.mockito.Mockito.never; 29 | import static org.mockito.Mockito.verify; 30 | import static org.mockito.Mockito.when; 31 | 32 | @RunWith(MockitoJUnitRunner.class) 33 | public class FileAgeManagerTest { 34 | 35 | @Mock 36 | private ScheduledExecutorService executorService; 37 | 38 | @Mock 39 | private File testFile; 40 | 41 | @Test 42 | public void simpleTest() { 43 | FileAgeManager manager = new FileAgeManager(executorService); 44 | List files = makeFileList(); 45 | manager.registerFiles(files); 46 | 47 | verify(testFile).getAbsoluteFile(); 48 | when(testFile.exists()).thenReturn(true); 49 | when(testFile.setLastModified(anyLong())).thenReturn(true); 50 | manager.run(); 51 | 52 | verify(testFile).exists(); 53 | verify(testFile).setLastModified(anyLong()); 54 | } 55 | 56 | private List makeFileList() { 57 | List files = new ArrayList<>(); 58 | files.add(testFile); 59 | when(testFile.getAbsoluteFile()).thenReturn(testFile); 60 | return files; 61 | } 62 | 63 | @Test 64 | public void missingFileTest() { 65 | FileAgeManager manager = new FileAgeManager(executorService); 66 | manager.registerFiles(makeFileList()); 67 | 68 | when(testFile.exists()).thenReturn(false); 69 | manager.run(); 70 | 71 | verify(testFile).exists(); 72 | verify(testFile, never()).setLastModified(anyLong()); 73 | 74 | } 75 | 76 | } -------------------------------------------------------------------------------- /aws/metrics/metrics_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_METRICS_METRICS_CONSTANTS_H_ 17 | #define AWS_METRICS_METRICS_CONSTANTS_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include 27 | 28 | namespace aws { 29 | namespace metrics { 30 | namespace constants { 31 | 32 | enum class Level { 33 | None = 0, 34 | Summary = 100, 35 | Detailed = 200 36 | }; 37 | 38 | enum class Granularity { 39 | Global = 0, 40 | Stream = 100, 41 | Shard = 200 42 | }; 43 | 44 | #define DEF_NAME(X) static constexpr const char* X = #X; 45 | struct Names { 46 | DEF_NAME(Test); 47 | 48 | DEF_NAME(UserRecordsReceived); 49 | DEF_NAME(UserRecordsPending); 50 | DEF_NAME(UserRecordsPut); 51 | DEF_NAME(UserRecordsDataPut); 52 | 53 | DEF_NAME(KinesisRecordsPut); 54 | DEF_NAME(KinesisRecordsDataPut); 55 | 56 | DEF_NAME(ErrorsByCode); 57 | DEF_NAME(AllErrors); 58 | DEF_NAME(RetriesPerRecord); 59 | DEF_NAME(UserRecordExpired); 60 | 61 | DEF_NAME(BufferingTime); 62 | DEF_NAME(RequestTime); 63 | 64 | DEF_NAME(UserRecordsPerKinesisRecord); 65 | DEF_NAME(KinesisRecordsPerPutRecordsRequest); 66 | DEF_NAME(UserRecordsPerPutRecordsRequest); 67 | }; 68 | 69 | struct Units { 70 | DEF_NAME(Count); 71 | DEF_NAME(Milliseconds); 72 | DEF_NAME(Bytes); 73 | DEF_NAME(None); 74 | }; 75 | 76 | struct DimensionNames { 77 | DEF_NAME(MetricName); 78 | DEF_NAME(StreamName); 79 | DEF_NAME(ShardId); 80 | DEF_NAME(ErrorCode); 81 | }; 82 | #undef DEF_NAME 83 | 84 | bool filter(const std::vector>& dimensions, 85 | Level max_level, 86 | Granularity max_granularity); 87 | 88 | std::string unit(const std::string& name); 89 | 90 | Level level(const std::string& s); 91 | 92 | Granularity granularity(const std::string& s); 93 | 94 | } //namespace constants 95 | } //namespace metrics 96 | } //namespace aws 97 | 98 | #endif //AWS_METRICS_METRICS_CONSTANTS_H_ 99 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/GlueSchemaRegistrySerializerInstance.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; 4 | import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializer; 5 | import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializerImpl; 6 | 7 | /** 8 | * Creates a lazy loaded Glue Schema Registry Serializer (GSR-Serializer) instance. 9 | * We only want to initialize the instance if there are UserRecords with schema attribute set. 10 | */ 11 | public final class GlueSchemaRegistrySerializerInstance { 12 | 13 | private volatile GlueSchemaRegistrySerializer instance = null; 14 | private static final String USER_AGENT_APP_NAME = "kpl-1.0.6"; 15 | 16 | /** 17 | * Instantiate GlueSchemaRegistrySerializer using the KinesisProducerConfiguration. 18 | * If the {@link GlueSchemaRegistryConfiguration} is set, it will be used. 19 | * If not, the region from {@link KinesisProducerConfiguration} is used to create one. 20 | * @param configuration {@link KinesisProducerConfiguration} 21 | * @return GlueSchemaRegistrySerializer 22 | */ 23 | public GlueSchemaRegistrySerializer get(KinesisProducerConfiguration configuration) { 24 | GlueSchemaRegistrySerializer local = instance; 25 | if (local == null) { 26 | synchronized (this) { 27 | local = instance; 28 | if (local == null) { 29 | GlueSchemaRegistryConfiguration config = getConfigFromKinesisProducerConfig(configuration); 30 | local = 31 | new GlueSchemaRegistrySerializerImpl( 32 | configuration.getGlueSchemaRegistryCredentialsProvider(), 33 | config); 34 | instance = local; 35 | } 36 | } 37 | } 38 | return instance; 39 | } 40 | 41 | private GlueSchemaRegistryConfiguration getConfigFromKinesisProducerConfig( 42 | KinesisProducerConfiguration configuration) { 43 | GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = configuration.getGlueSchemaRegistryConfiguration(); 44 | if (glueSchemaRegistryConfiguration == null) { 45 | //Reuse the region from KinesisProducerConfiguration. 46 | glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configuration.getRegion()); 47 | } 48 | 49 | glueSchemaRegistryConfiguration.setUserAgentApp(USER_AGENT_APP_NAME); 50 | 51 | return glueSchemaRegistryConfiguration; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/resources/cacerts/b204d74a.0: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCByjELMAkGA1UE 3 | BhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBO 4 | ZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVk 5 | IHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRp 6 | ZmljYXRpb24gQXV0aG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCB 7 | yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZWZXJpU2ln 8 | biBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2lnbiwgSW5jLiAtIEZvciBh 9 | dXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmlt 10 | YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw 11 | ggEKAoIBAQCvJAgIKXo1nmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKz 12 | j/i5Vbext0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIzSdhD 13 | Y2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQGBO+QueQA5N06tRn/ 14 | Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+rCpSx4/VBEnkjWNHiDxpg8v+R70r 15 | fk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/NIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/ 16 | BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2Uv 17 | Z2lmMCEwHzAHBgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy 18 | aXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKvMzEzMA0GCSqG 19 | SIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzEp6B4Eq1iDkVwZMXnl2YtmAl+ 20 | X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKE 21 | KQsTb47bDN0lAtukixlE0kF6BWlKWE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiC 22 | Km0oHw0LxOXnGiYZ4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vE 23 | ZV8NhnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq 24 | -----END CERTIFICATE----- 25 | -----BEGIN CERTIFICATE----- 26 | MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzEL 27 | MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQL 28 | Ey5DbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y 29 | aXR5MB4XDTk2MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UE 30 | BhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz 31 | cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGf 32 | MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69q 33 | RUCPhAwL0TPZ2RHP7gJYHyX3KqhEBarsAx94f56TuZoAqiN91qyFomNFx3In 34 | zPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/isI19wKTakyYbnsZogy1Olhec9vn2a 35 | /iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0GCSqGSIb3DQEBAgUAA4GBALtM 36 | EivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Dolbwdj2wsqFHMc9ikwFPw 37 | TtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNycAA9WjQKZ7aKQRUzk 38 | uxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k 39 | -----END CERTIFICATE----- 40 | -------------------------------------------------------------------------------- /aws/kinesis/core/test/test_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_TEST_TEST_UTILS_H_ 17 | #define AWS_KINESIS_CORE_TEST_TEST_UTILS_H_ 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | namespace aws { 27 | namespace kinesis { 28 | namespace test { 29 | 30 | using UserRecordSharedPtrVector = 31 | std::vector>; 32 | 33 | std::shared_ptr 34 | make_user_record(const std::string& partition_key = "abcd", 35 | const std::string& data = "1234", 36 | const std::string& explicit_hash_key = "", 37 | uint64_t deadline = 100000, 38 | const std::string& stream = "myStream", 39 | uint64_t source_id = 0); 40 | 41 | std::shared_ptr make_user_record_with_hashkey(const std::string& explicit_hash_key = ""); 42 | 43 | // Create a pipe with mkfifo, deleting it when the Fifo instance is destroyed 44 | class Fifo { 45 | public: 46 | Fifo(); 47 | ~Fifo(); 48 | operator const char*() const; 49 | private: 50 | std::string name_; 51 | }; 52 | 53 | // Use only printable chars so we can look at it on the console if tests fail 54 | std::string random_string(size_t len); 55 | 56 | void verify_format(const UserRecordSharedPtrVector& original, 57 | aws::kinesis::core::KinesisRecord& kr, 58 | aws::kinesis::protobuf::AggregatedRecord& container); 59 | 60 | void verify_content(const UserRecordSharedPtrVector& original, 61 | const aws::kinesis::protobuf::AggregatedRecord& result); 62 | 63 | void verify(const UserRecordSharedPtrVector& original, 64 | aws::kinesis::core::KinesisRecord& kr); 65 | 66 | void verify_unaggregated( 67 | const std::shared_ptr& ur, 68 | aws::kinesis::core::KinesisRecord& kr); 69 | 70 | } //namespace test 71 | } //namespace kinesis 72 | } //namespace aws 73 | 74 | #endif //AWS_KINESIS_CORE_TEST_TEST_UTILS_H_ 75 | -------------------------------------------------------------------------------- /aws/utils/test/token_bucket_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | BOOST_AUTO_TEST_SUITE(TokenBucket) 28 | 29 | BOOST_AUTO_TEST_CASE(Basic) { 30 | const size_t max = 200; 31 | const size_t rate = 1000; 32 | 33 | auto start = 34 | std::chrono::steady_clock::time_point( 35 | std::chrono::duration_cast( 36 | (std::chrono::steady_clock::now() + std::chrono::milliseconds(5)) 37 | .time_since_epoch())); 38 | aws::utils::sleep_until(start); 39 | 40 | aws::utils::TokenBucket b; 41 | b.add_token_stream(max, rate); 42 | 43 | BOOST_CHECK(!b.try_take({max + 1})); 44 | BOOST_CHECK(b.try_take({max})); 45 | 46 | aws::utils::sleep_until(start + std::chrono::milliseconds(100)); 47 | 48 | BOOST_CHECK(!b.try_take({110})); 49 | BOOST_CHECK(b.try_take({90})); 50 | 51 | aws::utils::sleep_until(start + std::chrono::milliseconds(max + 200)); 52 | 53 | BOOST_CHECK(!b.try_take({max + 1})); 54 | BOOST_CHECK(b.try_take({max})); 55 | } 56 | 57 | BOOST_AUTO_TEST_CASE(Multiple) { 58 | auto start = 59 | std::chrono::steady_clock::time_point( 60 | std::chrono::duration_cast( 61 | (std::chrono::steady_clock::now() + std::chrono::milliseconds(5)) 62 | .time_since_epoch())); 63 | aws::utils::sleep_until(start); 64 | 65 | aws::utils::TokenBucket b; 66 | b.add_token_stream(200, 1000); 67 | b.add_token_stream(500, 2000); 68 | 69 | BOOST_CHECK(!b.try_take({201, 0})); 70 | BOOST_CHECK(!b.try_take({0, 501})); 71 | BOOST_CHECK(b.try_take({200, 500})); 72 | 73 | aws::utils::sleep_until(start + std::chrono::milliseconds(100)); 74 | 75 | BOOST_CHECK(!b.try_take({110, 0})); 76 | BOOST_CHECK(!b.try_take({0, 220})); 77 | BOOST_CHECK(b.try_take({90, 180})); 78 | 79 | aws::utils::sleep_until(start + std::chrono::milliseconds(500)); 80 | 81 | BOOST_CHECK(!b.try_take({201, 0})); 82 | BOOST_CHECK(!b.try_take({0, 501})); 83 | BOOST_CHECK(b.try_take({200, 500})); 84 | } 85 | 86 | BOOST_AUTO_TEST_SUITE_END() 87 | -------------------------------------------------------------------------------- /aws/kinesis/core/user_record.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | 20 | namespace aws { 21 | namespace kinesis { 22 | namespace core { 23 | 24 | UserRecord::UserRecord(aws::kinesis::protobuf::Message& m) 25 | : hash_key_(0), 26 | finished_(false) { 27 | if (!m.has_put_record()) { 28 | throw std::runtime_error("Message is not a PutRecord"); 29 | } 30 | 31 | source_id_ = m.id(); 32 | auto put_record = m.put_record(); 33 | stream_ = std::move(put_record.stream_name()); 34 | partition_key_ = std::move(put_record.partition_key()); 35 | data_ = std::move(put_record.data()); 36 | has_explicit_hash_key_ = put_record.has_explicit_hash_key(); 37 | 38 | if (has_explicit_hash_key_) { 39 | hash_key_ = uint128_t(put_record.explicit_hash_key()); 40 | } else { 41 | auto digest = aws::utils::md5_binary(partition_key_); 42 | for (int i = 0; i < 16; i++) { 43 | uint128_t p(digest[i]); 44 | p <<= (16 - i - 1) * 8; 45 | hash_key_ += p; 46 | } 47 | } 48 | } 49 | 50 | aws::kinesis::protobuf::Message UserRecord::to_put_record_result() { 51 | aws::kinesis::protobuf::Message m; 52 | m.set_source_id(source_id_); 53 | m.set_id(::rand()); 54 | 55 | auto prr = m.mutable_put_record_result(); 56 | prr->set_success(false); 57 | 58 | for (size_t i = 0; i < attempts_.size(); i++) { 59 | auto a = prr->add_attempts(); 60 | auto delay = (i == 0) 61 | ? attempts_[i].start() - this->arrival() 62 | : attempts_[i].start() - attempts_[i - 1].end(); 63 | a->set_delay( 64 | std::chrono::duration_cast(delay).count()); 65 | a->set_duration(attempts_[i].duration().count()); 66 | a->set_success(attempts_[i]); 67 | 68 | if (attempts_[i]) { 69 | prr->set_shard_id(std::move(attempts_[i].shard_id())); 70 | prr->set_sequence_number(std::move(attempts_[i].sequence_number())); 71 | prr->set_success(true); 72 | } else { 73 | a->set_error_code(std::move(attempts_[i].error_code())); 74 | a->set_error_message(std::move(attempts_[i].error_message())); 75 | } 76 | } 77 | 78 | finished_ = true; 79 | return m; 80 | } 81 | 82 | } //namespace core 83 | } //namespace kinesis 84 | } //namespace aws 85 | -------------------------------------------------------------------------------- /aws/kinesis/core/attempt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_ATTEMPT_H_ 17 | #define AWS_KINESIS_CORE_ATTEMPT_H_ 18 | 19 | #include 20 | #include 21 | 22 | namespace aws { 23 | namespace kinesis { 24 | namespace core { 25 | 26 | class Attempt { 27 | public: 28 | using TimePoint = std::chrono::steady_clock::time_point; 29 | 30 | Attempt& set_start(TimePoint tp = std::chrono::steady_clock::now()) noexcept { 31 | start_ = tp; 32 | return *this; 33 | } 34 | 35 | Attempt& set_end(TimePoint tp = std::chrono::steady_clock::now()) noexcept { 36 | end_ = tp; 37 | return *this; 38 | } 39 | 40 | TimePoint start() const noexcept { 41 | return start_; 42 | } 43 | 44 | TimePoint end() const noexcept { 45 | return end_; 46 | } 47 | 48 | std::chrono::milliseconds duration() const noexcept { 49 | return std::chrono::duration_cast(end_ - start_); 50 | } 51 | 52 | Attempt& set_error(const std::string& code, const std::string& msg) noexcept { 53 | error_ = true; 54 | error_code_ = code; 55 | error_message_ = msg; 56 | return *this; 57 | } 58 | 59 | Attempt& set_result(const std::string& shard_id, 60 | const std::string& sequence_number) noexcept { 61 | error_ = false; 62 | shard_id_ = shard_id; 63 | sequence_number_ = sequence_number; 64 | return *this; 65 | } 66 | 67 | const std::string& shard_id() const noexcept { 68 | return shard_id_; 69 | } 70 | 71 | const std::string& sequence_number() const noexcept { 72 | return sequence_number_; 73 | } 74 | 75 | const std::string& error_code() const noexcept { 76 | return error_code_; 77 | } 78 | 79 | const std::string& error_message() const noexcept { 80 | return error_message_; 81 | } 82 | 83 | operator bool() const noexcept { 84 | return !error_; 85 | } 86 | 87 | private: 88 | TimePoint start_; 89 | TimePoint end_; 90 | bool error_; 91 | std::string error_code_; 92 | std::string error_message_; 93 | std::string sequence_number_; 94 | std::string shard_id_; 95 | }; 96 | 97 | } //namespace core 98 | } //namespace kinesis 99 | } //namespace aws 100 | 101 | #endif //AWS_KINESIS_CORE_ATTEMPT_H_ 102 | -------------------------------------------------------------------------------- /aws/utils/time_sensitive_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_UTILS_TIME_SENSITIVE_QUEUE_H_ 17 | #define AWS_UTILS_TIME_SENSITIVE_QUEUE_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace aws { 24 | namespace utils { 25 | 26 | template 27 | class TimeSensitiveQueue { 28 | public: 29 | void consume_by_deadline( 30 | const std::function&)>& f) { 31 | consume(f); 32 | } 33 | 34 | void consume_expired( 35 | const std::function&)>& f) { 36 | consume([&](const auto& p) { 37 | if (p->expired()) { 38 | f(p); 39 | return true; 40 | } 41 | return false; 42 | }); 43 | } 44 | 45 | template 46 | void insert(U&& item) { 47 | container_.insert(std::forward(item)); 48 | } 49 | 50 | size_t size() const { 51 | return container_.size(); 52 | } 53 | 54 | private: 55 | // Index tags 56 | struct Deadline {}; 57 | struct Expiration {}; 58 | 59 | struct GetDeadline { 60 | using result_type = decltype(((T*) nullptr)->deadline()); 61 | result_type operator ()(const std::shared_ptr& t) { 62 | return t->deadline(); 63 | } 64 | }; 65 | 66 | struct GetExpiration { 67 | using result_type = decltype(((T*) nullptr)->expiration()); 68 | result_type operator ()(const std::shared_ptr& t) { 69 | return t->expiration(); 70 | } 71 | }; 72 | 73 | template 74 | void consume(const std::function&)>& f) { 75 | auto& idx = container_.template get(); 76 | for (auto it = idx.begin(); it != idx.end() && f(*it); it = idx.erase(it)); 77 | } 78 | 79 | boost::multi_index::multi_index_container< 80 | std::shared_ptr, 81 | boost::multi_index::indexed_by< 82 | boost::multi_index::ordered_non_unique< 83 | boost::multi_index::tag, 84 | GetDeadline>, 85 | boost::multi_index::ordered_non_unique< 86 | boost::multi_index::tag, 87 | GetExpiration>>> container_; 88 | }; 89 | 90 | } //namespace utils 91 | } //namespace aws 92 | 93 | #endif //AWS_UTILS_TIME_SENSITIVE_QUEUE_H_ 94 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/FileAgeManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package software.amazon.kinesis.producer; 16 | 17 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.io.File; 22 | import java.util.Collection; 23 | import java.util.HashSet; 24 | import java.util.Set; 25 | import java.util.concurrent.Executors; 26 | import java.util.concurrent.ScheduledExecutorService; 27 | import java.util.concurrent.TimeUnit; 28 | 29 | class FileAgeManager implements Runnable { 30 | 31 | private static final Logger log = LoggerFactory.getLogger(FileAgeManager.class); 32 | 33 | private static final FileAgeManager instance = new FileAgeManager(); 34 | 35 | static synchronized FileAgeManager instance() { 36 | return instance; 37 | } 38 | 39 | private final ScheduledExecutorService executorService; 40 | 41 | private final Set watchedFiles; 42 | 43 | FileAgeManager() { 44 | this(Executors.newScheduledThreadPool(1, 45 | new ThreadFactoryBuilder().setDaemon(true).setNameFormat("KP-FileMaintenance-%04d").build())); 46 | } 47 | 48 | FileAgeManager(ScheduledExecutorService executorService) { 49 | this.watchedFiles = new HashSet<>(); 50 | this.executorService = executorService; 51 | this.executorService.scheduleAtFixedRate(this, 1, 1, TimeUnit.MINUTES); 52 | } 53 | 54 | public synchronized void registerFiles(Collection toRegister) { 55 | for(File f : toRegister) { 56 | watchedFiles.add(f.getAbsoluteFile()); 57 | } 58 | } 59 | 60 | @Override 61 | public synchronized void run() { 62 | for (File file : watchedFiles) { 63 | if (!file.exists()) { 64 | log.error("File '{}' doesn't exist or has been removed. " 65 | + "This could cause problems with the native components. " 66 | + "It's recommended to restart the JVM.", file.getAbsolutePath()); 67 | } else { 68 | if (!file.setLastModified(System.currentTimeMillis())) { 69 | log.warn("Failed to update the last modified time of '{}'.", file.getAbsolutePath()); 70 | } 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /aws/kinesis/protobuf/messages.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | import "config.proto"; 3 | 4 | package aws.kinesis.protobuf; 5 | option java_package = "software.amazon.kinesis.producer.protobuf"; 6 | 7 | message Tag { 8 | required string key = 1; 9 | optional string value = 2; 10 | } 11 | 12 | message Record { 13 | required uint64 partition_key_index = 1; 14 | optional uint64 explicit_hash_key_index = 2; 15 | required bytes data = 3; 16 | repeated Tag tags = 4; 17 | } 18 | 19 | message AggregatedRecord { 20 | repeated string partition_key_table = 1; 21 | repeated string explicit_hash_key_table = 2; 22 | repeated Record records = 3; 23 | } 24 | 25 | message Message { 26 | required uint64 id = 1; 27 | optional uint64 source_id = 2; 28 | 29 | oneof actual_message { 30 | PutRecord put_record = 3; 31 | Flush flush = 4; 32 | PutRecordResult put_record_result = 5; 33 | Configuration configuration = 6; 34 | MetricsRequest metrics_request = 7; 35 | MetricsResponse metrics_response = 8; 36 | SetCredentials set_credentials = 9; 37 | } 38 | } 39 | 40 | message PutRecord { 41 | required string stream_name = 1; 42 | required string partition_key = 2; 43 | optional string explicit_hash_key = 3; 44 | required bytes data = 4; 45 | } 46 | 47 | message Flush { 48 | optional string stream_name = 1; 49 | } 50 | 51 | message Attempt { 52 | required uint32 delay = 1; 53 | required uint32 duration = 2; 54 | required bool success = 3; 55 | optional string error_code = 4; 56 | optional string error_message = 5; 57 | } 58 | 59 | message PutRecordResult { 60 | repeated Attempt attempts = 1; 61 | required bool success = 2; 62 | optional string shard_id = 3; 63 | optional string sequence_number = 4; 64 | } 65 | 66 | // *********** Credentials ************ 67 | 68 | message Credentials { 69 | required string akid = 1; 70 | required string secret_key = 2; 71 | optional string token = 3; 72 | } 73 | 74 | message SetCredentials { 75 | optional bool for_metrics = 1; 76 | required Credentials credentials = 2; 77 | } 78 | 79 | // *********** Metrics ************ 80 | 81 | message Dimension { 82 | required string key = 1; 83 | required string value = 2; 84 | } 85 | 86 | message Stats { 87 | required double count = 1; 88 | required double sum = 2; 89 | required double mean = 3; 90 | required double min = 4; 91 | required double max = 5; 92 | } 93 | 94 | message Metric { 95 | required string name = 1; 96 | repeated Dimension dimensions = 2; 97 | required Stats stats = 3; 98 | required uint64 seconds = 4; 99 | } 100 | 101 | message MetricsRequest { 102 | optional string name = 1; 103 | optional uint64 seconds = 2; 104 | } 105 | 106 | message MetricsResponse { 107 | repeated Metric metrics = 1; 108 | } 109 | -------------------------------------------------------------------------------- /aws/utils/time_sensitive.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_UTILS_TIME_SENSITIVE_H_ 17 | #define AWS_UTILS_TIME_SENSITIVE_H_ 18 | 19 | #include 20 | 21 | #include 22 | 23 | namespace aws { 24 | namespace utils { 25 | 26 | class TimeSensitive : private boost::noncopyable { 27 | public: 28 | using Clock = std::chrono::steady_clock; 29 | using TimePoint = Clock::time_point; 30 | 31 | TimeSensitive() : arrival_(Clock::now()) {} 32 | 33 | TimeSensitive(TimePoint deadline, TimePoint expiration) 34 | : arrival_(Clock::now()), 35 | expiration_(expiration) { 36 | set_deadline(deadline); 37 | } 38 | 39 | TimePoint arrival() const noexcept { 40 | return arrival_; 41 | } 42 | 43 | TimePoint deadline() const noexcept { 44 | return deadline_; 45 | } 46 | 47 | TimePoint expiration() const noexcept { 48 | return expiration_; 49 | } 50 | 51 | void set_deadline(TimePoint tp) noexcept { 52 | deadline_ = tp; 53 | if (!is_undefined(expiration_) && expiration_ < deadline_) { 54 | deadline_ = expiration_; 55 | } 56 | } 57 | 58 | void set_expiration(TimePoint tp) noexcept { 59 | expiration_ = tp; 60 | } 61 | 62 | void set_deadline_from_now(std::chrono::milliseconds ms) { 63 | set_deadline(Clock::now() + ms); 64 | } 65 | 66 | void extend_deadline_from_now(std::chrono::milliseconds ms) { 67 | if (is_undefined(deadline())) { 68 | set_deadline_from_now(ms); 69 | } else { 70 | set_deadline(std::max(deadline_, Clock::now() + ms)); 71 | } 72 | } 73 | 74 | void set_expiration_from_now(std::chrono::milliseconds ms) { 75 | expiration_ = Clock::now() + ms; 76 | } 77 | 78 | bool expired() const noexcept { 79 | return Clock::now() > expiration_; 80 | } 81 | 82 | void inherit_deadline_and_expiration(const TimeSensitive& other) { 83 | if (is_undefined(this->deadline()) || 84 | other.deadline() < this->deadline()) { 85 | this->set_deadline(other.deadline()); 86 | } 87 | 88 | if (is_undefined(this->expiration()) || 89 | other.expiration() < this->expiration()) { 90 | this->set_expiration(other.expiration()); 91 | } 92 | } 93 | 94 | private: 95 | TimePoint arrival_; 96 | TimePoint deadline_; 97 | TimePoint expiration_; 98 | 99 | static bool is_undefined(const TimePoint& tp) { 100 | return tp.time_since_epoch().count() == 0; 101 | } 102 | 103 | }; 104 | 105 | } //namespace utils 106 | } //namespace aws 107 | 108 | #endif //AWS_UTILS_TIME_SENSITIVE_H_ 109 | -------------------------------------------------------------------------------- /aws/kinesis/core/test/put_records_request_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace { 26 | 27 | using KinesisRecordSharedPtrVector = 28 | std::vector>; 29 | 30 | auto make_kinesis_record(size_t min_serialized_size = 0) { 31 | auto kr = std::make_shared(); 32 | int num_user_records = 1 + (::rand() % 100); 33 | 34 | auto make_ur = [] { 35 | return aws::kinesis::test::make_user_record( 36 | std::to_string(::rand()), 37 | std::to_string(::rand()), 38 | std::to_string(::rand()), 39 | 10000, 40 | "myStream"); 41 | }; 42 | 43 | for (int i = 0; i < num_user_records; i++) { 44 | kr->add(make_ur()); 45 | } 46 | 47 | if (min_serialized_size > 0) { 48 | while (kr->accurate_size() < min_serialized_size) { 49 | kr->add(make_ur()); 50 | } 51 | } 52 | 53 | return kr; 54 | } 55 | 56 | } //namespace 57 | 58 | BOOST_AUTO_TEST_SUITE(PutRecordsRequest) 59 | 60 | BOOST_AUTO_TEST_CASE(SizePrediction) { 61 | aws::kinesis::core::PutRecordsRequest prr; 62 | BOOST_CHECK_EQUAL(prr.accurate_size(), 0); 63 | BOOST_CHECK_EQUAL(prr.estimated_size(), prr.accurate_size()); 64 | 65 | std::stack sizes; 66 | sizes.push(0); 67 | const int N = 100; 68 | 69 | for (int i = 0; i < N; i++) { 70 | auto kr = make_kinesis_record(); 71 | prr.add(kr); 72 | 73 | size_t expected_growth = 74 | kr->serialize().length() + kr->partition_key().length(); 75 | size_t expected_size = sizes.top() + expected_growth; 76 | BOOST_CHECK_EQUAL(prr.accurate_size(), expected_size); 77 | BOOST_CHECK_EQUAL(prr.estimated_size(), prr.accurate_size()); 78 | 79 | sizes.push(expected_size); 80 | } 81 | 82 | for (int i = 0; i < N; i++) { 83 | BOOST_CHECK_EQUAL(prr.accurate_size(), sizes.top()); 84 | BOOST_CHECK_EQUAL(prr.estimated_size(), prr.accurate_size()); 85 | 86 | sizes.pop(); 87 | prr.remove_last(); 88 | } 89 | 90 | BOOST_CHECK_EQUAL(prr.accurate_size(), 0); 91 | BOOST_CHECK_EQUAL(prr.estimated_size(), prr.accurate_size()); 92 | } 93 | 94 | BOOST_AUTO_TEST_CASE(StreamName) { 95 | aws::kinesis::core::PutRecordsRequest prr; 96 | auto kr = make_kinesis_record(); 97 | prr.add(kr); 98 | BOOST_CHECK_EQUAL(prr.stream(), "myStream"); 99 | } 100 | 101 | BOOST_AUTO_TEST_SUITE_END() 102 | -------------------------------------------------------------------------------- /aws/kinesis/core/serializable_container.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_SERIALIZABLE_CONTAINER_H_ 17 | #define AWS_KINESIS_CORE_SERIALIZABLE_CONTAINER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | namespace aws { 26 | namespace kinesis { 27 | namespace core { 28 | 29 | template 30 | class SerializableContainer : public aws::utils::TimeSensitive { 31 | public: 32 | using TimePoint = std::chrono::steady_clock::time_point; 33 | 34 | virtual ~SerializableContainer() = default; 35 | 36 | virtual void add(const std::shared_ptr& new_item) { 37 | deadlines_and_expirations_.push( 38 | std::make_pair( 39 | this->deadline(), 40 | this->expiration())); 41 | this->inherit_deadline_and_expiration(*new_item); 42 | 43 | items_.push_back(new_item); 44 | after_add(new_item); 45 | } 46 | 47 | virtual std::shared_ptr remove_last() { 48 | if (items_.empty()) { 49 | return std::shared_ptr(); 50 | } 51 | 52 | auto& prev_times = deadlines_and_expirations_.top(); 53 | set_deadline(prev_times.first); 54 | set_expiration(prev_times.second); 55 | deadlines_and_expirations_.pop(); 56 | 57 | auto i = std::move(items_.back()); 58 | items_.pop_back(); 59 | after_remove(i); 60 | return i; 61 | } 62 | 63 | virtual size_t estimated_size() { 64 | return accurate_size(); 65 | } 66 | 67 | virtual size_t accurate_size() = 0; 68 | 69 | virtual void clear() { 70 | set_deadline_from_now(std::chrono::hours(0xFFFFFFFF)); 71 | set_expiration_from_now(std::chrono::hours(0xFFFFFFFF)); 72 | while (!deadlines_and_expirations_.empty()) { 73 | deadlines_and_expirations_.pop(); 74 | } 75 | 76 | items_.clear(); 77 | after_clear(); 78 | } 79 | 80 | virtual std::string serialize() = 0; 81 | 82 | const std::vector>& items() noexcept { 83 | return items_; 84 | } 85 | 86 | size_t size() const noexcept { 87 | return items_.size(); 88 | } 89 | 90 | bool empty() const noexcept { 91 | return items_.empty(); 92 | } 93 | 94 | protected: 95 | virtual void after_add(const std::shared_ptr& new_item) {} 96 | virtual void after_remove(const std::shared_ptr& item) {} 97 | virtual void after_clear() {} 98 | 99 | std::vector> items_; 100 | std::stack> deadlines_and_expirations_; 101 | }; 102 | 103 | } //namespace core 104 | } //namespace kinesis 105 | } //namespace aws 106 | 107 | #endif //AWS_KINESIS_CORE_SERIALIZABLE_CONTAINER_H_ 108 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/Attempt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | import software.amazon.kinesis.producer.protobuf.Messages; 19 | 20 | /** 21 | * Represents one attempt at writing a record to the backend. The attempt may or 22 | * may not be successful. If unsuccessful, an error code and error message are 23 | * provided. In addition, data about latency are also provided. Each record may 24 | * have multiple attempts. 25 | * 26 | * @author chaodeng 27 | * @see UserRecordResult 28 | */ 29 | public class Attempt { 30 | private int delay; 31 | private int duration; 32 | private String errorMessage; 33 | private String errorCode; 34 | private boolean success; 35 | 36 | public Attempt(int delay, int duration, String errorMessage, String errorCode, boolean success) { 37 | this.delay = delay; 38 | this.duration = duration; 39 | this.errorMessage = errorMessage; 40 | this.errorCode = errorCode; 41 | this.success = success; 42 | } 43 | 44 | /** 45 | * 46 | * @return Delay in milliseconds between the start of this attempt and the 47 | * previous attempt. If this is the first attempt, then returns the 48 | * delay between the message reaching the daemon and the first 49 | * attempt. 50 | */ 51 | public int getDelay() { 52 | return delay; 53 | } 54 | 55 | /** 56 | * @return Duration of the attempt. Mainly consists of network and server 57 | * latency, but also includes processing overhead within the daemon. 58 | */ 59 | public int getDuration() { 60 | return duration; 61 | } 62 | 63 | /** 64 | * 65 | * @return Error message associated with this attempt. Null if no error 66 | * (i.e. successful). 67 | */ 68 | public String getErrorMessage() { 69 | return errorMessage; 70 | } 71 | 72 | /** 73 | * 74 | * @return Error code associated with this attempt. Null if no error 75 | * (i.e. successful). 76 | */ 77 | public String getErrorCode() { 78 | return errorCode; 79 | } 80 | 81 | /** 82 | * 83 | * @return Whether the attempt was successful. If true, then the record has 84 | * been confirmed by the backend. 85 | */ 86 | public boolean isSuccessful() { 87 | return success; 88 | } 89 | 90 | protected static Attempt fromProtobufMessage(Messages.Attempt a) { 91 | return new Attempt( 92 | a.getDelay(), 93 | a.getDuration(), 94 | a.hasErrorMessage() ? a.getErrorMessage() : null, 95 | a.hasErrorCode() ? a.getErrorCode() : null, 96 | a.getSuccess()); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /aws/utils/token_bucket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_UTILS_TOKEN_BUCKET_H_ 17 | #define AWS_UTILS_TOKEN_BUCKET_H_ 18 | 19 | #include 20 | 21 | #include 22 | 23 | namespace aws { 24 | namespace utils { 25 | 26 | namespace detail { 27 | 28 | class TokenStream { 29 | public: 30 | TokenStream(double max, double rate) noexcept 31 | : max_(max), 32 | rate_(rate), 33 | tokens_(0) {} 34 | 35 | double tokens() noexcept { 36 | auto now = Clock::now(); 37 | // We don't set the last_ timestamp if growth is zero because we might end 38 | // up never growing the tokens if the invocations are so close together 39 | // that growth is always zero. This can happen if the clock does not have 40 | // enough resolution such that the number of seconds returned is 0. 41 | double growth = rate_ * aws::utils::seconds_between(last_, now); 42 | if (growth > 0) { 43 | tokens_ = std::min(max_, tokens_ + growth); 44 | last_ = now; 45 | } 46 | return tokens_; 47 | } 48 | 49 | void take(double n) noexcept { 50 | if (n > tokens()) { 51 | throw std::runtime_error("Not enough tokens"); 52 | } 53 | tokens_ -= n; 54 | } 55 | 56 | private: 57 | using Clock = std::chrono::steady_clock; 58 | 59 | double max_; 60 | double rate_; 61 | double tokens_; 62 | Clock::time_point last_; 63 | }; 64 | 65 | } //namespace detail 66 | 67 | class TokenBucket { 68 | public: 69 | void add_token_stream(double max, double rate) { 70 | streams_.emplace_back(max, rate); 71 | } 72 | 73 | bool try_take(const std::initializer_list& num_tokens) { 74 | if (!can_take(num_tokens)) { 75 | return false; 76 | } 77 | 78 | auto stream_it = streams_.begin(); 79 | auto nt_it = num_tokens.begin(); 80 | while (stream_it != streams_.end()) { 81 | stream_it->take(*nt_it); 82 | stream_it++; 83 | nt_it++; 84 | } 85 | 86 | return true; 87 | } 88 | 89 | bool can_take(const std::initializer_list& num_tokens) { 90 | if (num_tokens.size() != streams_.size()) { 91 | throw std::runtime_error("Size of num_tokens list must be the same as " 92 | "the number of token streams in the bucket"); 93 | } 94 | 95 | auto stream_it = streams_.begin(); 96 | auto nt_it = num_tokens.begin(); 97 | while (stream_it != streams_.end()) { 98 | if (*nt_it > stream_it->tokens()) { 99 | return false; 100 | } 101 | stream_it++; 102 | nt_it++; 103 | } 104 | return true; 105 | } 106 | 107 | private: 108 | std::vector streams_; 109 | }; 110 | 111 | } //namespace utils 112 | } //namespace aws 113 | 114 | #endif //AWS_UTILS_TOKEN_BUCKET_H_ 115 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/UserRecordResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | import software.amazon.kinesis.producer.protobuf.Messages; 22 | import com.google.common.collect.ImmutableList; 23 | 24 | /** 25 | * The result of a {@link KinesisProducer#addUserRecord} operation. If 26 | * successful, the shard id and sequence number assigned by the backend are 27 | * provided. A list of {@link Attempt}s is also provided with details about each 28 | * attempt made. 29 | * 30 | * @author chaodeng 31 | * @see Attempt 32 | */ 33 | public class UserRecordResult { 34 | private List attempts; 35 | private String sequenceNumber; 36 | private String shardId; 37 | private boolean successful; 38 | 39 | public UserRecordResult(List attempts, String sequenceNumber, String shardId, boolean successful) { 40 | this.attempts = attempts; 41 | this.sequenceNumber = sequenceNumber; 42 | this.shardId = shardId; 43 | this.successful = successful; 44 | } 45 | 46 | /** 47 | * 48 | * @return List of {@link Attempt}s, in the order they were made. 49 | */ 50 | public List getAttempts() { 51 | return attempts; 52 | } 53 | 54 | /** 55 | * 56 | * @return The sequence number assigned by the backend to this record. 57 | * Multiple records may have the same sequence number if aggregation 58 | * is enabled. Will be null if the put failed. 59 | */ 60 | public String getSequenceNumber() { 61 | return sequenceNumber; 62 | } 63 | 64 | /** 65 | * 66 | * @return Shard ID returned by the backend. The record was written to this 67 | * shard. Will be null if the put failed. 68 | */ 69 | public String getShardId() { 70 | return shardId; 71 | } 72 | 73 | /** 74 | * 75 | * @return Whether the record put was successful. If true, then the record 76 | * has been confirmed by the backend. 77 | */ 78 | public boolean isSuccessful() { 79 | return successful; 80 | } 81 | 82 | protected static UserRecordResult fromProtobufMessage(Messages.PutRecordResult r) { 83 | final List attempts = new ArrayList<>(r.getAttemptsCount()); 84 | for (Messages.Attempt a : r.getAttemptsList()) { 85 | attempts.add(Attempt.fromProtobufMessage(a)); 86 | } 87 | return new UserRecordResult( 88 | new ImmutableList.Builder().addAll(attempts).build(), 89 | r.hasSequenceNumber() ? r.getSequenceNumber() : null, 90 | r.hasShardId() ? r.getShardId() : null, 91 | r.getSuccess()); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /aws/kinesis/core/user_record.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_USER_RECORD_H_ 17 | #define AWS_KINESIS_CORE_USER_RECORD_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace aws { 31 | namespace kinesis { 32 | namespace core { 33 | 34 | class UserRecord : public aws::utils::TimeSensitive { 35 | public: 36 | using uint128_t = boost::multiprecision::uint128_t; 37 | 38 | // This will move strings out of m; m will not be valid after this. 39 | UserRecord(aws::kinesis::protobuf::Message& m); 40 | 41 | void add_attempt(const Attempt& a) { 42 | attempts_.push_back(a); 43 | } 44 | 45 | uint64_t source_id() const noexcept { 46 | return source_id_; 47 | } 48 | 49 | const std::string& stream() const noexcept { 50 | return stream_; 51 | } 52 | 53 | const std::string& partition_key() const noexcept { 54 | return partition_key_; 55 | } 56 | 57 | const uint128_t& hash_key() const noexcept { 58 | return hash_key_; 59 | } 60 | 61 | const std::string& data() const noexcept { 62 | return data_; 63 | } 64 | 65 | const std::vector& attempts() const noexcept { 66 | return attempts_; 67 | } 68 | 69 | const bool finished() const noexcept { 70 | return finished_; 71 | } 72 | 73 | boost::optional predicted_shard() const noexcept { 74 | return predicted_shard_; 75 | } 76 | 77 | void predicted_shard(uint64_t sid) noexcept { 78 | predicted_shard_ = sid; 79 | } 80 | 81 | void reset_predicted_shard() noexcept { 82 | predicted_shard_ = boost::none; 83 | } 84 | 85 | std::string hash_key_decimal_str() const noexcept { 86 | std::stringstream ss; 87 | ss << hash_key_; 88 | return ss.str(); 89 | } 90 | 91 | boost::optional explicit_hash_key() const noexcept { 92 | if (has_explicit_hash_key_) { 93 | return hash_key_decimal_str(); 94 | } else { 95 | return boost::none; 96 | } 97 | } 98 | 99 | // This will move strings from this instance into the Message. This instance 100 | // will not be valid after this. 101 | aws::kinesis::protobuf::Message to_put_record_result(); 102 | 103 | private: 104 | uint64_t source_id_; 105 | std::string stream_; 106 | std::string partition_key_; 107 | uint128_t hash_key_; 108 | std::string data_; 109 | std::vector attempts_; 110 | boost::optional predicted_shard_; 111 | bool has_explicit_hash_key_; 112 | bool finished_; 113 | }; 114 | 115 | } //namespace core 116 | } //namespace kinesis 117 | } //namespace aws 118 | 119 | #endif //AWS_KINESIS_CORE_USER_RECORD_H_ 120 | -------------------------------------------------------------------------------- /aws/utils/test/spin_lock_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace { 26 | 27 | template 30 | void test(const std::string test_name, 31 | const size_t num_threads = aws::thread::hardware_concurrency()) { 32 | auto start = std::chrono::high_resolution_clock::now() + 33 | std::chrono::milliseconds(100); 34 | 35 | Mutex mutex; 36 | volatile size_t counter = 0; 37 | std::vector threads; 38 | #ifdef DEBUG 39 | std::vector debug_stats; 40 | debug_stats.resize(num_threads); 41 | #endif 42 | 43 | std::size_t thread_count = 0; 44 | while (threads.size() < num_threads) { 45 | std::size_t thread_index = thread_count++; 46 | threads.push_back(aws::thread([&, thread_index] { 47 | aws::utils::sleep_until(start); 48 | for (size_t i = 0; i < cycles_per_thread; i++) { 49 | aws::lock_guard lk(mutex); 50 | for (size_t j = 0; j < loop_per_cycle; j++) { 51 | counter++; 52 | } 53 | } 54 | #ifdef DEBUG 55 | debug_stats[thread_index] = mutex.get_debug_stats(); 56 | #endif 57 | })); 58 | } 59 | 60 | for (auto& t : threads) { 61 | t.join(); 62 | } 63 | 64 | double seconds = aws::utils::seconds_since(start); 65 | 66 | BOOST_REQUIRE_EQUAL(counter, 67 | cycles_per_thread * loop_per_cycle * num_threads); 68 | 69 | LOG(info) << test_name << ": " << num_threads << " threads: " 70 | << cycles_per_thread * num_threads / seconds 71 | << " ops per sec"; 72 | 73 | #ifdef DEBUG 74 | LOG(info) << test_name << ": DebugStats"; 75 | LOG(info) << "\t" 76 | << std::setw(20) << "Acquired Count" 77 | << std::setw(20) << "Acquired with Lock" 78 | << std::setw(20) << "Total Spins"; 79 | std::for_each(debug_stats.begin(), debug_stats.end(), [](typename Mutex::DebugStats& d) { 80 | LOG(info) << "\t" 81 | << std::setw(20) << d.acquired_count 82 | << std::setw(20) << d.acquired_with_lock 83 | << std::setw(20) << d.total_spins; 84 | }); 85 | #endif 86 | } 87 | 88 | } //namespace 89 | 90 | BOOST_AUTO_TEST_SUITE(SpinLock) 91 | 92 | BOOST_AUTO_TEST_CASE(SpinLock) { 93 | for (size_t i : {1, 4, 8}) { 94 | test("SpinLock", i); 95 | } 96 | } 97 | 98 | BOOST_AUTO_TEST_CASE(TicketSpinLock) { 99 | for (size_t i : {1, 4, 8, 16, 32}) { 100 | test("TicketSpinLock", i); 101 | } 102 | } 103 | 104 | BOOST_AUTO_TEST_CASE(StdMutex) { 105 | for (size_t i : {1, 4, 8}) { 106 | test("std::mutex", i); 107 | } 108 | } 109 | 110 | BOOST_AUTO_TEST_SUITE_END() 111 | -------------------------------------------------------------------------------- /aws/auth/mutable_static_creds_provider.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "mutable_static_creds_provider.h" 17 | 18 | namespace { 19 | // 20 | // Provides a thread scoped copy of the current credentials to an executing thread. 21 | // This makes the most difference when using a thread pool, as the retrieval of the 22 | // credentials will only require a lock when the credentials version changes. 23 | // 24 | thread_local aws::auth::VersionedCredentials current_creds; 25 | } 26 | 27 | using namespace aws::auth; 28 | 29 | VersionedCredentials::VersionedCredentials(std::uint64_t version, const std::string& akid, const std::string& sk, const std::string& token) : 30 | version_(version), creds_(Aws::Auth::AWSCredentials(akid, sk, token)) { 31 | } 32 | 33 | MutableStaticCredentialsProvider::MutableStaticCredentialsProvider(const std::string& akid, 34 | const std::string& sk, 35 | std::string token) : 36 | creds_(std::make_shared(1, akid, sk, token)), version_(1) { 37 | } 38 | 39 | void MutableStaticCredentialsProvider::set_credentials(const std::string& akid, const std::string& sk, std::string token) { 40 | std::lock_guard lock(update_mutex_); 41 | 42 | std::uint64_t next_version = version_ + 1; 43 | 44 | // 45 | // Since the credentials are created with the expected next version, and the entire update 46 | // is protected by a lock we can't get into a scenario where one of the consumers has 47 | // a mismatched version and credentials. 48 | // 49 | std::shared_ptr new_credentials = std::make_shared(next_version, akid, sk, token); 50 | 51 | // 52 | // This update the credentials atomically using the shared_ptr specific atomic operations, 53 | // and doesn't require a specific lock on the shared_ptr during the update. The lock 54 | // taken previously is to prevent two credential updates at the same time. 55 | // 56 | // The global version change allows the threads to detect the updated version. Once detected 57 | // the threads will pull the updated credential to their own thread local copy. 58 | // 59 | std::atomic_store(&creds_, new_credentials); 60 | version_ = next_version; 61 | } 62 | 63 | Aws::Auth::AWSCredentials MutableStaticCredentialsProvider::GetAWSCredentials() { 64 | // 65 | // Check to see if the credentials have been updated. If they have load the credentials 66 | // and update the thread local. 67 | // 68 | // If the credentials are changing rapidly it's possible that the thread will read an 69 | // old version of the credentials. Should that occur the next read will update to the 70 | // most current version. 71 | // 72 | // This check still works in the very unlikely event that the next_version value 73 | // wraps around. 74 | // 75 | if (current_creds.version_ != version_) { 76 | std::shared_ptr updated = std::atomic_load(&creds_); 77 | current_creds = *updated; 78 | } 79 | return current_creds.creds_; 80 | } 81 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/awslabs/amazon-kinesis-producer/issues), or [recently closed](https://github.com/awslabs/amazon-kinesis-producer/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/amazon-kinesis-producer/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/awslabs/amazon-kinesis-producer/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer-sample/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | software.amazon.kinesis 5 | amazon-kinesis-producer-sample 6 | 1.0.6 7 | 8 | src 9 | 10 | 11 | maven-compiler-plugin 12 | 3.11.0 13 | 14 | 1.8 15 | 1.8 16 | 17 | 18 | 19 | org.apache.maven.plugins 20 | maven-shade-plugin 21 | 3.4.1 22 | 23 | 24 | package 25 | 26 | shade 27 | 28 | 29 | 30 | 31 | software.amazon.kinesis.producer.sample.SampleProducer 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | KinesisProducerLibrary Sample Application 41 | 42 | 43 | 44 | software.amazon.kinesis 45 | amazon-kinesis-client 46 | 3.0.1 47 | 48 | 49 | com.google.protobuf 50 | protobuf-java 51 | 52 | 53 | 54 | 55 | software.amazon.kinesis 56 | amazon-kinesis-producer 57 | 1.0.6 58 | 59 | 60 | org.slf4j 61 | slf4j-simple 62 | 2.0.0 63 | 64 | 65 | javax.validation 66 | validation-api 67 | 2.0.1.Final 68 | 69 | 70 | javax.el 71 | javax.el-api 72 | 3.0.0 73 | 74 | 75 | org.glassfish.web 76 | javax.el 77 | 2.2.6 78 | 79 | 80 | org.hibernate.validator 81 | hibernate-validator 82 | 6.2.0.Final 83 | 84 | 85 | org.hibernate.validator 86 | hibernate-validator-annotation-processor 87 | 6.0.18.Final 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /aws/kinesis/core/collector.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_COLLECTOR_H_ 17 | #define AWS_KINESIS_CORE_COLLECTOR_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace aws { 26 | namespace kinesis { 27 | namespace core { 28 | 29 | class Collector : boost::noncopyable { 30 | public: 31 | using FlushCallback = 32 | std::function)>; 33 | 34 | Collector( 35 | const std::shared_ptr& executor, 36 | const FlushCallback& flush_callback, 37 | const std::shared_ptr& config, 38 | aws::utils::flush_statistics_aggregator& flush_stats, 39 | const std::shared_ptr& metrics_manager = 40 | std::make_shared()) 41 | : flush_callback_(flush_callback), 42 | reducer_(executor, 43 | [this](auto prr) { this->handle_flush(std::move(prr)); }, 44 | config->collection_max_size(), 45 | config->collection_max_count(), 46 | flush_stats, 47 | [this](auto kr) { return this->should_flush(kr); }), 48 | buffered_data_([](auto) { return new std::atomic(0); }) {} 49 | 50 | std::shared_ptr 51 | put(const std::shared_ptr& kr) { 52 | auto prr = reducer_.add(kr); 53 | decrease_buffered_data(prr); 54 | return prr; 55 | } 56 | 57 | void flush() { 58 | reducer_.flush(); 59 | } 60 | 61 | private: 62 | // We don't want any individual shard to accumulate too much data 63 | // because that makes traffic to that shard bursty, and might cause 64 | // throttling, so we flush whenever a shard reaches a certain limit. 65 | bool should_flush(const std::shared_ptr kr) { 66 | auto shard_id = kr->items().front()->predicted_shard(); 67 | if (shard_id) { 68 | auto d = buffered_data_[*shard_id] += kr->accurate_size(); 69 | if (d >= 256 * 1024) { 70 | return true; 71 | } 72 | } 73 | return false; 74 | } 75 | 76 | void decrease_buffered_data(const std::shared_ptr& prr) { 77 | if (!prr) { 78 | return; 79 | } 80 | 81 | for (auto& kr : prr->items()) { 82 | auto shard_id = kr->items().front()->predicted_shard(); 83 | if (shard_id) { 84 | buffered_data_[*shard_id] -= kr->accurate_size(); 85 | } 86 | } 87 | } 88 | 89 | void handle_flush(std::shared_ptr prr) { 90 | decrease_buffered_data(prr); 91 | flush_callback_(std::move(prr)); 92 | } 93 | 94 | FlushCallback flush_callback_; 95 | std::shared_ptr metrics_manager_; 96 | Reducer reducer_; 97 | aws::utils::ConcurrentHashMap> buffered_data_; 98 | }; 99 | 100 | } //namespace core 101 | } //namespace kinesis 102 | } //namespace aws 103 | 104 | #endif //AWS_KINESIS_CORE_COLLECTOR_H_ 105 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/Metric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package software.amazon.kinesis.producer; 17 | 18 | import java.util.Collections; 19 | import java.util.LinkedHashMap; 20 | import java.util.Map; 21 | 22 | import software.amazon.kinesis.producer.protobuf.Messages.Dimension; 23 | import software.amazon.kinesis.producer.protobuf.Messages.Stats; 24 | 25 | /** 26 | * A metric consists of a name, a list of dimensions, a set of statistics, and 27 | * the duration over which the statistics were collected. 28 | * 29 | *

30 | * There are typically many Metric instances for each metric name. Each one will 31 | * have a different list of dimensions. 32 | * 33 | *

34 | * This class is immutable. 35 | * 36 | * @author chaodeng 37 | * 38 | */ 39 | public class Metric { 40 | private final String name; 41 | private long duration; 42 | private final Map dimensions; 43 | private final double sum; 44 | private final double mean; 45 | private final double sampleCount; 46 | private final double min; 47 | private final double max; 48 | 49 | /** 50 | * Gets the dimensions of this metric. The returned map has appropriate 51 | * iteration order and is immutable. 52 | * 53 | * @return Immutable map containing the dimensions. 54 | */ 55 | public Map getDimensions() { 56 | return Collections.unmodifiableMap(dimensions); 57 | } 58 | 59 | public double getSum() { 60 | return sum; 61 | } 62 | 63 | public double getMean() { 64 | return mean; 65 | } 66 | 67 | public double getSampleCount() { 68 | return sampleCount; 69 | } 70 | 71 | public double getMin() { 72 | return min; 73 | } 74 | 75 | public double getMax() { 76 | return max; 77 | } 78 | 79 | public String getName() { 80 | return name; 81 | } 82 | 83 | /** 84 | * @return The number of seconds over which the statistics in this Metric 85 | * instance was accumulated. For example, a duration of 10 means 86 | * that the statistics in this Metric instance represents 10 seconds 87 | * worth of samples. 88 | */ 89 | public long getDuration() { 90 | return duration; 91 | } 92 | 93 | protected Metric(software.amazon.kinesis.producer.protobuf.Messages.Metric m) { 94 | this.name = m.getName(); 95 | this.duration = m.getSeconds(); 96 | 97 | dimensions = new LinkedHashMap(); 98 | for (Dimension d : m.getDimensionsList()) { 99 | dimensions.put(d.getKey(), d.getValue()); 100 | } 101 | 102 | Stats s = m.getStats(); 103 | this.max = s.getMax(); 104 | this.mean = s.getMean(); 105 | this.min = s.getMin(); 106 | this.sum = s.getSum(); 107 | this.sampleCount = s.getCount(); 108 | } 109 | 110 | @Override 111 | public String toString() { 112 | return "Metric [name=" + name + ", duration=" + duration + ", dimensions=" + dimensions + ", sum=" + sum 113 | + ", mean=" + mean + ", sampleCount=" + sampleCount + ", min=" + min + ", max=" + max + "]"; 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /aws/kinesis/core/put_records_context.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_PUT_RECORDS_CONTEXT_H_ 17 | #define AWS_KINESIS_CORE_PUT_RECORDS_CONTEXT_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace aws { 31 | namespace kinesis { 32 | namespace core { 33 | 34 | class PutRecordsContext : public Aws::Client::AsyncCallerContext { 35 | public: 36 | PutRecordsContext(std::string stream, 37 | std::string stream_arn, 38 | std::vector> records) 39 | : stream_(std::move(stream)), 40 | stream_arn_(std::move(stream_arn)), 41 | records_(std::move(records)) {} 42 | 43 | const std::string& get_stream() const { 44 | return stream_; 45 | } 46 | 47 | const std::string& get_stream_arn() const { 48 | return stream_arn_; 49 | } 50 | 51 | std::chrono::steady_clock::time_point get_start() const { 52 | return start_; 53 | } 54 | 55 | std::chrono::steady_clock::time_point get_end() const { 56 | return end_; 57 | } 58 | 59 | size_t duration_millis() const { 60 | return 61 | std::chrono::duration_cast( 62 | end_ - start_).count(); 63 | } 64 | 65 | const std::vector>& get_records() const { 66 | return records_; 67 | } 68 | 69 | const Aws::Kinesis::Model::PutRecordsOutcome& get_outcome() const { 70 | return outcome_; 71 | } 72 | 73 | Aws::Kinesis::Model::PutRecordsRequest to_sdk_request() const { 74 | Aws::Kinesis::Model::PutRecordsRequest req; 75 | for (auto& kr : records_) { 76 | auto serialized = kr->serialize(); 77 | Aws::Kinesis::Model::PutRecordsRequestEntry e; 78 | e.SetData(Aws::Utils::ByteBuffer((const unsigned char*) serialized.data(), 79 | serialized.size())); 80 | e.SetPartitionKey(kr->partition_key()); 81 | e.SetExplicitHashKey(kr->explicit_hash_key()); 82 | req.AddRecords(std::move(e)); 83 | } 84 | req.SetStreamName(stream_); 85 | if (!stream_arn_.empty()) req.SetStreamARN(stream_arn_); 86 | return req; 87 | } 88 | 89 | PutRecordsContext& set_start(std::chrono::steady_clock::time_point t) { 90 | start_ = t; 91 | return *this; 92 | } 93 | 94 | PutRecordsContext& set_end(std::chrono::steady_clock::time_point t) { 95 | end_ = t; 96 | return *this; 97 | } 98 | 99 | PutRecordsContext& set_outcome(Aws::Kinesis::Model::PutRecordsOutcome o) { 100 | outcome_ = std::move(o); 101 | return *this; 102 | } 103 | 104 | private: 105 | std::string stream_; 106 | std::string stream_arn_; 107 | std::chrono::steady_clock::time_point start_; 108 | std::chrono::steady_clock::time_point end_; 109 | std::vector> records_; 110 | Aws::Kinesis::Model::PutRecordsOutcome outcome_; 111 | }; 112 | 113 | } //namespace core 114 | } //namespace kinesis 115 | } //namespace aws 116 | 117 | #endif //AWS_KINESIS_CORE_PUT_RECORDS_CONTEXT_H_ 118 | -------------------------------------------------------------------------------- /aws/kinesis/core/ipc_manager.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include 19 | 20 | namespace aws { 21 | namespace kinesis { 22 | namespace core { 23 | 24 | namespace detail { 25 | 26 | void IpcReader::start() { 27 | if (channel_->open_read_channel()) { 28 | while (!shutdown_) { 29 | if (read(sizeof(len_t))) { 30 | len_t msg_len = 0; 31 | 32 | for (size_t i = 0; i < sizeof(len_t); i++) { 33 | int shift = (sizeof(len_t) - i - 1) * 8; 34 | len_t octet = (len_t) buffer_[i]; 35 | msg_len += octet << shift; 36 | } 37 | 38 | if (msg_len > kMaxMessageSize) { 39 | std::stringstream ss; 40 | ss << "Incoming message too large, was " << msg_len 41 | << " bytes, max allowed is " << kMaxMessageSize << " bytes"; 42 | throw std::runtime_error(ss.str().c_str()); 43 | } 44 | 45 | if (read(msg_len)) { 46 | queue_->put(std::string((const char*) buffer_.data(), msg_len)); 47 | } 48 | } 49 | } 50 | } 51 | } 52 | 53 | bool IpcReader::read(size_t len) { 54 | size_t read = 0; 55 | 56 | while (read < len) { 57 | auto num_read = channel_->read(buffer_.data() + read, len - read); 58 | 59 | if (num_read <= 0) { 60 | if (!shutdown_) { 61 | std::stringstream ss; 62 | if (num_read < 0) { 63 | ss << "IO error reading from ipc channel, errno = " << errno; 64 | } else if (num_read == 0) { 65 | ss << "EOF reached while reading from ipc channel"; 66 | } 67 | throw std::runtime_error(ss.str().c_str()); 68 | } else { 69 | return false; 70 | } 71 | } 72 | 73 | read += num_read; 74 | } 75 | return true; 76 | } 77 | 78 | void IpcWriter::start() { 79 | if (channel_->open_write_channel()) { 80 | std::string s; 81 | 82 | while (!shutdown_) { 83 | if (!queue_->try_take(s)) { 84 | aws::utils::sleep_for(std::chrono::milliseconds(5)); 85 | continue; 86 | } 87 | 88 | for (size_t i = 0; i < sizeof(len_t); i++) { 89 | auto shift = (sizeof(len_t) - i - 1) * 8; 90 | buffer_[i] = (uint8_t)((s.length() >> shift) & 0xFF); 91 | } 92 | std::memcpy(buffer_.data() + sizeof(len_t), s.data(), s.length()); 93 | 94 | write(sizeof(len_t) + s.length()); 95 | } 96 | } 97 | } 98 | 99 | void IpcWriter::write(size_t len) { 100 | size_t wrote = 0; 101 | 102 | while (wrote < len) { 103 | auto num_written = channel_->write(buffer_.data() + wrote, len - wrote); 104 | 105 | if (num_written < 0) { 106 | if (!shutdown_) { 107 | std::stringstream ss; 108 | ss << "IO error writing to ipc channel, errno = " << errno; 109 | throw std::runtime_error(ss.str().c_str()); 110 | } 111 | return; 112 | } 113 | 114 | wrote += num_written; 115 | } 116 | } 117 | 118 | } //namespace detail 119 | 120 | void IpcManager::put(std::string&& data) { 121 | if (data.length() > kMaxMessageSize) { 122 | std::stringstream ss; 123 | ss << "Outgoing message too large, was " << data.length() 124 | << " bytes, max allowed is " << kMaxMessageSize << " bytes"; 125 | throw std::runtime_error(ss.str().c_str()); 126 | } 127 | 128 | out_queue_->put(std::move(data)); 129 | } 130 | 131 | } //namespace core 132 | } //namespace kinesis 133 | } //namespace aws 134 | -------------------------------------------------------------------------------- /java/amazon-kinesis-producer/src/main/java/software/amazon/kinesis/producer/UserRecord.java: -------------------------------------------------------------------------------- 1 | package software.amazon.kinesis.producer; 2 | 3 | import com.amazonaws.services.schemaregistry.common.Schema; 4 | 5 | import java.nio.ByteBuffer; 6 | 7 | import lombok.ToString; 8 | 9 | @ToString 10 | public class UserRecord { 11 | /** 12 | * Stream to put to. 13 | */ 14 | private String streamName; 15 | 16 | /** 17 | * ARN of the stream, e.g., arn:aws:kinesis:us-east-2:123456789012:stream/mystream 18 | */ 19 | private String streamARN; 20 | 21 | /** 22 | * Partition key. Length must be at least one, and at most 256 (inclusive). 23 | */ 24 | private String partitionKey; 25 | 26 | /** 27 | * The hash value used to explicitly determine the shard the data 28 | * record is assigned to by overriding the partition key hash. 29 | * Must be a valid string representation of a positive integer 30 | * with value between 0 and 2^128 - 1 (inclusive). 31 | */ 32 | private String explicitHashKey; 33 | 34 | /** 35 | * Binary data of the record. Maximum size 1MiB. 36 | */ 37 | private ByteBuffer data; 38 | 39 | /** 40 | * Specify Schema for the data. Schemas are administered by AWS Glue Schema Registry. 41 | * Read Glue Schema Registry docs on how to get started on using Schema for your data. 42 | * This is an optional field. 43 | */ 44 | private Schema schema; 45 | 46 | public UserRecord() { 47 | } 48 | 49 | public UserRecord(String streamName, String partitionKey, ByteBuffer data) { 50 | this.streamName = streamName; 51 | this.partitionKey = partitionKey; 52 | this.data = data; 53 | } 54 | 55 | public UserRecord(String streamName, String partitionKey, String explicitHashKey, ByteBuffer data) { 56 | this.streamName = streamName; 57 | this.partitionKey = partitionKey; 58 | this.explicitHashKey = explicitHashKey; 59 | this.data = data; 60 | } 61 | 62 | public UserRecord(String streamName, String partitionKey, String explicitHashKey, ByteBuffer data, Schema schema) { 63 | this.streamName = streamName; 64 | this.partitionKey = partitionKey; 65 | this.explicitHashKey = explicitHashKey; 66 | this.data = data; 67 | this.schema = schema; 68 | } 69 | 70 | public String getStreamName() { 71 | return streamName; 72 | } 73 | 74 | public void setStreamName(String streamName) { 75 | this.streamName = streamName; 76 | } 77 | 78 | public UserRecord withStreamName(String streamName) { 79 | this.streamName = streamName; 80 | return this; 81 | } 82 | 83 | public String getPartitionKey() { 84 | return partitionKey; 85 | } 86 | 87 | public void setPartitionKey(String partitionKey) { 88 | this.partitionKey = partitionKey; 89 | } 90 | 91 | public UserRecord withPartitionKey(String partitionKey) { 92 | this.partitionKey = partitionKey; 93 | return this; 94 | } 95 | 96 | public ByteBuffer getData() { 97 | return data; 98 | } 99 | 100 | public void setData(ByteBuffer data) { 101 | this.data = data; 102 | } 103 | 104 | public UserRecord withData(ByteBuffer byteBuffer) { 105 | this.data = byteBuffer; 106 | return this; 107 | } 108 | 109 | public String getExplicitHashKey() { 110 | return explicitHashKey; 111 | } 112 | 113 | public void setExplicitHashKey(String explicitHashKey) { 114 | this.explicitHashKey = explicitHashKey; 115 | } 116 | 117 | public UserRecord withExplicitHashKey(String explicitHashKey) { 118 | this.explicitHashKey = explicitHashKey; 119 | return this; 120 | } 121 | 122 | public Schema getSchema() { 123 | return schema; 124 | } 125 | 126 | public void setSchema(Schema schema) { 127 | this.schema = schema; 128 | } 129 | 130 | public UserRecord withSchema(Schema schema) { 131 | this.schema = schema; 132 | return this; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /aws/kinesis/core/aggregator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_AGGREGATOR_H_ 17 | #define AWS_KINESIS_CORE_AGGREGATOR_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | namespace aws { 32 | namespace kinesis { 33 | namespace core { 34 | 35 | class Aggregator : boost::noncopyable { 36 | public: 37 | using DeadlineCallback = std::function)>; 38 | using ReducerMap = 39 | aws::utils::ConcurrentHashMap>; 41 | 42 | Aggregator( 43 | const std::shared_ptr& executor, 44 | const std::shared_ptr shard_map, 45 | const DeadlineCallback& deadline_callback, 46 | const std::shared_ptr& config, 47 | aws::utils::flush_statistics_aggregator& flush_stats, 48 | const std::shared_ptr& metrics_manager = 49 | std::make_shared()) 50 | : executor_(executor), 51 | shard_map_(shard_map), 52 | deadline_callback_(deadline_callback), 53 | config_(config), 54 | flush_stats_(flush_stats), 55 | metrics_manager_(metrics_manager), 56 | reducers_([this](auto) { return this->make_reducer(); }) {} 57 | 58 | std::shared_ptr put(const std::shared_ptr& ur) { 59 | // If shard map is not available, or aggregation is disabled, just send the 60 | // record by itself, and do not attempt to aggrgegate. 61 | boost::optional shard_id; 62 | if (config_->aggregation_enabled() && shard_map_) { 63 | shard_id = shard_map_->shard_id(ur->hash_key()); 64 | } 65 | if (!shard_id) { 66 | auto kr = std::make_shared(); 67 | // during retries, the records can have the predicted shard set from the last run. Clearing out the state here 68 | // because retrier expects these records to not have predicted shard so they don't get retried due to this. 69 | ur->reset_predicted_shard(); 70 | kr->add(ur); 71 | return kr; 72 | } else { 73 | ur->predicted_shard(*shard_id); 74 | return reducers_[*shard_id].add(ur); 75 | } 76 | } 77 | 78 | // TODO unit test for this 79 | void flush() { 80 | reducers_.foreach([](auto&, auto v) { v->flush(); }); 81 | } 82 | 83 | private: 84 | // This cannot be inlined in the lambda because msvc cannot compile that 85 | Reducer* make_reducer() { 86 | return new Reducer( 87 | executor_, 88 | deadline_callback_, 89 | config_->aggregation_max_size(), 90 | config_->aggregation_max_count(), 91 | flush_stats_ 92 | ); 93 | } 94 | 95 | std::shared_ptr executor_; 96 | std::shared_ptr shard_map_; 97 | DeadlineCallback deadline_callback_; 98 | std::shared_ptr config_; 99 | std::shared_ptr metrics_manager_; 100 | aws::utils::flush_statistics_aggregator& flush_stats_; 101 | ReducerMap reducers_; 102 | }; 103 | 104 | } //namespace core 105 | } //namespace kinesis 106 | } //namespace aws 107 | 108 | #endif //AWS_KINESIS_CORE_AGGREGATOR_H_ 109 | -------------------------------------------------------------------------------- /aws/kinesis/core/kinesis_producer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. 3 | * Licensed under the Apache License, Version 2.0 (the 4 | * "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef AWS_KINESIS_CORE_KINESIS_PRODUCER_H_ 17 | #define AWS_KINESIS_CORE_KINESIS_PRODUCER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace aws { 26 | namespace kinesis { 27 | namespace core { 28 | 29 | class KinesisProducer : boost::noncopyable { 30 | public: 31 | using Configuration = aws::kinesis::core::Configuration; 32 | 33 | KinesisProducer( 34 | std::shared_ptr ipc_manager, 35 | std::string region, 36 | std::shared_ptr& config, 37 | std::shared_ptr 38 | kinesis_creds_provider, 39 | std::shared_ptr 40 | cw_creds_provider, 41 | std::shared_ptr executor, 42 | std::string ca_path, 43 | std::string ca_file) 44 | : region_(std::move(region)), 45 | config_(std::move(config)), 46 | kinesis_creds_provider_(std::move(kinesis_creds_provider)), 47 | cw_creds_provider_(std::move(cw_creds_provider)), 48 | executor_(std::move(executor)), 49 | ipc_manager_(std::move(ipc_manager)), 50 | pipelines_([this](auto& stream) { 51 | return this->create_pipeline(stream); 52 | }), 53 | shutdown_(false) { 54 | create_kinesis_client(ca_path, ca_file); 55 | create_cw_client(ca_path, ca_file); 56 | create_metrics_manager(); 57 | report_outstanding(); 58 | message_drainer_ = aws::thread([this] { this->drain_messages(); }); 59 | } 60 | 61 | ~KinesisProducer() { 62 | shutdown_ = true; 63 | message_drainer_.join(); 64 | } 65 | 66 | void join() { 67 | executor_->join(); 68 | } 69 | 70 | private: 71 | static const std::chrono::microseconds kMessageDrainMinBackoff; 72 | static const std::chrono::microseconds kMessageDrainMaxBackoff; 73 | static constexpr const size_t kMessageMaxBatchSize = 16; 74 | 75 | void create_metrics_manager(); 76 | 77 | void create_kinesis_client(const std::string& ca_path, const std::string& ca_file); 78 | 79 | void create_cw_client(const std::string& ca_path, const std::string& ca_file); 80 | 81 | Pipeline* create_pipeline(const std::string& stream); 82 | 83 | void drain_messages(); 84 | 85 | void on_ipc_message(std::string&& message) noexcept; 86 | 87 | void on_put_record(aws::kinesis::protobuf::Message& m); 88 | 89 | void on_flush(const aws::kinesis::protobuf::Flush& flush_msg); 90 | 91 | void on_metrics_request(const aws::kinesis::protobuf::Message& m); 92 | 93 | void on_set_credentials( 94 | const aws::kinesis::protobuf::SetCredentials& set_creds); 95 | 96 | void report_outstanding(); 97 | 98 | std::string region_; 99 | 100 | std::shared_ptr config_; 101 | std::shared_ptr 102 | kinesis_creds_provider_; 103 | std::shared_ptr 104 | cw_creds_provider_; 105 | std::shared_ptr kinesis_client_; 106 | std::shared_ptr cw_client_; 107 | std::shared_ptr executor_; 108 | 109 | std::shared_ptr ipc_manager_; 110 | std::shared_ptr metrics_manager_; 111 | 112 | aws::utils::ConcurrentHashMap pipelines_; 113 | bool shutdown_; 114 | aws::thread message_drainer_; 115 | 116 | std::shared_ptr report_outstanding_; 117 | }; 118 | 119 | } //namespace core 120 | } //namespace kinesis 121 | } //namespace aws 122 | 123 | #endif //AWS_KINESIS_CORE_KINESIS_PRODUCER_H_ 124 | --------------------------------------------------------------------------------