├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── .github └── workflows │ └── maven.yml ├── src ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── org.apache.kafka.common.config.provider.ConfigProvider │ └── java │ │ └── com │ │ └── amazonaws │ │ └── kafka │ │ └── config │ │ └── providers │ │ ├── S3ImportConfig.java │ │ ├── common │ │ ├── CommonConfigUtils.java │ │ └── AwsServiceConfigProvider.java │ │ ├── SsmParamStoreConfig.java │ │ ├── SecretsManagerConfig.java │ │ ├── SsmParamStoreConfigProvider.java │ │ ├── S3ImportConfigProvider.java │ │ └── SecretsManagerConfigProvider.java └── test │ └── java │ └── com │ └── amazonaws │ └── kafka │ └── config │ └── providers │ ├── MockedSsmParamStoreConfigProvider.java │ ├── MockedSecretsManagerConfigProvider.java │ ├── SsmParamStoreConfigProviderTest.java │ └── SecretsManagerConfigProviderTest.java ├── CONTRIBUTING.md ├── README.md └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | .classpath 4 | .DS_Store 5 | .project 6 | .idea 7 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | software and associated documentation files (the "Software"), to deal in the Software 7 | without restriction, including without limitation the rights to use, copy, modify, 8 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 9 | permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 12 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 13 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 14 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 15 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 16 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Java CI with Maven 10 | 11 | on: 12 | push: 13 | branches: [ "trunk" ] 14 | pull_request: 15 | branches: [ "trunk" ] 16 | 17 | jobs: 18 | build: 19 | 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Set up JDK 11 25 | uses: actions/setup-java@v3 26 | with: 27 | java-version: '11' 28 | distribution: 'temurin' 29 | cache: maven 30 | - name: Build with Maven 31 | run: mvn -B package --file pom.xml 32 | 33 | # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive 34 | - name: Update dependency graph 35 | uses: advanced-security/maven-dependency-submission-action@571e99aab1055c2e71a1e2309b9691de18d6b7d6 36 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.kafka.common.config.provider.ConfigProvider: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | # software and associated documentation files (the "Software"), to deal in the Software 6 | # without restriction, including without limitation the rights to use, copy, modify, 7 | # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | # 17 | 18 | com.amazonaws.kafka.config.providers.S3ImportConfigProvider 19 | com.amazonaws.kafka.config.providers.SecretsManagerConfigProvider 20 | com.amazonaws.kafka.config.providers.SsmParamStoreConfigProvider 21 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/S3ImportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.common.config.AbstractConfig; 22 | import org.apache.kafka.common.config.ConfigDef; 23 | 24 | import com.amazonaws.kafka.config.providers.common.CommonConfigUtils; 25 | 26 | public class S3ImportConfig extends AbstractConfig{ 27 | 28 | public static final String LOCAL_DIR = "local_dir"; 29 | private static final String LOCAL_DIR_DOC = 30 | "Local directory to store imported from S3 files. " + 31 | "If not provided, temporary directory defined in OS will be used.)"; 32 | 33 | public S3ImportConfig(Map originals) { 34 | super(config(), originals); 35 | } 36 | 37 | private static ConfigDef config() { 38 | return new ConfigDef(CommonConfigUtils.COMMON_CONFIG) 39 | .define( 40 | LOCAL_DIR, 41 | ConfigDef.Type.STRING, 42 | null, 43 | ConfigDef.Importance.HIGH, 44 | LOCAL_DIR_DOC 45 | ) 46 | ; 47 | } 48 | } -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/kafka/config/providers/MockedSsmParamStoreConfigProvider.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.kafka.config.providers; 2 | 3 | import org.mockito.stubbing.Answer; 4 | import software.amazon.awssdk.services.ssm.SsmClient; 5 | import software.amazon.awssdk.services.ssm.model.GetParameterRequest; 6 | import software.amazon.awssdk.services.ssm.model.GetParameterResponse; 7 | import software.amazon.awssdk.services.ssm.model.Parameter; 8 | import software.amazon.awssdk.services.ssm.model.ParameterNotFoundException; 9 | 10 | import static org.mockito.Mockito.*; 11 | 12 | public class MockedSsmParamStoreConfigProvider extends SsmParamStoreConfigProvider { 13 | 14 | @Override 15 | protected SsmClient checkOrInitSsmClient() { 16 | SsmClient ssmClient = mock(SsmClient.class); 17 | when(ssmClient.getParameter(request("/test/stringParam"))).thenAnswer( 18 | (Answer) invocation -> response("stringParam", "test string value") 19 | ); 20 | when(ssmClient.getParameter(request("/test/stringParamWithTTL"))).thenAnswer( 21 | (Answer) invocation -> response("stringParamWithTTL", "test string value") 22 | ); 23 | when(ssmClient.getParameter(request("/test/intParam"))).thenAnswer( 24 | (Answer) invocation -> response("intParam", "777") 25 | ); 26 | when(ssmClient.getParameter(request("/test/listParam"))).thenAnswer( 27 | (Answer) invocation -> response("listParam", "el1,el2,el3") 28 | ); 29 | when(ssmClient.getParameter(request("/test/secretParam"))).thenAnswer( 30 | (Answer) invocation -> response("secretParam", "secret password") 31 | ); 32 | when(ssmClient.getParameter(request("/test/notFound"))).thenThrow(ParameterNotFoundException.class); 33 | 34 | return ssmClient; 35 | } 36 | 37 | private GetParameterRequest request(String param) { 38 | return GetParameterRequest.builder().name(param).withDecryption(true).build(); 39 | } 40 | 41 | private GetParameterResponse response(String param, String value) { 42 | return GetParameterResponse.builder().parameter(Parameter.builder().name(param).value(value).build()).build(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/kafka/config/providers/MockedSecretsManagerConfigProvider.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.kafka.config.providers; 2 | 3 | import org.mockito.stubbing.Answer; 4 | import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; 5 | import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; 6 | import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; 7 | import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException; 8 | 9 | 10 | import static org.mockito.Mockito.mock; 11 | import static org.mockito.Mockito.when; 12 | 13 | public class MockedSecretsManagerConfigProvider extends SecretsManagerConfigProvider { 14 | @Override 15 | protected SecretsManagerClient checkOrInitSecretManagerClient() { 16 | SecretsManagerClient secretsClient = mock(SecretsManagerClient.class); 17 | when(secretsClient.getSecretValue(request("AmazonMSK_TestKafkaConfig"))).thenAnswer( 18 | (Answer) invocation -> response("{\"username\": \"John\", \"password\":\"Password123\"}") 19 | ); 20 | when(secretsClient.getSecretValue(request("arn:aws:secretsmanager:ap-southeast-2:123456789:secret:AmazonMSK_my_service/my_secret"))).thenAnswer( 21 | (Answer) invocation -> response("{\"username\": \"John2\", \"password\":\"Password567\"}") 22 | ); 23 | when(secretsClient.getSecretValue(request("arn:aws:secretsmanager:ap-southeast-2:123456789:secret:AmazonMSK_my_service/my_secret%3A"))).thenAnswer( 24 | (Answer) invocation -> response("{\"username\": \"John3\", \"password\":\"Password321\"}") 25 | ); 26 | when(secretsClient.getSecretValue(request("AmazonMSK_TestTTL"))).thenAnswer( 27 | (Answer) invocation -> response("{\"username\": \"John\", \"password\":\"Password123\"}") 28 | ); 29 | when(secretsClient.getSecretValue(request("notFound"))).thenThrow(ResourceNotFoundException.class); 30 | return secretsClient; 31 | } 32 | 33 | private GetSecretValueRequest request(String path) { 34 | return GetSecretValueRequest.builder().secretId(path).build(); 35 | } 36 | 37 | private GetSecretValueResponse response(String value) { 38 | return GetSecretValueResponse.builder().secretString(value).build(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/common/CommonConfigUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers.common; 18 | 19 | 20 | import org.apache.kafka.common.config.ConfigDef; 21 | 22 | /** 23 | * This utility class provides common functionality that can be reused for multiple Config Providers. 24 | * 25 | */ 26 | public class CommonConfigUtils { 27 | 28 | /** Common configuration parameters to be reused in multiple Config Providers */ 29 | public static final ConfigDef COMMON_CONFIG = getConfig(); 30 | 31 | 32 | public static final String REGION = "region"; 33 | private static final String REGION_DOC = "Specify region if needed. Default region is the same where connector is running"; 34 | 35 | public static final String ENDPOINT = "endpoint"; 36 | private static final String ENDPOINT_DOC = "(Optional) Specify an endpoint. Default endpoint will be used. " 37 | + "If there is an endpoint for a service in a VPC, and it should be used instead of default one, " 38 | + "this is the way to explicitly provide one."; 39 | 40 | 41 | /** 42 | * 43 | * @return common configuration 44 | */ 45 | static ConfigDef getConfig() { 46 | return new ConfigDef() 47 | .define( 48 | REGION, 49 | ConfigDef.Type.STRING, 50 | "", 51 | ConfigDef.Importance.MEDIUM, 52 | REGION_DOC 53 | ) 54 | .define( 55 | ENDPOINT, 56 | ConfigDef.Type.STRING, 57 | "", 58 | ConfigDef.Importance.MEDIUM, 59 | ENDPOINT_DOC 60 | ) 61 | ; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/SsmParamStoreConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.common.config.AbstractConfig; 22 | import org.apache.kafka.common.config.ConfigDef; 23 | import org.apache.kafka.common.config.ConfigDef.ValidString; 24 | 25 | import com.amazonaws.kafka.config.providers.common.CommonConfigUtils; 26 | 27 | public class SsmParamStoreConfig extends AbstractConfig{ 28 | 29 | public static final String NOT_FOUND_STRATEGY = "NotFoundStrategy"; 30 | public static final String NOT_FOUND_FAIL = "fail"; 31 | public static final String NOT_FOUND_IGNORE = "ignore"; 32 | 33 | private static final String NOT_FOUND_STRATEGY_DOC = 34 | "An action to take in case a secret cannot be found. " 35 | + "Possible actions are: `ignore` and `fail`.
" 36 | + "If `ignore` is selected and a secret cannot be found, the empty string will be assigned to a parameter.
" 37 | + "If `fail` is selected, the config provider will throw an exception to signal the issue.
" 38 | + "If there is a connectivity or access issue with AWS Secrets Manager service, an exception will be thrown."; 39 | 40 | public SsmParamStoreConfig(Map originals) { 41 | super(config(), originals); 42 | } 43 | 44 | private static ConfigDef config() { 45 | return new ConfigDef(CommonConfigUtils.COMMON_CONFIG) 46 | .define( 47 | NOT_FOUND_STRATEGY, 48 | ConfigDef.Type.STRING, 49 | NOT_FOUND_IGNORE, 50 | ValidString.in(NOT_FOUND_FAIL, NOT_FOUND_IGNORE), 51 | ConfigDef.Importance.LOW, 52 | NOT_FOUND_STRATEGY_DOC 53 | ) 54 | ; 55 | } 56 | } -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/SecretsManagerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.common.config.AbstractConfig; 22 | import org.apache.kafka.common.config.ConfigDef; 23 | import org.apache.kafka.common.config.ConfigDef.ValidString; 24 | 25 | import com.amazonaws.kafka.config.providers.common.CommonConfigUtils; 26 | 27 | public class SecretsManagerConfig extends AbstractConfig{ 28 | 29 | public static final String NOT_FOUND_STRATEGY = "NotFoundStrategy"; 30 | public static final String NOT_FOUND_FAIL = "fail"; 31 | public static final String NOT_FOUND_IGNORE = "ignore"; 32 | 33 | private static final String NOT_FOUND_STRATEGY_DOC = 34 | "An action to take in case a secret cannot be found. " 35 | + "Possible actions are: `ignore` and `fail`.
" 36 | + "If `ignore` is selected and a secret cannot be found, the empty string will be assigned to a parameter.
" 37 | + "If `fail` is selected, the config provider will throw an exception to signal the issue.
" 38 | + "If there is a connectivity or access issue with AWS Secrets Manager service, an exception will be thrown."; 39 | 40 | public SecretsManagerConfig(Map originals) { 41 | super(config(), originals); 42 | } 43 | 44 | private static ConfigDef config() { 45 | return new ConfigDef(CommonConfigUtils.COMMON_CONFIG) 46 | .define( 47 | NOT_FOUND_STRATEGY, 48 | ConfigDef.Type.STRING, 49 | NOT_FOUND_IGNORE, 50 | ValidString.in(NOT_FOUND_FAIL, NOT_FOUND_IGNORE), 51 | ConfigDef.Importance.LOW, 52 | NOT_FOUND_STRATEGY_DOC 53 | ) 54 | ; 55 | } 56 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/kafka/config/providers/SsmParamStoreConfigProviderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | 20 | import static org.junit.jupiter.api.Assertions.assertEquals; 21 | import static org.junit.jupiter.api.Assertions.assertThrows; 22 | 23 | import java.util.Arrays; 24 | import java.util.Collections; 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | import org.apache.kafka.common.config.AbstractConfig; 29 | import org.apache.kafka.common.config.ConfigDef; 30 | import org.apache.kafka.common.config.ConfigDef.Importance; 31 | import org.apache.kafka.common.config.ConfigDef.Type; 32 | import org.junit.jupiter.api.BeforeEach; 33 | import org.junit.jupiter.api.Test; 34 | import software.amazon.awssdk.services.ssm.model.ParameterNotFoundException; 35 | 36 | public class SsmParamStoreConfigProviderTest { 37 | 38 | static Map props; 39 | 40 | @BeforeEach 41 | public void setup() { 42 | props = new HashMap<>(); 43 | props.put("config.providers", "ssm"); 44 | props.put("config.providers.ssm.class", "com.amazonaws.kafka.config.providers.MockedSsmParamStoreConfigProvider"); 45 | props.put("config.providers.ssm.param.region", "us-west-2"); 46 | props.put("config.providers.ssm.param.NotFoundStrategy", "fail"); 47 | } 48 | 49 | @Test 50 | public void testExistingKeys() { 51 | // props.put("client.id", "${bootstrap.servers}"); // this doesn't work 52 | props.put("stringKey", "${ssm::/test/stringParam}"); 53 | props.put("stringKeyWithTTL", "${ssm::/test/stringParamWithTTL?ttl=60000}"); 54 | props.put("intKey", "${ssm::/test/intParam}"); 55 | props.put("listKey", "${ssm::/test/listParam}"); 56 | props.put("secretKey", "${ssm::/test/secretParam}"); 57 | 58 | CustomConfig testConfig = new CustomConfig(props); 59 | 60 | 61 | assertEquals("test string value", testConfig.getString("stringKey")); 62 | assertEquals("test string value", testConfig.getString("stringKeyWithTTL")); 63 | assertEquals(777L, testConfig.getLong("intKey").longValue()); 64 | assertEquals(Arrays.asList("el1,el2,el3".split(",")), testConfig.getList("listKey")); 65 | assertEquals("secret password", testConfig.getString("secretKey")); 66 | } 67 | 68 | @Test 69 | public void testNonExistingKeys() { 70 | props.put("notFound", "${ssm::/test/notFound}"); 71 | assertThrows(ParameterNotFoundException.class, () -> new CustomConfig(props)); 72 | } 73 | 74 | 75 | static class CustomConfig extends AbstractConfig { 76 | final static String DEFAULT_DOC = "Default Doc"; 77 | final static ConfigDef CONFIG = new ConfigDef() 78 | .define("stringKey", Type.STRING, "defaultValue", Importance.HIGH, DEFAULT_DOC) 79 | .define("stringKeyWithTTL", Type.STRING, "defaultValue", Importance.HIGH, DEFAULT_DOC) 80 | .define("intKey", Type.LONG, 0, Importance.MEDIUM, DEFAULT_DOC) 81 | .define("listKey", Type.LIST, Collections.emptyList(), Importance.LOW, DEFAULT_DOC) 82 | .define("secretKey", Type.STRING, "", Importance.LOW, DEFAULT_DOC) 83 | .define("notFound", Type.STRING, "", Importance.LOW, DEFAULT_DOC) 84 | ; 85 | public CustomConfig(Map originals) { 86 | super(CONFIG, originals); 87 | } 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apache Kafka Config Providers 2 | 3 | This repository provides samples of Apache Kafka Config Providers that can be used to integrate Kafka client properties with other systems. Following Kafka clients can use config providers: 4 | 5 | - Kafka admin client 6 | - Kafka consumer 7 | - Kafka producer 8 | - Kafka Connect (both workers and connectors) 9 | - Kafka Streams 10 | 11 | 12 | ## How to use 13 | 14 | Config providers, their configuration and usage are defined as properties of the Kafka client: 15 | 16 | ``` 17 | # define names of config providers: 18 | config.providers=secretsmanager,ssm,s3import 19 | 20 | # provide implementation classes for each provider: 21 | config.providers.secretsmanager.class = com.amazonaws.kafka.config.providers.SecretsManagerConfigProvider 22 | config.providers.ssm.class = com.amazonaws.kafka.config.providers.SsmParamStoreConfigProvider 23 | config.providers.s3import.class = com.amazonaws.kafka.config.providers.S3ImportConfigProvider 24 | 25 | # configure a config provider (if it needs additional initialization), for example you can provide a region where the secrets or S3 buckets are located: 26 | config.providers.secretsmanager.param.region = us-west-2 27 | config.providers.s3import.param.region = us-west-2 28 | 29 | # below is an example of config provider usage to supply a truststore location and its password. 30 | # Actual parameter names depend on how those config providers are used in the client's configuration. 31 | database.ssl.truststore.password = ${secretsmanager:mySslCertCredentials:ssl_trust_pass} 32 | database.ssl.truststore.location = ${s3import:us-west-2:my_cert_bucket/pass/to/trustore_unique_filename.jks} 33 | 34 | # Example of accessing a Secret via an ARN which has been URL Encoded 35 | database.password = ${secretsmanager:arn%3Aaws%3Asecretsmanager%3Aus-west-2%3A123477892456%3Asecret%3A%2Fdb%2Fadmin_user_credentials-ieAE11:password} 36 | ``` 37 | 38 | > NOTE: For the `secretsmanager` provider, the base parser from Kafka does not support using ARNs due to the `:` character. 39 | > In this case, URL encode the ARN first, and it will be decoded correctly by this Provider. 40 | 41 | More information about configuration of the config providers and usage, see below per config provider. 42 | 43 | For a detailed documentation on config providers in general, please follow the Apache Kafka documentation. 44 | 45 | ### Kafka Connect Users: 46 | 47 | ** Configuration in distributed mode ** 48 | 49 | Please note, that Kafka Connect uses two levels of configurations: workers and connectors. 50 | 51 | To avoid issues with validation of the connector configuration, users need to define config providers in workers properties, and then use the tokens in the connector properties. 52 | 53 | ** TTL Support ** 54 | 55 | Currently Secrets and System Manager Config Providers support time-to-live (TTL) configuration in milliseconds. 56 | 57 | ``` 58 | # Reload configuration and restart connector every 5 minutes: 59 | database.ssl.truststore.password = ${secretsmanager:mySslCertCredentials:ssl_trust_pass?ttl=300000} 60 | ``` 61 | 62 | Note, 63 | - TTL value is in milliseconds 64 | - upon an expiration, the entire connector will be restarted, regardless whether a value has been changed or not. 65 | 66 | 67 | ## Build 68 | 69 | 70 | ``` 71 | git clone 72 | cd msk-config-providers 73 | mvn clean package 74 | ``` 75 | 76 | Expected target artifacts: 77 | 78 | ``` 79 | > tree target/ 80 | target 81 | ├── lib 82 | │   ├── 83 | │   └── ................. 84 | └─── msk-config-providers-.jar 85 | ``` 86 | 87 | Additionally, there is a flat UBER jar at the following location: 88 | 89 | ``` 90 | target 91 | ├── shade-uber 92 | │   └── msk-config-providers--uber.jar 93 | └── ....... 94 | ``` 95 | 96 | 97 | ## Access Management 98 | 99 | Currently config providers do not support credentials to be explicitly provided. Config provider will inherit any type of credentials of hosting application, OS or service. 100 | 101 | Access Policy/Role associated with the application that is running a config provider should have sufficient but least privileged permissions to access the services that are configured/referenced in the configuration. E.g., Supplying secrets through AWS Secrets Manager provider will need read permissions for the client to read that particular secret from AWS Secrets Manager service. 102 | 103 | ## Road Map 104 | 105 | - add support for more authentication options 106 | - support TTL for S3Import Config Provider 107 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/common/AwsServiceConfigProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers.common; 18 | 19 | import java.io.IOException; 20 | import java.net.URI; 21 | import java.net.URISyntaxException; 22 | import java.util.LinkedHashMap; 23 | import java.util.Map; 24 | import java.util.Set; 25 | import java.util.regex.Matcher; 26 | import java.util.regex.Pattern; 27 | 28 | import org.apache.kafka.common.config.AbstractConfig; 29 | import org.apache.kafka.common.config.ConfigChangeCallback; 30 | import org.apache.kafka.common.config.provider.ConfigProvider; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder; 35 | import software.amazon.awssdk.regions.Region; 36 | 37 | public abstract class AwsServiceConfigProvider implements ConfigProvider { 38 | 39 | private final Logger log = LoggerFactory.getLogger(getClass()); 40 | 41 | private String region; 42 | private String endpoint; 43 | 44 | 45 | public String getRegion() { 46 | return region; 47 | } 48 | 49 | public void setRegion(String region) { 50 | this.region = region; 51 | } 52 | 53 | public String getEndpoint() { 54 | return endpoint; 55 | } 56 | 57 | public void setEndpoint(String endpoint) { 58 | this.endpoint = endpoint; 59 | } 60 | 61 | /** 62 | * Set common AWS configuration parameters, like region, endpoint, etc... 63 | * @param config 64 | */ 65 | public void setCommonConfig(AbstractConfig config) { 66 | 67 | // default region from configuration. It can be null, empty or blank. 68 | this.region = config.getString(CommonConfigUtils.REGION); 69 | this.endpoint = config.getString(CommonConfigUtils.ENDPOINT); 70 | } 71 | 72 | @Override 73 | public void subscribe(String path, Set keys, ConfigChangeCallback callback) { 74 | log.info("Subscription is not implemented and will be ignored"); 75 | } 76 | 77 | 78 | public void close() throws IOException { 79 | } 80 | 81 | /** 82 | * Set configuration that is common to most AWS Clients. 83 | * @param 84 | * @param cBuilder 85 | * @return 86 | */ 87 | protected > T setClientCommonConfig(T cBuilder) { 88 | 89 | if (this.region != null && !this.region.isBlank()) { 90 | cBuilder.region(Region.of(this.region)); 91 | } 92 | 93 | if (this.endpoint != null && !endpoint.isBlank()) 94 | try { 95 | cBuilder.endpointOverride(new URI(this.endpoint)); 96 | } catch (URISyntaxException e) { 97 | log.error("Invalid syntax, ", e); 98 | throw new RuntimeException(e); 99 | } 100 | 101 | return cBuilder; 102 | } 103 | 104 | 105 | protected String parseKey(String keyWithOptions) { 106 | if (keyWithOptions == null) return keyWithOptions; 107 | 108 | return keyWithOptions.split("\\?", 2)[0]; 109 | } 110 | 111 | protected Map parseKeyOptions(String keyWithOptions) { 112 | 113 | Map options = new LinkedHashMap<>(); 114 | 115 | if (keyWithOptions == null) return options; 116 | 117 | String[] parsed = keyWithOptions.split("\\?", 2); 118 | if (parsed.length < 2) return options; 119 | 120 | Matcher m = Pattern.compile("(\\w+)=(.*?)(?=,\\w+=|$)").matcher(parsed[1]); 121 | while (m.find()) { 122 | options.put(m.group(1), m.group(2)); 123 | } 124 | 125 | return options; 126 | } 127 | 128 | protected Long getUpdatedTtl(Long currentTtl, Map options) { 129 | if (options == null || options.isEmpty()) return currentTtl; 130 | 131 | String newTtlStr = options.get("ttl"); 132 | if (newTtlStr == null) return currentTtl; 133 | 134 | try { 135 | Long newTtl = Long.valueOf(newTtlStr); 136 | return currentTtl == null || currentTtl > newTtl 137 | ? newTtl 138 | : currentTtl; 139 | }catch(Exception e) { 140 | log.warn("TTL value '{}' is not a number", newTtlStr, e); 141 | } 142 | return currentTtl; 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | software.amazon.msk 4 | msk-config-providers 5 | 0.4.0-SNAPSHOT 6 | msk-config-providers 7 | custom config plugins for Kafka Connect 8 | 9 | 10 | UTF-8 11 | 11 12 | ${maven.compiler.source} 13 | 5.9.1 14 | 1.7.32 15 | 2.29.11 16 | 17 | 18 | 19 | 20 | org.apache.kafka 21 | kafka-clients 22 | 3.7.2 23 | provided 24 | 25 | 26 | software.amazon.awssdk 27 | ssm 28 | ${software.amazon.awssdk.version} 29 | 30 | 31 | software.amazon.awssdk 32 | secretsmanager 33 | ${software.amazon.awssdk.version} 34 | 35 | 36 | software.amazon.awssdk 37 | s3 38 | ${software.amazon.awssdk.version} 39 | 40 | 41 | com.fasterxml.jackson.core 42 | jackson-databind 43 | 2.13.4.2 44 | provided 45 | 46 | 47 | org.junit.jupiter 48 | junit-jupiter-engine 49 | ${junit.jupiter.version} 50 | test 51 | 52 | 53 | org.mockito 54 | mockito-core 55 | 2.21.0 56 | test 57 | 58 | 59 | org.slf4j 60 | slf4j-nop 61 | 1.7.24 62 | test 63 | 64 | 65 | 66 | 67 | 68 | 69 | org.apache.maven.plugins 70 | maven-surefire-plugin 71 | 2.22.2 72 | 73 | 74 | org.apache.maven.plugins 75 | maven-dependency-plugin 76 | 3.2.0 77 | 78 | 79 | copy-dependencies 80 | prepare-package 81 | 82 | copy-dependencies 83 | 84 | 85 | ${project.build.directory}/lib 86 | false 87 | false 88 | true 89 | runtime 90 | 91 | 92 | 93 | 94 | 95 | org.apache.maven.plugins 96 | maven-jar-plugin 97 | 3.0.0 98 | 99 | 100 | 101 | true 102 | 103 | 104 | lib/ 105 | 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-shade-plugin 112 | 3.2.4 113 | 114 | false 115 | 116 | 117 | *:* 118 | 119 | module-info.class 120 | 121 | 122 | 123 | 124 | 125 | 126 | package 127 | 128 | shade 129 | 130 | 131 | ${project.build.directory}/shade-uber 132 | ${project.artifactId}-${project.version}-all 133 | 134 | 135 | org.apache.kafka:* 136 | slf4j-api:* 137 | com.fasterxml.jackson.annotation:* 138 | com.fasterxml.jackson.core:* 139 | com.fasterxml.jackson.dataformat:* 140 | com.fasterxml.jackson.dataformat:* 141 | jackson*:jackson-databind:jar: 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | org.apache.maven.plugins 150 | maven-compiler-plugin 151 | 3.10.0 152 | 153 | 11 154 | 155 | 156 | 157 | 158 | 159 | 160 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/kafka/config/providers/SecretsManagerConfigProviderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | 20 | import static org.junit.jupiter.api.Assertions.assertEquals; 21 | import static org.junit.jupiter.api.Assertions.assertThrows; 22 | 23 | import java.net.URLEncoder; 24 | import java.nio.charset.StandardCharsets; 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | import org.apache.commons.codec.Charsets; 29 | import org.apache.kafka.common.config.AbstractConfig; 30 | import org.apache.kafka.common.config.ConfigDef; 31 | import org.apache.kafka.common.config.ConfigDef.Importance; 32 | import org.apache.kafka.common.config.ConfigDef.Type; 33 | import org.apache.kafka.common.config.ConfigException; 34 | import org.junit.jupiter.api.BeforeEach; 35 | import org.junit.jupiter.api.Test; 36 | import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException; 37 | 38 | public class SecretsManagerConfigProviderTest { 39 | 40 | Map props; 41 | @BeforeEach 42 | public void setup() { 43 | props = new HashMap<>(); 44 | props.put("config.providers", "secretsmanager"); 45 | props.put("config.providers.secretsmanager.class", "com.amazonaws.kafka.config.providers.MockedSecretsManagerConfigProvider"); 46 | props.put("config.providers.secretsmanager.param.region", "us-west-2"); 47 | props.put("config.providers.secretsmanager.param.NotFoundStrategy", "fail"); 48 | } 49 | 50 | @Test 51 | public void testExistingKeys() { 52 | props.put("username", "${secretsmanager:AmazonMSK_TestKafkaConfig:username}"); 53 | props.put("password", "${secretsmanager:AmazonMSK_TestKafkaConfig:password}"); 54 | 55 | CustomConfig testConfig = new CustomConfig(props); 56 | 57 | assertEquals("John", testConfig.getString("username")); 58 | assertEquals("Password123", testConfig.getString("password")); 59 | } 60 | 61 | @Test 62 | public void testExistingKeysViaArn() { 63 | String arn = URLEncoder.encode("arn:aws:secretsmanager:ap-southeast-2:123456789:secret:AmazonMSK_my_service/my_secret", StandardCharsets.UTF_8); 64 | props.put("username", "${secretsmanager:" + arn + ":username}"); 65 | props.put("password", "${secretsmanager:" + arn + ":password}"); 66 | 67 | CustomConfig testConfig = new CustomConfig(props); 68 | 69 | assertEquals("John2", testConfig.getString("username")); 70 | assertEquals("Password567", testConfig.getString("password")); 71 | } 72 | 73 | @Test 74 | public void testExistingKeysViaArnWithEncodedValue() { 75 | String arn = URLEncoder.encode("arn:aws:secretsmanager:ap-southeast-2:123456789:secret:AmazonMSK_my_service/my_secret%3A", StandardCharsets.UTF_8); 76 | props.put("username", "${secretsmanager:" + arn + ":username}"); 77 | props.put("password", "${secretsmanager:" + arn + ":password}"); 78 | 79 | CustomConfig testConfig = new CustomConfig(props); 80 | 81 | assertEquals("John3", testConfig.getString("username")); 82 | assertEquals("Password321", testConfig.getString("password")); 83 | } 84 | 85 | @Test 86 | public void testExistingKeysViaHandEncodedArn() { 87 | String arn = "arn%3Aaws%3Asecretsmanager%3Aap-southeast-2%3A123456789%3Asecret%3AAmazonMSK_my_service%2Fmy_secret"; 88 | props.put("username", "${secretsmanager:" + arn + ":username}"); 89 | props.put("password", "${secretsmanager:" + arn + ":password}"); 90 | 91 | CustomConfig testConfig = new CustomConfig(props); 92 | 93 | assertEquals("John2", testConfig.getString("username")); 94 | assertEquals("Password567", testConfig.getString("password")); 95 | } 96 | 97 | @Test 98 | public void testTtl() { 99 | props.put("username", "${secretsmanager:AmazonMSK_TestKafkaConfig:username?ttl=60000}"); 100 | props.put("password", "${secretsmanager:AmazonMSK_TestKafkaConfig:password}"); 101 | 102 | CustomConfig testConfig = new CustomConfig(props); 103 | 104 | assertEquals("John", testConfig.getString("username")); 105 | assertEquals("Password123", testConfig.getString("password")); 106 | } 107 | 108 | @Test 109 | public void testNonExistingSecret() { 110 | props.put("notFound", "${secretsmanager:notFound:noKey}"); 111 | assertThrows(ResourceNotFoundException.class, () ->new CustomConfig(props)); 112 | } 113 | 114 | @Test 115 | public void testNonExistingKey() { 116 | props.put("notFound", "${secretsmanager:AmazonMSK_TestKafkaConfig:noKey}"); 117 | assertThrows(ConfigException.class, () ->new CustomConfig(props)); 118 | } 119 | 120 | static class CustomConfig extends AbstractConfig { 121 | final static String DEFAULT_DOC = "Default Doc"; 122 | final static ConfigDef CONFIG = new ConfigDef() 123 | .define("username", Type.STRING, "defaultValue", Importance.HIGH, DEFAULT_DOC) 124 | .define("password", Type.STRING, "defaultValue", Importance.HIGH, DEFAULT_DOC) 125 | ; 126 | public CustomConfig(Map originals) { 127 | super(CONFIG, originals); 128 | } 129 | } 130 | 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/SsmParamStoreConfigProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | import java.io.IOException; 20 | import java.util.Collections; 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | import java.util.Set; 24 | 25 | import org.apache.kafka.common.config.ConfigData; 26 | import org.apache.kafka.common.config.ConfigException; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import com.amazonaws.kafka.config.providers.common.AwsServiceConfigProvider; 31 | 32 | import software.amazon.awssdk.services.ssm.SsmClient; 33 | import software.amazon.awssdk.services.ssm.SsmClientBuilder; 34 | import software.amazon.awssdk.services.ssm.model.GetParameterRequest; 35 | import software.amazon.awssdk.services.ssm.model.GetParameterResponse; 36 | import software.amazon.awssdk.services.ssm.model.ParameterNotFoundException; 37 | 38 | /** 39 | * This class implements a ConfigProvider for AWS System Manager / Parameter Store.
40 | * 41 | *

Usage:
42 | * In a configuration file (e.g. {@code client.properties}) define following properties:
43 | * 44 | * 

 45 |  * #        Step1. Configure the System Manager Param Store as config provider:
 46 |  * config.providers=ssm
 47 |  * config.providers.ssm.class=com.amazonaws.kafka.config.providers.SsmParamStoreConfigProvider
 48 |  * # optional parameter for region:
 49 |  * config.providers.ssm.param.region=us-west-2
 50 |  * # optional parameter, see more details below
 51 |  * config.providers.ssm.param.NotFoundStrategy=fail
 52 |  * 
 53 |  * #        Step 2. Usage of AWS SSM Parameter Store as config provider:
 54 |  * db.username=${ssm::/msk/TestKafkaConfig/username}
 55 |  * db.password=${ssm::/msk/TestKafkaConfig/password}
 56 |  *
57 | * 58 | * Note, this config provide implementation assumes path as a parameter name. 59 | * Nested values aren't supported at this point.
60 | * 61 | * SsmParamStoreConfigProvider can be configured using parameters.
62 | * Format:
63 | * {@code config.providers.ssm.param. = }
64 | * 65 | * @param region - defines a region to get a secret from. 66 | * @param NotFoundStrategy - defines an action in case requested secret or a key in a value cannot be resolved.
67 | *
    Possible values are: 68 | *
      {@code fail} - (Default) the code will throw an exception {@code ConfigNotFoundException}
    69 | *
      {@code ignore} - a value will remain with tokens without any change
    70 | *
71 | * 72 | * 73 | * Expression usage:
74 | * property_name=${ssm::/Path/To/Parameter} 75 | * 76 | */ 77 | public class SsmParamStoreConfigProvider extends AwsServiceConfigProvider { 78 | 79 | private final Logger log = LoggerFactory.getLogger(getClass()); 80 | 81 | private String notFoundStrategy; 82 | 83 | private SsmParamStoreConfig config; 84 | private SsmClient ssmClient; 85 | 86 | @Override 87 | public void configure(Map configs) { 88 | this.config = new SsmParamStoreConfig(configs); 89 | configure(); 90 | } 91 | 92 | private void configure() { 93 | setCommonConfig(this.config); 94 | 95 | this.notFoundStrategy = this.config.getString(SsmParamStoreConfig.NOT_FOUND_STRATEGY); 96 | 97 | // set up a builder: 98 | SsmClientBuilder cBuilder = SsmClient.builder(); 99 | setClientCommonConfig(cBuilder); 100 | this.ssmClient = cBuilder.build(); 101 | } 102 | 103 | /** 104 | * Retrieves all parameters at the given path in SSM Parameters Store. 105 | * 106 | * @param path the path in Parameters Store 107 | * @return the configuration data 108 | */ 109 | @Override 110 | public ConfigData get(String path) { 111 | 112 | return get(path, Collections.emptySet()); 113 | } 114 | 115 | 116 | /** 117 | * Retrieves all parameters at the given path in SSM Parameters Store with given key. 118 | * 119 | * @param path the path in Parameters Store 120 | * @return the configuration data 121 | */ 122 | @Override 123 | public ConfigData get(String path, Set keys) { 124 | Map data = new HashMap<>(); 125 | if ( (path == null || path.isEmpty()) 126 | && (keys== null || keys.isEmpty()) ) { 127 | return new ConfigData(data); 128 | } 129 | 130 | SsmClient ssmClient = checkOrInitSsmClient(); 131 | Long ttl = null; 132 | 133 | for (String keyWithOptions: keys) { 134 | String key = parseKey(keyWithOptions); 135 | Map options = parseKeyOptions(keyWithOptions); 136 | ttl = getUpdatedTtl(ttl, options); 137 | 138 | GetParameterRequest parameterRequest = GetParameterRequest.builder().name(key).withDecryption(true).build(); 139 | try { 140 | GetParameterResponse parameterResponse = ssmClient.getParameter(parameterRequest); 141 | String value = parameterResponse.parameter().value(); 142 | data.put(keyWithOptions, value); 143 | } catch(ParameterNotFoundException e) { 144 | log.info("Parameter " + key + "not found. Value will be handled according to a strategy defined by 'NotFoundStrategy'"); 145 | handleNotFoundByStrategy(data, path, key, e); 146 | } 147 | } 148 | return ttl == null ? new ConfigData(data) : new ConfigData(data, ttl); 149 | } 150 | 151 | protected synchronized SsmClient checkOrInitSsmClient() { 152 | if (this.ssmClient == null) { 153 | configure(); 154 | } 155 | return this.ssmClient; 156 | } 157 | 158 | @Override 159 | public void close() throws IOException { 160 | log.info("Closing provider, called by thread: {}", 161 | Thread.currentThread().getName()); 162 | if (this.ssmClient != null) { 163 | this.ssmClient.close(); 164 | this.ssmClient = null; 165 | } 166 | super.close(); 167 | } 168 | 169 | private void handleNotFoundByStrategy(Map data, String path, String key, RuntimeException e) { 170 | if (SsmParamStoreConfig.NOT_FOUND_IGNORE.equals(this.notFoundStrategy) 171 | && key != null && !key.isBlank()) { 172 | data.put(key, ""); 173 | } else if (SsmParamStoreConfig.NOT_FOUND_FAIL.equals(this.notFoundStrategy)) { 174 | if (e != null) { 175 | throw e; 176 | }else { 177 | throw new ConfigException(String.format("Secret undefined %s:%s", path, key)); 178 | } 179 | } 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/S3ImportConfigProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | import java.io.IOException; 20 | import java.nio.file.Files; 21 | import java.nio.file.Path; 22 | import java.util.Collections; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | import java.util.Set; 26 | 27 | import org.apache.kafka.common.config.ConfigChangeCallback; 28 | import org.apache.kafka.common.config.ConfigData; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import com.amazonaws.kafka.config.providers.common.AwsServiceConfigProvider; 33 | 34 | import software.amazon.awssdk.regions.Region; 35 | import software.amazon.awssdk.services.s3.S3Client; 36 | import software.amazon.awssdk.services.s3.S3ClientBuilder; 37 | import software.amazon.awssdk.services.s3.model.GetObjectRequest; 38 | import software.amazon.awssdk.services.s3.model.NoSuchKeyException; 39 | 40 | /** 41 | * This class implements a ConfigProvider for AWS S3 File Importer.
42 | * 43 | *

Usage:
44 | * In a configuration file (e.g. {@code client.properties}) define following properties:
45 | * 46 | * 

 47 |  * #        Step1. Configure the secrets manager as config provider:
 48 |  * config.providers=s3import
 49 |  * config.providers.s3import.class=com.amazonaws.kafka.config.providers.S3ImportConfigProvider
 50 |  * # optional parameter for default region:
 51 |  * config.providers.s3import.param.region=us-west-2
 52 |  * 
 53 |  * #        Step 2. Usage of AWS S3 Importer as config provider with explicitly defined region:
 54 |  * database.sslcert=${s3import:us-west-2:my-bucket/full/path/file.jks}
 55 |  * #        Alternatively, use default or current region:
 56 |  * database.sslcert=${s3import::my-bucket/full/path/file.jks}
 57 |  *
58 | * 59 | * Note, you must have permissions to access an object on S3. 60 | * 61 | * @param region - defines a region to get a secret from. 62 | * 63 | * Expression usage:
64 | * property_name=${s3import::} 65 | * 66 | */ 67 | public class S3ImportConfigProvider extends AwsServiceConfigProvider { 68 | 69 | private final Logger log = LoggerFactory.getLogger(getClass()); 70 | 71 | private S3ImportConfig config; 72 | 73 | private String localDir; 74 | 75 | 76 | @Override 77 | public void configure(Map configs) { 78 | this.config = new S3ImportConfig(configs); 79 | setCommonConfig(config); 80 | 81 | this.localDir = this.config.getString(S3ImportConfig.LOCAL_DIR); 82 | if (this.localDir == null || this.localDir.isBlank()) { 83 | // if not defined, use temp dir defined in OS 84 | this.localDir = System.getProperty("java.io.tmpdir"); 85 | } 86 | } 87 | 88 | /** 89 | * Retrieves all parameters at the given path in SSM Parameters Store. 90 | * 91 | * @param path the path in Parameters Store 92 | * @return the configuration data 93 | */ 94 | @Override 95 | public ConfigData get(String path) { 96 | return get(path, Collections.emptySet()); 97 | } 98 | 99 | /** 100 | * Copies a file from S3 to a local (to a process) file system. 101 | * 102 | * @param path (optional) a region where an S3 object is located. If null, 103 | * a default region from config provider's configuration will be used. 104 | * @return the configuration data with resolved variables. 105 | */ 106 | @Override 107 | public ConfigData get(String path, Set keys) { 108 | Map data = new HashMap<>(); 109 | if ( (path == null || path.isEmpty()) 110 | && (keys== null || keys.isEmpty()) ) { 111 | return new ConfigData(data); 112 | } 113 | 114 | S3Client s3 = checkOrInitS3Client(path); 115 | 116 | for (String key: keys) { 117 | try { 118 | Path pKey = Path.of(key); 119 | Path destination = getDestination(this.localDir, pKey); 120 | log.debug("Local destination for file: {}", destination); 121 | 122 | if (Files.exists(destination)) { 123 | // Imported file may already exist on a file system. If tasks are restarting, 124 | // or more than one task is running on a worker, they may use the same file 125 | log.info("File already imported at destination: {}", destination); 126 | data.put(key, destination.toString()); 127 | continue; 128 | } 129 | GetObjectRequest s3GetObjectRequest = GetObjectRequest.builder() 130 | .bucket(getBucket(pKey)) 131 | .key(getS3ObjectKey(pKey)) 132 | .build(); 133 | s3.getObject(s3GetObjectRequest, destination); 134 | log.debug("Successfully imported a file from S3 bucket: s3://{}", key); 135 | data.put(key, destination.toString()); 136 | } catch(NoSuchKeyException nske) { 137 | // Simply throw an exception to indicate there are issues with the objects on S3 138 | throw new RuntimeException("No object found at " + key, nske); 139 | } 140 | } 141 | 142 | return new ConfigData(data); 143 | } 144 | 145 | private static Path getDestination(String localDir, Path pKey) { 146 | Path pDest = Path.of(localDir, pKey.getName(pKey.getNameCount()-1).toString()); 147 | return pDest; 148 | } 149 | 150 | private static String getS3ObjectKey(Path pKey) { 151 | return pKey.subpath(1, pKey.getNameCount()).toString(); 152 | } 153 | 154 | private static String getBucket(Path pKey) { 155 | 156 | return pKey.getName(0).toString(); 157 | } 158 | 159 | @Override 160 | public void subscribe(String path, Set keys, ConfigChangeCallback callback) { 161 | log.info("Subscription is not implemented and will be ignored"); 162 | } 163 | 164 | @Override 165 | public void close() throws IOException { 166 | } 167 | 168 | protected S3Client checkOrInitS3Client(String regionStr) { 169 | S3ClientBuilder s3cb = S3Client.builder(); 170 | 171 | setClientCommonConfig(s3cb); 172 | 173 | // If region is not provided as path, then Common Config sets default region. 174 | // No need to override. 175 | if (regionStr != null && !regionStr.isBlank()) { 176 | s3cb.region(Region.of(regionStr)); 177 | } 178 | 179 | return s3cb.build(); 180 | } 181 | 182 | 183 | 184 | } 185 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/kafka/config/providers/SecretsManagerConfigProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of this 5 | * software and associated documentation files (the "Software"), to deal in the Software 6 | * without restriction, including without limitation the rights to use, copy, modify, 7 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | * permit persons to whom the Software is furnished to do so. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 11 | * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 12 | * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 13 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 14 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | */ 17 | package com.amazonaws.kafka.config.providers; 18 | 19 | import java.io.IOException; 20 | import java.net.URLDecoder; 21 | import java.nio.charset.StandardCharsets; 22 | import java.util.Collections; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | import java.util.Set; 26 | 27 | import org.apache.kafka.common.config.ConfigChangeCallback; 28 | import org.apache.kafka.common.config.ConfigData; 29 | import org.apache.kafka.common.config.ConfigException; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | import com.amazonaws.kafka.config.providers.common.AwsServiceConfigProvider; 34 | import com.fasterxml.jackson.core.type.TypeReference; 35 | import com.fasterxml.jackson.databind.ObjectMapper; 36 | 37 | import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; 38 | import software.amazon.awssdk.services.secretsmanager.SecretsManagerClientBuilder; 39 | import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; 40 | import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; 41 | import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException; 42 | 43 | /** 44 | * This class implements a ConfigProvider for AWS Secrets Manager.
45 | * 46 | *

Usage:
47 | * In a configuration file (e.g. {@code client.properties}) define following properties:
48 | * 49 | * 

 50 |  * #        Step1. Configure the secrets manager as config provider:
 51 |  * config.providers=secretsmanager
 52 |  * config.providers.secretsmanager.class=com.amazonaws.kafka.config.providers.SecretsMamagerConfigProvider
 53 |  * # optional parameter for region:
 54 |  * config.providers.secretsmanager.param.region=us-west-2
 55 |  * # optional parameter, see more details below
 56 |  * config.providers.secretsmanager.param.NotFoundStrategy=fail
 57 |  *
 58 |  * #        Step 2. Usage of AWS secrets manager as config provider:
 59 |  * db.username=${secretsmanager:AmazonMSK_TestKafkaConfig:username}
 60 |  * db.password=${secretsmanager:AmazonMSK_TestKafkaConfig:password}
 61 |  *
62 | * 63 | * Note, this config provider implementation assumes secret values will be returned in Json format. 64 | * Nested values aren't supported at this point.
65 | * 66 | * SecretsManagerConfigProvider can be configured using parameters.
67 | * Format:
68 | * {@code config.providers.secretsmanager.param. = }
69 | * 70 | * @param region - defines a region to get a secret from. 71 | * @param NotFoundStrategy - defines an action in case requested secret or a key in a value cannot be resolved.
72 | *
    Passible values are: 73 | *
      {@code fail} - (Default) the code will throw an exception {@code ConfigNotFoundException}
    74 | *
      {@code ignore} - a value will remain with tokens without any change
    75 | *
76 | * 77 | * 78 | * Expression usage:
79 | * property_name=${secretsmanager:secret.id:secret.key} 80 | * 81 | */ 82 | public class SecretsManagerConfigProvider extends AwsServiceConfigProvider { 83 | 84 | private final Logger log = LoggerFactory.getLogger(getClass()); 85 | 86 | private static final String EMPTY = ""; 87 | 88 | private SecretsManagerConfig config; 89 | private String notFoundStrategy; 90 | 91 | private SecretsManagerClient secretsManager; 92 | 93 | @Override 94 | public void configure(Map configs) { 95 | this.config = new SecretsManagerConfig(configs); 96 | configure(); 97 | } 98 | 99 | public void configure() { 100 | setCommonConfig(config); 101 | 102 | this.notFoundStrategy = config.getString(SecretsManagerConfig.NOT_FOUND_STRATEGY); 103 | 104 | // set up a builder: 105 | SecretsManagerClientBuilder cBuilder = SecretsManagerClient.builder(); 106 | setClientCommonConfig(cBuilder); 107 | this.secretsManager = cBuilder.build(); 108 | } 109 | 110 | /** 111 | * Retrieves a secret from AWS Secrets Manager 112 | * 113 | * @param path the path in Parameters Store 114 | * @return the configuration data 115 | */ 116 | @Override 117 | public ConfigData get(String path) { 118 | return get(path, Collections.emptySet()); 119 | } 120 | 121 | 122 | /** 123 | * Retrieves secret's fields from a given secret. 124 | * 125 | * @param encodedPath AWS Secrets Manager secret name or encoded ARN 126 | * @param keys fields inside a given secret. 127 | * @return the configuration data 128 | */ 129 | @Override 130 | public ConfigData get(String encodedPath, Set keys) { 131 | Map data = new HashMap<>(); 132 | if (encodedPath == null || encodedPath.isEmpty() 133 | || keys== null || keys.isEmpty()) { 134 | // if no fields provided, just ignore this usage 135 | return new ConfigData(data); 136 | } 137 | 138 | String path = URLDecoder.decode(encodedPath, StandardCharsets.UTF_8); 139 | GetSecretValueRequest request = GetSecretValueRequest.builder().secretId(path).build(); 140 | Map secretJson = null; 141 | try { 142 | SecretsManagerClient secretsClient = checkOrInitSecretManagerClient(); 143 | GetSecretValueResponse response = secretsClient.getSecretValue(request); 144 | String value = response.secretString(); 145 | 146 | try { 147 | secretJson = new ObjectMapper().readValue(value, new TypeReference<>() {}); 148 | } catch (Exception e) { 149 | log.error("Unexpected value of a secret's structure", e); 150 | throw new ConfigException(path, value, "Unexpected value of a secret's structure"); 151 | } 152 | } catch(ResourceNotFoundException e) { 153 | log.info("Secret id {} not found. Value will be handled according to a strategy defined by 'NotFoundStrategy'", path); 154 | handleNotFoundByStrategy(data, path, null, e); 155 | } 156 | 157 | Long ttl = null; 158 | for (String keyWithOptions: keys) { 159 | String key = parseKey(keyWithOptions); 160 | Map options = parseKeyOptions(keyWithOptions); 161 | ttl = getUpdatedTtl(ttl, options); 162 | 163 | // secretJson can be null at this point only if there is a permissive strategy. 164 | if (secretJson == null) { 165 | data.put(key, EMPTY); 166 | continue; 167 | } 168 | if (secretJson.containsKey(key)) { 169 | data.put(keyWithOptions, secretJson.get(key)); 170 | } else { 171 | log.info("Secret {} doesn't have a key {}.", path, key); 172 | handleNotFoundByStrategy(data, path, key, null); 173 | } 174 | } 175 | 176 | return ttl == null ? new ConfigData(data) : new ConfigData(data, ttl); 177 | } 178 | 179 | protected synchronized SecretsManagerClient checkOrInitSecretManagerClient() { 180 | if (secretsManager == null) { 181 | configure(); 182 | } 183 | return this.secretsManager; 184 | } 185 | 186 | @Override 187 | public void subscribe(String path, Set keys, ConfigChangeCallback callback) { 188 | log.info("Subscription is not implemented and will be ignored"); 189 | } 190 | 191 | @Override 192 | public void close() throws IOException { 193 | log.info("Closing provider, called by thread: {}", 194 | Thread.currentThread().getName()); 195 | if (this.secretsManager != null) { 196 | this.secretsManager.close(); 197 | this.secretsManager = null; 198 | } 199 | super.close(); 200 | } 201 | 202 | private void handleNotFoundByStrategy(Map data, String path, String key, RuntimeException e) { 203 | if (SecretsManagerConfig.NOT_FOUND_IGNORE.equals(this.notFoundStrategy) 204 | && key != null && !key.isBlank()) { 205 | data.put(key, ""); 206 | } else if (SecretsManagerConfig.NOT_FOUND_FAIL.equals(this.notFoundStrategy)) { 207 | if (e != null) { 208 | throw e; 209 | }else { 210 | throw new ConfigException(String.format("Secret undefined {}:{}", path, key)); 211 | } 212 | } 213 | } 214 | } 215 | --------------------------------------------------------------------------------