├── .gitignore ├── Jenkinsfile ├── README.md ├── bin └── debug.sh ├── config ├── dummy.properties └── token.properties ├── docker-compose.yml ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── github │ │ └── jcustenborder │ │ └── kafka │ │ └── config │ │ └── aws │ │ ├── SecretsManagerConfigProvider.java │ │ ├── SecretsManagerConfigProviderConfig.java │ │ ├── SecretsManagerFactory.java │ │ ├── SecretsManagerFactoryImpl.java │ │ └── package-info.java └── resources │ └── META-INF │ └── services │ └── org.apache.kafka.common.config.provider.ConfigProvider └── test ├── java └── com │ └── github │ └── jcustenborder │ └── kafka │ └── config │ └── aws │ ├── DocumentationTest.java │ └── SecretsManagerConfigProviderTest.java └── resources ├── com └── github │ └── jcustenborder │ └── kafka │ └── config │ └── aws │ └── SecretsManagerConfigProvider │ └── simple.json └── logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .idea 3 | *.iml 4 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!groovy 2 | @Library('jenkins-pipeline') import com.github.jcustenborder.jenkins.pipeline.KafkaConnectPipeline 3 | 4 | def pipe = new KafkaConnectPipeline() 5 | pipe.execute() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | [Documentation](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-config-provider-aws) | [Download from the Confluent Hub](https://www.confluent.io/hub/jcustenborder/kafka-config-provider-aws) 3 | 4 | This plugin provides integration with the AWS Secrets Manager service. 5 | 6 | # Installation 7 | 8 | ## Confluent Hub 9 | 10 | The following command can be used to install the plugin directly from the Confluent Hub using the 11 | [Confluent Hub Client](https://docs.confluent.io/current/connect/managing/confluent-hub/client.html). 12 | 13 | ```bash 14 | confluent-hub install jcustenborder/kafka-config-provider-aws:latest 15 | ``` 16 | 17 | ## Manually 18 | 19 | The zip file that is deployed to the [Confluent Hub](https://www.confluent.io/hub/jcustenborder/kafka-config-provider-aws) is available under 20 | `target/components/packages/`. You can manually extract this zip file which includes all dependencies. All the dependencies 21 | that are required to deploy the plugin are under `target/kafka-connect-target` as well. Make sure that you include all the dependencies that are required 22 | to run the plugin. 23 | 24 | 1. Create a directory under the `plugin.path` on your Connect worker. 25 | 2. Copy all of the dependencies under the newly created subdirectory. 26 | 3. Restart the Connect worker. 27 | 28 | # Config Providers 29 | ## [SecretsManagerConfigProvider](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-config-provider-aws/configProviders/SecretsManagerConfigProvider.html) 30 | 31 | ``` 32 | com.github.jcustenborder.kafka.config.aws.SecretsManagerConfigProvider 33 | ``` 34 | This config provider is used to retrieve secrets from the AWS Secrets Manager service. 35 | ### Tip 36 | 37 | Config providers can be used with anything that supports the AbstractConfig base class that is shipped with Apache Kafka. 38 | 39 | 40 | 41 | 42 | 43 | 44 | # Development 45 | 46 | ## Building the source 47 | 48 | ```bash 49 | mvn clean package 50 | ``` 51 | 52 | ## Contributions 53 | 54 | Contributions are always welcomed! Before you start any development please create an issue and 55 | start a discussion. Create a pull request against your newly created issue and we're happy to see 56 | if we can merge your pull request. First and foremost any time you're adding code to the code base 57 | you need to include test coverage. Make sure that you run `mvn clean package` before submitting your 58 | pull to ensure that all of the tests, checkstyle rules, and the package can be successfully built. -------------------------------------------------------------------------------- /bin/debug.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | : ${DEBUG_SUSPEND_FLAG:='y'} 19 | export KAFKA_DEBUG='n' 20 | export DEBUG_SUSPEND_FLAG='n' 21 | # export KAFKA_OPTS='-agentpath:/Applications/YourKit-Java-Profiler-2017.02.app/Contents/Resources/bin/mac/libyjpagent.jnilib=disablestacktelemetry,exceptions=disable,delay=10000' 22 | set -e 23 | 24 | mvn clean package -DskipTests 25 | 26 | connect-standalone config/token.properties config/dummy.properties 27 | -------------------------------------------------------------------------------- /config/dummy.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | name=local-file-source 18 | connector.class=FileStreamSource 19 | tasks.max=1 20 | file=/tmp/test.txt 21 | topic=${secretsManager:foo/bar/baz:kafka} -------------------------------------------------------------------------------- /config/token.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | group.id=foo 18 | bootstrap.servers=kafka:9092 19 | key.converter=org.apache.kafka.connect.json.JsonConverter 20 | value.converter=org.apache.kafka.connect.json.JsonConverter 21 | internal.key.converter=org.apache.kafka.connect.json.JsonConverter 22 | internal.value.converter=org.apache.kafka.connect.json.JsonConverter 23 | internal.key.converter.schemas.enable=false 24 | internal.value.converter.schemas.enable=false 25 | offset.storage.file.filename=/tmp/connect.offsets 26 | plugin.path=target/kafka-connect-target/usr/share/kafka-connect 27 | 28 | config.storage.replication.factor=1 29 | config.storage.topic=connect_config 30 | offset.storage.replication.factor=1 31 | offset.storage.topic=connect_offset 32 | status.storage.replication.factor=1 33 | status.storage.topic=connect_status 34 | 35 | config.providers=secretsManager 36 | config.providers.secretsManager.class=com.github.jcustenborder.kafka.config.aws.SecretsManagerConfigProvider 37 | config.providers.secretsManager.param.aws.region=us-east-1 -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | version: "2" 18 | services: 19 | zookeeper: 20 | image: confluentinc/cp-zookeeper:6.0.0 21 | ports: 22 | - "2181:2181" 23 | environment: 24 | ZOOKEEPER_CLIENT_PORT: 2181 25 | kafka: 26 | image: confluentinc/cp-kafka:6.0.0 27 | depends_on: 28 | - zookeeper 29 | ports: 30 | - "9092:9092" 31 | environment: 32 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 33 | KAFKA_ADVERTISED_LISTENERS: "plaintext://kafka:9092" 34 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 23 | 4.0.0 24 | 25 | com.github.jcustenborder.kafka.connect 26 | kafka-connect-parent 27 | 2.8.0-1 28 | 29 | kafka-config-provider-aws 30 | 0.1-SNAPSHOT 31 | kafka-config-provider-aws 32 | A Kafka Connect plugin for interacting with Redis. 33 | https://github.com/jcustenborder/kafka-config-provider-aws 34 | 2021 35 | 36 | 37 | The Apache License, Version 2.0 38 | https://www.apache.org/licenses/LICENSE-2.0 39 | repo 40 | 41 | 42 | 43 | 44 | jcustenborder 45 | Jeremy Custenborder 46 | https://github.com/jcustenborder 47 | 48 | Committer 49 | 50 | 51 | 52 | 53 | scm:git:https://github.com/jcustenborder/kafka-config-provider-aws.git 54 | scm:git:git@github.com:jcustenborder/kafka-config-provider-aws.git 55 | 56 | https://github.com/jcustenborder/kafka-config-provider-aws 57 | 58 | 59 | github 60 | https://github.com/jcustenborder/kafka-config-provider-aws/issues 61 | 62 | 63 | 64 | 65 | com.amazonaws 66 | aws-java-sdk-bom 67 | 1.11.1025 68 | pom 69 | import 70 | 71 | 72 | 73 | 74 | 75 | com.github.jcustenborder.kafka.connect 76 | connect-utils-jackson 77 | 78 | 79 | com.amazonaws 80 | aws-java-sdk-secretsmanager 81 | 82 | 83 | com.amazonaws 84 | aws-java-sdk-sts 85 | 86 | 87 | 88 | 89 | 90 | io.confluent 91 | kafka-connect-maven-plugin 92 | 0.11.2 93 | 94 | 95 | hub 96 | 97 | kafka-connect 98 | 99 | 100 | true 101 | https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-config-provider-aws/ 102 | 103 | converter 104 | 105 | 106 | aws 107 | secrets manager 108 | 109 | Kafka AWS Secrets Manager Config Provider 110 | https://github.com/jcustenborder/kafka-config-provider-aws/issues 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/config/aws/SecretsManagerConfigProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.config.aws; 17 | 18 | import com.amazonaws.services.secretsmanager.AWSSecretsManager; 19 | import com.amazonaws.services.secretsmanager.model.DecryptionFailureException; 20 | import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; 21 | import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; 22 | import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; 23 | import com.fasterxml.jackson.databind.JsonNode; 24 | import com.fasterxml.jackson.databind.ObjectMapper; 25 | import com.fasterxml.jackson.databind.node.ObjectNode; 26 | import com.github.jcustenborder.kafka.connect.utils.config.Description; 27 | import com.github.jcustenborder.kafka.connect.utils.config.DocumentationSection; 28 | import com.github.jcustenborder.kafka.connect.utils.config.DocumentationSections; 29 | import com.github.jcustenborder.kafka.connect.utils.config.DocumentationTip; 30 | import com.google.common.collect.ImmutableSet; 31 | import org.apache.kafka.common.config.ConfigData; 32 | import org.apache.kafka.common.config.ConfigDef; 33 | import org.apache.kafka.common.config.ConfigException; 34 | import org.apache.kafka.common.config.provider.ConfigProvider; 35 | import org.slf4j.Logger; 36 | import org.slf4j.LoggerFactory; 37 | 38 | import java.io.IOException; 39 | import java.nio.file.Path; 40 | import java.nio.file.Paths; 41 | import java.util.Collections; 42 | import java.util.LinkedHashMap; 43 | import java.util.Map; 44 | import java.util.Set; 45 | 46 | @Description("This config provider is used to retrieve secrets from the AWS Secrets Manager service.") 47 | @DocumentationTip("Config providers can be used with anything that supports the AbstractConfig base class that is shipped with Apache Kafka.") 48 | @DocumentationSections( 49 | sections = { 50 | @DocumentationSection(title = "Secret Value", text = "The value for the secret must be formatted as a JSON object. " + 51 | "This allows multiple keys of data to be stored in a single secret. The name of the secret in AWS Secrets Manager " + 52 | "will correspond to the path that is requested by the config provider.\n" + 53 | "\n" + 54 | ".. code-block:: json\n" + 55 | " :caption: Example Secret Value\n" + 56 | "\n" + 57 | " {\n" + 58 | " \"username\" : \"${secretManager:secret/test/some/connector:username}\",\n" + 59 | " \"password\" : \"${secretManager:secret/test/some/connector:password}\"\n" + 60 | " }\n" + 61 | "") 62 | } 63 | ) 64 | public class SecretsManagerConfigProvider implements ConfigProvider { 65 | private static final Logger log = LoggerFactory.getLogger(SecretsManagerConfigProvider.class); 66 | SecretsManagerConfigProviderConfig config; 67 | SecretsManagerFactory secretsManagerFactory = new SecretsManagerFactoryImpl(); 68 | AWSSecretsManager secretsManager; 69 | ObjectMapper mapper = new ObjectMapper(); 70 | 71 | @Override 72 | public ConfigData get(String path) { 73 | return get(path, Collections.emptySet()); 74 | } 75 | 76 | @Override 77 | public ConfigData get(String p, Set keys) { 78 | log.info("get() - path = '{}' keys = '{}'", p, keys); 79 | 80 | Path path = (null != this.config.prefix && !this.config.prefix.isEmpty()) ? 81 | Paths.get(this.config.prefix, p) : 82 | Paths.get(p); 83 | 84 | try { 85 | log.debug("Requesting {} from Secrets Manager", path); 86 | GetSecretValueRequest request = new GetSecretValueRequest() 87 | .withSecretId(path.toString()); 88 | 89 | GetSecretValueResult result = this.secretsManager.getSecretValue(request); 90 | ObjectNode node; 91 | 92 | if (null != result.getSecretString()) { 93 | node = mapper.readValue(result.getSecretString(), ObjectNode.class); 94 | } else if (null != result.getSecretBinary()) { 95 | byte[] arr = new byte[result.getSecretBinary().remaining()]; 96 | result.getSecretBinary().get(arr); 97 | node = mapper.readValue(arr, ObjectNode.class); 98 | } else { 99 | throw new ConfigException(""); 100 | } 101 | 102 | Set propertiesToRead = (null == keys || keys.isEmpty()) ? ImmutableSet.copyOf(node.fieldNames()) : keys; 103 | Map results = new LinkedHashMap<>(propertiesToRead.size()); 104 | for (String propertyName : propertiesToRead) { 105 | JsonNode propertyNode = node.get(propertyName); 106 | if (null != propertyNode && !propertyNode.isNull()) { 107 | results.put(propertyName, propertyNode.textValue()); 108 | } 109 | } 110 | return new ConfigData(results, config.minimumSecretTTL); 111 | } catch (DecryptionFailureException ex) { 112 | throw createException(ex, "Could not decrypt secret '%s'", path); 113 | } catch (ResourceNotFoundException ex) { 114 | throw createException(ex, "Could not find secret '%s'", path); 115 | } catch (IOException ex) { 116 | throw createException(ex, "Exception thrown while reading secret '%s'", path); 117 | } 118 | } 119 | 120 | ConfigException createException(Throwable cause, String message, Object... args) { 121 | String exceptionMessage = String.format(message, args); 122 | ConfigException configException = new ConfigException(exceptionMessage); 123 | configException.initCause(cause); 124 | return configException; 125 | } 126 | 127 | @Override 128 | public void close() throws IOException { 129 | if (null != this.secretsManager) { 130 | this.secretsManager.shutdown(); 131 | } 132 | } 133 | 134 | @Override 135 | public void configure(Map settings) { 136 | this.config = new SecretsManagerConfigProviderConfig(settings); 137 | this.secretsManager = this.secretsManagerFactory.create(this.config); 138 | } 139 | 140 | public static ConfigDef config() { 141 | return SecretsManagerConfigProviderConfig.config(); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/config/aws/SecretsManagerConfigProviderConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.config.aws; 17 | 18 | import com.amazonaws.auth.AWSCredentials; 19 | import com.amazonaws.auth.BasicAWSCredentials; 20 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; 21 | import org.apache.kafka.common.config.AbstractConfig; 22 | import org.apache.kafka.common.config.ConfigDef; 23 | 24 | import java.time.Duration; 25 | import java.util.Map; 26 | 27 | class SecretsManagerConfigProviderConfig extends AbstractConfig { 28 | public static final String REGION_CONFIG = "aws.region"; 29 | static final String REGION_DOC = "Sets the region to be used by the client. For example `us-west-2`"; 30 | 31 | public static final String PREFIX_CONFIG = "secret.prefix"; 32 | static final String PREFIX_DOC = "Sets a prefix that will be added to all paths. For example you can use `staging` or `production` " + 33 | "and all of the calls to Secrets Manager will be prefixed with that path. This allows the same configuration settings to be used across " + 34 | "multiple environments."; 35 | 36 | public static final String MIN_TTL_MS_CONFIG = "secret.ttl.ms"; 37 | static final String MIN_TTL_MS_DOC = "The minimum amount of time that a secret should be used. " + 38 | "After this TTL has expired Secrets Manager will be queried again in case there is an updated configuration."; 39 | 40 | public static final String AWS_ACCESS_KEY_ID_CONFIG = "aws.access.key"; 41 | public static final String AWS_ACCESS_KEY_ID_DOC = "AWS access key ID to connect with. If this value is not " + 42 | "set the `DefaultAWSCredentialsProviderChain `_ " + 43 | "will be used to attempt loading the credentials from several default locations."; 44 | public static final String AWS_SECRET_KEY_CONFIG = "aws.secret.key"; 45 | public static final String AWS_SECRET_KEY_DOC = "AWS secret access key to connect with."; 46 | 47 | public final String region; 48 | public final long minimumSecretTTL; 49 | public final AWSCredentials credentials; 50 | public final String prefix; 51 | 52 | public SecretsManagerConfigProviderConfig(Map settings) { 53 | super(config(), settings); 54 | this.minimumSecretTTL = getLong(MIN_TTL_MS_CONFIG); 55 | this.region = getString(REGION_CONFIG); 56 | 57 | String awsAccessKeyId = getString(AWS_ACCESS_KEY_ID_CONFIG); 58 | String awsSecretKey = getPassword(AWS_SECRET_KEY_CONFIG).value(); 59 | 60 | if (null != awsAccessKeyId && !awsAccessKeyId.isEmpty()) { 61 | credentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretKey); 62 | } else { 63 | credentials = null; 64 | } 65 | prefix = getString(PREFIX_CONFIG); 66 | } 67 | 68 | public static ConfigDef config() { 69 | return new ConfigDef() 70 | .define( 71 | ConfigKeyBuilder.of(REGION_CONFIG, ConfigDef.Type.STRING) 72 | .documentation(REGION_DOC) 73 | .importance(ConfigDef.Importance.HIGH) 74 | .defaultValue("") 75 | .build() 76 | ).define( 77 | ConfigKeyBuilder.of(AWS_ACCESS_KEY_ID_CONFIG, ConfigDef.Type.STRING) 78 | .documentation(AWS_ACCESS_KEY_ID_DOC) 79 | .importance(ConfigDef.Importance.HIGH) 80 | .defaultValue("") 81 | .build() 82 | ).define( 83 | ConfigKeyBuilder.of(AWS_SECRET_KEY_CONFIG, ConfigDef.Type.PASSWORD) 84 | .documentation(AWS_SECRET_KEY_DOC) 85 | .importance(ConfigDef.Importance.HIGH) 86 | .defaultValue("") 87 | .build() 88 | ) 89 | .define( 90 | ConfigKeyBuilder.of(PREFIX_CONFIG, ConfigDef.Type.STRING) 91 | .documentation(PREFIX_DOC) 92 | .importance(ConfigDef.Importance.LOW) 93 | .defaultValue("") 94 | .build() 95 | ).define( 96 | ConfigKeyBuilder.of(MIN_TTL_MS_CONFIG, ConfigDef.Type.LONG) 97 | .documentation(MIN_TTL_MS_DOC) 98 | .importance(ConfigDef.Importance.LOW) 99 | .defaultValue(Duration.ofMinutes(5L).toMillis()) 100 | .validator(ConfigDef.Range.atLeast(1000L)) 101 | .build() 102 | ); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/config/aws/SecretsManagerFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.config.aws; 17 | 18 | import com.amazonaws.services.secretsmanager.AWSSecretsManager; 19 | 20 | interface SecretsManagerFactory { 21 | AWSSecretsManager create(SecretsManagerConfigProviderConfig config); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/config/aws/SecretsManagerFactoryImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.config.aws; 17 | 18 | import com.amazonaws.auth.AWSStaticCredentialsProvider; 19 | import com.amazonaws.services.secretsmanager.AWSSecretsManager; 20 | import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; 21 | 22 | class SecretsManagerFactoryImpl implements SecretsManagerFactory { 23 | @Override 24 | public AWSSecretsManager create(SecretsManagerConfigProviderConfig config) { 25 | AWSSecretsManagerClientBuilder builder = AWSSecretsManagerClientBuilder.standard(); 26 | 27 | if (null != config.region && !config.region.isEmpty()) { 28 | builder = builder.withRegion(config.region); 29 | } 30 | if (null != config.credentials) { 31 | builder = builder.withCredentials(new AWSStaticCredentialsProvider(config.credentials)); 32 | } 33 | 34 | return builder.build(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/config/aws/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | @PluginOwner("jcustenborder") 17 | @PluginName("kafka-config-provider-aws") 18 | @Introduction("This plugin provides integration with the AWS Secrets Manager service.") 19 | @Title("AWS Secrets Manager Config Provider") 20 | package com.github.jcustenborder.kafka.config.aws; 21 | 22 | import com.github.jcustenborder.kafka.connect.utils.config.Introduction; 23 | import com.github.jcustenborder.kafka.connect.utils.config.PluginName; 24 | import com.github.jcustenborder.kafka.connect.utils.config.PluginOwner; 25 | import com.github.jcustenborder.kafka.connect.utils.config.Title; -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.kafka.common.config.provider.ConfigProvider: -------------------------------------------------------------------------------- 1 | com.github.jcustenborder.kafka.config.aws.SecretsManagerConfigProvider -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/config/aws/DocumentationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.config.aws; 17 | 18 | import com.github.jcustenborder.kafka.connect.utils.BaseDocumentationTest; 19 | 20 | public class DocumentationTest extends BaseDocumentationTest { 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/config/aws/SecretsManagerConfigProviderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2021 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.config.aws; 17 | 18 | import com.amazonaws.services.secretsmanager.AWSSecretsManager; 19 | import com.amazonaws.services.secretsmanager.model.DecryptionFailureException; 20 | import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; 21 | import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; 22 | import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; 23 | import com.google.common.collect.ImmutableMap; 24 | import com.google.common.collect.ImmutableSet; 25 | import org.apache.kafka.common.config.ConfigData; 26 | import org.apache.kafka.common.config.ConfigException; 27 | import org.junit.jupiter.api.BeforeEach; 28 | import org.junit.jupiter.api.Test; 29 | import org.mockito.invocation.InvocationOnMock; 30 | import org.mockito.stubbing.Answer; 31 | 32 | import java.io.IOException; 33 | import java.util.Map; 34 | 35 | import static org.junit.jupiter.api.Assertions.assertEquals; 36 | import static org.junit.jupiter.api.Assertions.assertNotNull; 37 | import static org.junit.jupiter.api.Assertions.assertThrows; 38 | import static org.mockito.ArgumentMatchers.any; 39 | import static org.mockito.Mockito.mock; 40 | import static org.mockito.Mockito.when; 41 | 42 | public class SecretsManagerConfigProviderTest { 43 | AWSSecretsManager secretsManager; 44 | SecretsManagerConfigProvider provider; 45 | 46 | @BeforeEach 47 | public void beforeEach() { 48 | this.secretsManager = mock(AWSSecretsManager.class); 49 | this.provider = new SecretsManagerConfigProvider(); 50 | this.provider.secretsManagerFactory = mock(SecretsManagerFactory.class); 51 | when(this.provider.secretsManagerFactory.create(any())).thenReturn(this.secretsManager); 52 | this.provider.configure( 53 | ImmutableMap.of() 54 | ); 55 | } 56 | 57 | @Test 58 | public void afterEach() throws IOException { 59 | this.provider.close(); 60 | } 61 | 62 | @Test 63 | public void notFound() { 64 | Throwable expected = new ResourceNotFoundException("Resource 'not/found' was not found."); 65 | when(secretsManager.getSecretValue(any())).thenThrow(expected); 66 | ConfigException configException = assertThrows(ConfigException.class, () -> { 67 | this.provider.get("not/found"); 68 | }); 69 | assertEquals(expected, configException.getCause()); 70 | } 71 | 72 | @Test 73 | public void decryptionFailure() { 74 | Throwable expected = new DecryptionFailureException("Could not decrypt resource 'not/found'."); 75 | when(secretsManager.getSecretValue(any())).thenThrow(expected); 76 | ConfigException configException = assertThrows(ConfigException.class, () -> { 77 | this.provider.get("not/found"); 78 | }); 79 | assertEquals(expected, configException.getCause()); 80 | } 81 | 82 | @Test 83 | public void get() { 84 | final String secretName = "foo/bar/baz"; 85 | GetSecretValueResult result = new GetSecretValueResult() 86 | .withName(secretName) 87 | .withSecretString("{\n" + 88 | " \"username\": \"asdf\",\n" + 89 | " \"password\": \"asdf\"\n" + 90 | "}"); 91 | Map expected = ImmutableMap.of( 92 | "username", "asdf", 93 | "password", "asdf" 94 | ); 95 | when(secretsManager.getSecretValue(any())).thenAnswer(invocationOnMock -> { 96 | GetSecretValueRequest request = invocationOnMock.getArgument(0); 97 | assertEquals(secretName, request.getSecretId()); 98 | return result; 99 | }); 100 | ConfigData configData = this.provider.get(secretName, ImmutableSet.of()); 101 | assertNotNull(configData); 102 | assertEquals(expected, configData.data()); 103 | 104 | } 105 | 106 | @Test 107 | public void getPrefixed() { 108 | this.provider.configure( 109 | ImmutableMap.of(SecretsManagerConfigProviderConfig.PREFIX_CONFIG, "prefixed") 110 | ); 111 | final String secretName = "foo/bar/baz"; 112 | final String prefixedName = "prefixed/foo/bar/baz"; 113 | GetSecretValueResult result = new GetSecretValueResult() 114 | .withName(prefixedName) 115 | .withSecretString("{\n" + 116 | " \"username\": \"asdf\",\n" + 117 | " \"password\": \"asdf\"\n" + 118 | "}"); 119 | Map expected = ImmutableMap.of( 120 | "username", "asdf", 121 | "password", "asdf" 122 | ); 123 | when(secretsManager.getSecretValue(any())).thenAnswer(invocationOnMock -> { 124 | GetSecretValueRequest request = invocationOnMock.getArgument(0); 125 | assertEquals(prefixedName, request.getSecretId()); 126 | return result; 127 | }); 128 | ConfigData configData = this.provider.get(secretName, ImmutableSet.of()); 129 | assertNotNull(configData); 130 | assertEquals(expected, configData.data()); 131 | 132 | } 133 | 134 | 135 | 136 | } 137 | -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/config/aws/SecretsManagerConfigProvider/simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Token", 3 | "prefix": "secretManager", 4 | "description": "The following example uses the us-west-2 region and prefixes all secrets with `staging`.", 5 | "config": { 6 | "aws.region": "us-west-2", 7 | "secret.prefix": "staging" 8 | }, 9 | "connectorConfig": { 10 | "username": "${secretManager:secret/test/some/connector:username}", 11 | "password": "${secretManager:secret/test/some/connector:password}" 12 | }, 13 | "tip": "Some of the settings of this config provider can be configured via environment variables." 14 | } -------------------------------------------------------------------------------- /src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | --------------------------------------------------------------------------------