readRequestHeaders(HttpServletRequest request) {
81 | return Collections.list(request.getHeaderNames()).stream().collect(toMap(identity(), request::getHeader));
82 | }
83 |
84 | }
85 |
--------------------------------------------------------------------------------
/secret-provider/src/test/scala/io/lenses/connect/secrets/providers/DecodeTest.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.providers
8 |
9 | import io.lenses.connect.secrets.connect
10 | import io.lenses.connect.secrets.connect.Encoding
11 | import org.apache.commons.io.FileUtils
12 | import org.scalatest.matchers.should.Matchers
13 | import org.scalatest.wordspec.AnyWordSpec
14 |
15 | import java.io.File
16 | import java.nio.file.FileSystems
17 | import java.time.OffsetDateTime
18 | import java.util.Base64
19 |
20 | class DecodeTest extends AnyWordSpec with Matchers {
21 |
22 | val separator: String = FileSystems.getDefault.getSeparator
23 | val tmp: String =
24 | System.getProperty("java.io.tmpdir") + separator + "decoder-tests"
25 |
26 | def cleanUp(fileName: String): AnyVal = {
27 | val tmpFile = new File(fileName)
28 | if (tmpFile.exists) tmpFile.delete()
29 | }
30 |
31 | "should decode UTF" in {
32 | connect.decodeKey(None, "my-key", "secret", _ => fail("No files here")) shouldBe "secret"
33 | connect.decodeKey(Some(Encoding.UTF8), "my-key", "secret", _ => fail("No files here")) shouldBe "secret"
34 | }
35 |
36 | "should decode BASE64" in {
37 | val value = Base64.getEncoder.encodeToString("secret".getBytes)
38 | connect.decodeKey(Some(Encoding.BASE64), s"my-key", value, _ => fail("No files here")) shouldBe "secret"
39 | }
40 |
41 | "should decode BASE64 and write to a file" in {
42 | val fileName = s"${tmp}my-file-base64"
43 |
44 | val value = Base64.getEncoder.encodeToString("secret".getBytes)
45 | var written = false
46 | connect.decodeKey(
47 | Some(Encoding.BASE64_FILE),
48 | s"my-key",
49 | value,
50 | { _ =>
51 | written = true
52 | fileName
53 | },
54 | ) shouldBe fileName
55 | written shouldBe true
56 | }
57 |
58 | "should decode and write a jks" in {
59 | val fileName = s"${tmp}my-file-base64-jks"
60 | val jksFile: String =
61 | getClass.getClassLoader.getResource("keystore.jks").getPath
62 | val fileContent = FileUtils.readFileToByteArray(new File(jksFile))
63 | val jksEncoded = Base64.getEncoder.encodeToString(fileContent)
64 |
65 | var written = false
66 | connect.decodeKey(
67 | Some(Encoding.BASE64_FILE),
68 | s"keystore.jks",
69 | jksEncoded,
70 | { _ =>
71 | written = true
72 | fileName
73 | },
74 | ) shouldBe fileName
75 |
76 | written shouldBe true
77 | }
78 |
79 | "should decode UTF8 and write to a file" in {
80 | val fileName = s"${tmp}my-file-utf8"
81 | var written = false
82 |
83 | connect.decodeKey(
84 | Some(Encoding.UTF8_FILE),
85 | s"my-key",
86 | "secret",
87 | { _ =>
88 | written = true
89 | fileName
90 | },
91 | ) shouldBe fileName
92 | written shouldBe true
93 | }
94 |
95 | "min list test" in {
96 | val now = OffsetDateTime.now()
97 | val secrets = Map(
98 | "ke3" -> ("value", Some(OffsetDateTime.now().plusHours(3))),
99 | "key1" -> ("value", Some(now)),
100 | "key2" -> ("value", Some(OffsetDateTime.now().plusHours(1))),
101 | )
102 |
103 | val (expiry, _) = connect.getSecretsAndExpiry(secrets)
104 | expiry shouldBe Some(now)
105 | }
106 | }
107 |
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelper.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.providers
2 |
3 | import java.security.SecureRandom
4 | import java.util.Base64
5 | import javax.crypto.Cipher
6 | import javax.crypto.spec.IvParameterSpec
7 | import javax.crypto.spec.SecretKeySpec
8 | import scala.util.Failure
9 | import scala.util.Try
10 |
11 | private[providers] object Aes256DecodingHelper {
12 |
13 | /** characters used to separate initialisation vector from encoded text */
14 | val INITIALISATION_VECTOR_SEPARATOR = " "
15 |
16 | private val BYTES_AMOUNT = 32
17 | private val CHARSET = "UTF-8"
18 |
19 | /**
20 | * Initializes AES256 decoder for valid key or fails for invalid key
21 | *
22 | * @param key key of 32 bytes
23 | * @return AES256 decoder
24 | */
25 | def init(key: String): Either[String, Aes256DecodingHelper] =
26 | key.getBytes(CHARSET).length match {
27 | case BYTES_AMOUNT =>
28 | Right(new Aes256DecodingHelper(key, INITIALISATION_VECTOR_SEPARATOR))
29 | case n =>
30 | Left(s"Invalid secret key length ($n - required $BYTES_AMOUNT bytes)")
31 | }
32 | }
33 |
34 | private[providers] class Aes256DecodingHelper private (
35 | key: String,
36 | ivSeparator: String,
37 | ) {
38 |
39 | import Aes256DecodingHelper.CHARSET
40 | import B64._
41 |
42 | private val secret = new SecretKeySpec(key.getBytes(CHARSET), "AES")
43 |
44 | def decrypt(s: String): Try[String] =
45 | for {
46 | (iv, encoded) <- InitializationVector.extractInitialisationVector(
47 | s,
48 | ivSeparator,
49 | )
50 | decoded <- base64Decode(encoded)
51 | decrypted <- decryptBytes(iv, decoded)
52 | } yield new String(decrypted, CHARSET)
53 |
54 | private def decryptBytes(
55 | iv: InitializationVector,
56 | bytes: Array[Byte],
57 | ): Try[Array[Byte]] =
58 | for {
59 | cipher <- getCipher(Cipher.DECRYPT_MODE, iv)
60 | encrypted <- Try(cipher.doFinal(bytes))
61 | } yield encrypted
62 |
63 | private def getCipher(mode: Int, iv: InitializationVector): Try[Cipher] =
64 | Try {
65 | val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
66 | val ivSpec = new IvParameterSpec(iv.bytes)
67 | cipher.init(mode, secret, ivSpec)
68 | cipher
69 | }
70 | }
71 |
72 | private case class InitializationVector private (bytes: Array[Byte])
73 |
74 | private object InitializationVector {
75 |
76 | import B64._
77 |
78 | private val random = new SecureRandom()
79 | private val length = 16
80 |
81 | def apply(): InitializationVector = {
82 | val bytes = Array.fill[Byte](length)(0.byteValue)
83 | random.nextBytes(bytes)
84 |
85 | new InitializationVector(bytes)
86 | }
87 |
88 | def extractInitialisationVector(
89 | s: String,
90 | ivSeparator: String,
91 | ): Try[(InitializationVector, String)] =
92 | s.indexOf(ivSeparator) match {
93 | case -1 =>
94 | Failure(
95 | new IllegalStateException(
96 | "Invalid format: missing initialization vector",
97 | ),
98 | )
99 | case i =>
100 | base64Decode(s.substring(0, i)).map(b => (new InitializationVector(b), s.substring(i + 1)))
101 | }
102 | }
103 |
104 | private object B64 {
105 | def base64Decode(s: String): Try[Array[Byte]] =
106 | Try(Base64.getDecoder().decode(s))
107 | }
108 |
--------------------------------------------------------------------------------
/secret-provider/src/test/java/io/lenses/connect/secrets/vault/MockVault.java:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.vault;
8 |
9 | import io.github.jopenlibs.vault.json.JsonObject;
10 | import org.eclipse.jetty.server.Request;
11 | import org.eclipse.jetty.server.handler.AbstractHandler;
12 |
13 | import jakarta.servlet.ServletException;
14 | import jakarta.servlet.http.HttpServletRequest;
15 | import jakarta.servlet.http.HttpServletResponse;
16 | import java.io.IOException;
17 | import java.util.Map;
18 | import java.util.Optional;
19 |
20 | import static io.lenses.connect.secrets.vault.VaultTestUtils.readRequestBody;
21 | import static io.lenses.connect.secrets.vault.VaultTestUtils.readRequestHeaders;
22 |
23 | /**
24 | * This class is used to mock out a Vault server in unit tests involving retry logic. As it extends Jetty's
25 | * AbstractHandler, it can be passed to an embedded Jetty server and respond to actual (albeit
26 | * localhost) HTTP requests.
27 | *
28 | * This basic version simply responds to requests with a pre-determined HTTP status code and response body.
29 | * Example usage:
30 | *
31 | *
32 | * {@code
33 | * final Server server = new Server(8999);
34 | * server.setHandler( new MockVault(200, "{\"data\":{\"value\":\"mock\"}}") );
35 | * server.start();
36 | *
37 | * final VaultConfig vaultConfig = new VaultConfig().address("http://127.0.0.1:8999").token("mock_token").build();
38 | * final Vault vault = new Vault(vaultConfig);
39 | * final LogicalResponse response = vault.logical().read("secret/hello");
40 | *
41 | * assertEquals(200, response.getRestResponse().getStatus());
42 | * assertEquals("mock", response.getData().get("value"));
43 | *
44 | * VaultTestUtils.shutdownMockVault(server);
45 | * }
46 | *
47 | */
48 | public class MockVault extends AbstractHandler {
49 |
50 | private int mockStatus;
51 | private String mockResponse;
52 | private JsonObject requestBody;
53 | private Map requestHeaders;
54 | private String requestUrl;
55 |
56 | MockVault() {
57 | }
58 |
59 | public MockVault(final int mockStatus, final String mockResponse) {
60 | this.mockStatus = mockStatus;
61 | this.mockResponse = mockResponse;
62 | }
63 |
64 | @Override
65 | public void handle(
66 | final String target,
67 | final Request baseRequest,
68 | final HttpServletRequest request,
69 | final HttpServletResponse response
70 | ) throws IOException, ServletException {
71 | requestBody = readRequestBody(request).orElse(null);
72 | requestHeaders = readRequestHeaders(request);
73 | requestUrl = request.getRequestURL().toString();
74 | response.setContentType("application/json");
75 | baseRequest.setHandled(true);
76 | System.out.println("MockVault is sending an HTTP " + mockStatus + " code, with expected success payload...");
77 | response.setStatus(mockStatus);
78 | if (mockResponse != null) {
79 | response.getWriter().println(mockResponse);
80 | }
81 | }
82 |
83 | public Optional getRequestBody() {
84 | return Optional.ofNullable(requestBody);
85 | }
86 |
87 | public Map getRequestHeaders() {
88 | return requestHeaders;
89 | }
90 |
91 | public String getRequestUrl() {
92 | return requestUrl;
93 | }
94 |
95 | }
96 |
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/AzureHelper.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.providers
8 |
9 | import com.azure.core.credential.TokenCredential
10 | import com.azure.identity.ClientSecretCredentialBuilder
11 | import com.azure.identity.DefaultAzureCredentialBuilder
12 | import com.azure.security.keyvault.secrets.SecretClient
13 | import com.azure.security.keyvault.secrets.models.SecretProperties
14 | import com.typesafe.scalalogging.StrictLogging
15 | import io.lenses.connect.secrets.config.AzureProviderSettings
16 | import io.lenses.connect.secrets.connect
17 | import io.lenses.connect.secrets.connect.Encoding.Encoding
18 | import io.lenses.connect.secrets.connect._
19 | import org.apache.kafka.connect.errors.ConnectException
20 |
21 | import java.nio.file.FileSystems
22 | import java.time.OffsetDateTime
23 | import scala.jdk.CollectionConverters.MapHasAsScala
24 | import scala.util.Failure
25 | import scala.util.Success
26 | import scala.util.Try
27 |
28 | trait AzureHelper extends StrictLogging {
29 |
30 | private val separator: String = FileSystems.getDefault.getSeparator
31 |
32 | // look up secret in Azure
33 | def getSecretValue(
34 | rootDir: String,
35 | path: String,
36 | client: SecretClient,
37 | key: String,
38 | ): (String, Option[OffsetDateTime]) =
39 | Try(client.getSecret(key)) match {
40 | case Success(secret) =>
41 | val value = secret.getValue
42 | val props = secret.getProperties
43 |
44 | val content = retrieveEncodingFromTags(props) match {
45 | case Encoding.UTF8 =>
46 | value
47 |
48 | case Encoding.UTF8_FILE =>
49 | val fileName =
50 | getFileName(rootDir, path, key.toLowerCase, separator)
51 | fileWriter(
52 | fileName,
53 | value.getBytes,
54 | key.toLowerCase,
55 | )
56 | fileName
57 |
58 | case Encoding.BASE64 =>
59 | decode(key, value)
60 |
61 | // write to file and set the file name as the value
62 | case Encoding.BASE64_FILE | Encoding.UTF8_FILE =>
63 | val fileName =
64 | getFileName(rootDir, path, key.toLowerCase, separator)
65 | val decoded = decodeToBytes(key, value)
66 | fileWriter(
67 | fileName,
68 | decoded,
69 | key.toLowerCase,
70 | )
71 | fileName
72 | }
73 |
74 | val expiry = Option(props.getExpiresOn)
75 | (content, expiry)
76 |
77 | case Failure(e) =>
78 | throw new ConnectException(
79 | s"Failed to look up secret [$key] at [${client.getVaultUrl}]",
80 | e,
81 | )
82 | }
83 |
84 | private def retrieveEncodingFromTags(props: SecretProperties): connect.Encoding.Value =
85 | // check the file-encoding
86 | {
87 | for {
88 | propsMap <- Option(props.getTags.asScala)
89 | fileEncodingFromPropsMap <- propsMap.get(FILE_ENCODING)
90 | enc <- Encoding.withoutHyphensInsensitiveOpt(fileEncodingFromPropsMap)
91 | } yield enc
92 | }.getOrElse(Encoding.UTF8)
93 |
94 | // setup azure credentials
95 | def createCredentials(settings: AzureProviderSettings): TokenCredential = {
96 |
97 | logger.info(
98 | s"Initializing client with mode [${settings.authMode.toString}]",
99 | )
100 |
101 | settings.authMode match {
102 | case AuthMode.CREDENTIALS =>
103 | new ClientSecretCredentialBuilder()
104 | .clientId(settings.clientId)
105 | .clientSecret(settings.secretId.value())
106 | .tenantId(settings.tenantId)
107 | .build()
108 |
109 | case _ =>
110 | new DefaultAzureCredentialBuilder().build()
111 |
112 | }
113 | }
114 | }
115 |
--------------------------------------------------------------------------------
/secret-provider/src/it/scala/io/lenses/connect/secrets/testcontainers/VaultContainer.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.testcontainers
2 |
3 | import cats.effect.IO
4 | import cats.effect.Resource
5 | import io.github.jopenlibs.vault.Vault
6 | import io.github.jopenlibs.vault.VaultConfig
7 | import io.github.jopenlibs.vault.api.sys.mounts.MountPayload
8 | import io.github.jopenlibs.vault.api.sys.mounts.MountType
9 | import io.github.jopenlibs.vault.api.sys.mounts.TimeToLive
10 | import io.github.jopenlibs.vault.response.MountResponse
11 | import com.typesafe.scalalogging.LazyLogging
12 | import io.lenses.connect.secrets.testcontainers.VaultContainer._
13 | import org.scalatest.matchers.should.Matchers
14 | import org.testcontainers.utility.DockerImageName
15 | import org.testcontainers.vault.{ VaultContainer => JavaVaultContainer }
16 |
17 | import java.util.UUID
18 | import java.util.concurrent.TimeUnit
19 | import scala.jdk.CollectionConverters.MapHasAsJava
20 |
21 | case class LeaseInfo(defaultLeaseTimeSeconds: Int, maxLeaseTimeSeconds: Int) {
22 | def toDurString(): String =
23 | s""""default_lease_ttl": "${defaultLeaseTimeSeconds}s", "max_lease_ttl": "${maxLeaseTimeSeconds}s","""
24 | }
25 |
26 | class VaultContainer(maybeLeaseInfo: Option[LeaseInfo] = Option.empty)
27 | extends SingleContainer[JavaVaultContainer[_]]
28 | with Matchers
29 | with LazyLogging {
30 |
31 | private val token: String = UUID.randomUUID().toString
32 |
33 | override val container: JavaVaultContainer[_] = new JavaVaultContainer(
34 | VaultDockerImageName,
35 | )
36 | .withNetworkAliases(defaultNetworkAlias)
37 |
38 | container.withEnv(
39 | "VAULT_LOCAL_CONFIG", //"listener": [{"tcp": { "address": "0.0.0.0:8200", "tls_disable": true}}],
40 | s"""
41 | |{${maybeLeaseInfo.fold("")(_.toDurString())} "ui": true}""".stripMargin,
42 | )
43 |
44 | container.withVaultToken(token)
45 |
46 | def vaultClientResource: Resource[IO, Vault] =
47 | Resource.make(
48 | IO(
49 | new Vault(
50 | new VaultConfig()
51 | .address(container.getHttpHostAddress)
52 | .token(token)
53 | .engineVersion(2)
54 | .build(),
55 | ),
56 | ),
57 | )(_ => IO.unit)
58 |
59 | def configureMount(secretTtlSeconds: Long): IO[MountResponse] =
60 | vaultClientResource.use(client =>
61 | IO {
62 | logger.info("Configuring mount")
63 | val mountPayload = new MountPayload()
64 | mountPayload.defaultLeaseTtl(TimeToLive.of(secretTtlSeconds.intValue(), TimeUnit.SECONDS))
65 | client.mounts().enable(vaultRootPath, MountType.KEY_VALUE, mountPayload)
66 | },
67 | )
68 |
69 | def rotateSecret(
70 | secretTtlSeconds: Long,
71 | ): IO[String] =
72 | vaultClientResource.use(client =>
73 | IO {
74 | val time = System.currentTimeMillis()
75 | val newSecretValue = s"${vaultSecretPrefix}_$time"
76 | logger.info(s"Rotating secret to {}", newSecretValue)
77 | client
78 | .logical()
79 | .write(
80 | vaultSecretPath,
81 | Map[String, Object](
82 | vaultSecretKey -> s"${vaultSecretPrefix}_$time",
83 | "ttl" -> s"${secretTtlSeconds}s",
84 | "created" -> Long.box(time),
85 | "expires" -> Long.box(time + (secretTtlSeconds * 1000)),
86 | ).asJava,
87 | )
88 | newSecretValue
89 | },
90 | )
91 |
92 | def vaultToken = token
93 |
94 | def vaultAddress = container.getHttpHostAddress
95 |
96 | def networkVaultAddress: String =
97 | String.format("http://%s:%s", defaultNetworkAlias, vaultPort)
98 |
99 | }
100 |
101 | object VaultContainer {
102 | val VaultDockerImageName: DockerImageName =
103 | DockerImageName.parse("vault").withTag("1.12.3")
104 | val vaultRootPath = "rotate-test" // was: secret/
105 | val vaultSecretPath = s"$vaultRootPath/myVaultSecretPath"
106 | val vaultSecretKey = "myVaultSecretKey"
107 | val vaultSecretPrefix = "myVaultSecretValue"
108 |
109 | private val defaultNetworkAlias = "vault"
110 | private val vaultPort: Int = 8200
111 | }
112 |
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/AzureSecretProvider.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.providers
8 |
9 | import java.time.Duration
10 |
11 | import com.azure.core.credential.TokenCredential
12 | import com.azure.security.keyvault.secrets.SecretClient
13 | import com.azure.security.keyvault.secrets.SecretClientBuilder
14 | import io.lenses.connect.secrets.config.AzureProviderConfig
15 | import io.lenses.connect.secrets.config.AzureProviderSettings
16 | import io.lenses.connect.secrets.connect.getSecretsAndExpiry
17 | import org.apache.kafka.common.config.ConfigData
18 | import org.apache.kafka.common.config.provider.ConfigProvider
19 |
20 | import java.time.OffsetDateTime
21 | import java.util
22 | import scala.collection.mutable
23 | import scala.jdk.CollectionConverters._
24 |
25 | class AzureSecretProvider() extends ConfigProvider with AzureHelper {
26 |
27 | private var rootDir: String = _
28 | private var credentials: Option[TokenCredential] = None
29 | val clientMap: mutable.Map[String, SecretClient] = mutable.Map.empty
30 | val cache = mutable.Map.empty[String, (Option[OffsetDateTime], Map[String, String])]
31 |
32 | // configure the vault client
33 | override def configure(configs: util.Map[String, _]): Unit = {
34 | val settings = AzureProviderSettings(AzureProviderConfig(configs))
35 | rootDir = settings.fileDir
36 | credentials = Some(createCredentials(settings))
37 | }
38 |
39 | // lookup secrets at a path
40 | // returns and empty map since Azure is flat
41 | // and we need to know the secret to lookup
42 | override def get(path: String): ConfigData =
43 | new ConfigData(Map.empty[String, String].asJava)
44 |
45 | // get secret keys at a path
46 | // paths is expected to be the url of the azure keyvault without the protocol (https://)
47 | // since the connect work will not parse it correctly do to the :
48 | // including the azure environment.
49 | override def get(path: String, keys: util.Set[String]): ConfigData = {
50 |
51 | val keyVaultUrl =
52 | if (path.startsWith("https://")) path else s"https://$path"
53 |
54 | // don't need to cache but allows for testing
55 | // this way we don't require a keyvault set in the
56 | // worker properties and we take the path as the target keyvault
57 | val client = clientMap.getOrElse(
58 | keyVaultUrl,
59 | new SecretClientBuilder()
60 | .vaultUrl(keyVaultUrl)
61 | .credential(credentials.get)
62 | .buildClient,
63 | )
64 |
65 | clientMap += (keyVaultUrl -> client)
66 |
67 | val now = OffsetDateTime.now()
68 |
69 | val (expiry, data) = cache.get(keyVaultUrl) match {
70 | case Some((expiresAt, data)) =>
71 | // we have all the keys and are before the expiry
72 |
73 | if (
74 | keys.asScala.subsetOf(data.keySet) && expiresAt
75 | .getOrElse(now.plusSeconds(1))
76 | .isAfter(now)
77 | ) {
78 | logger.info("Fetching secrets from cache")
79 | (expiresAt, data.view.filterKeys(k => keys.contains(k)).toMap)
80 | } else {
81 | // missing some or expired so reload
82 | getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet))
83 | }
84 |
85 | case None =>
86 | getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet))
87 | }
88 |
89 | var ttl = 0L
90 | expiry.foreach { exp =>
91 | ttl = Duration.between(now, exp).toMillis
92 | logger.info(s"Min expiry for TTL set to [${exp.toString}]")
93 | }
94 | cache.put(keyVaultUrl, (expiry, data))
95 | new ConfigData(data.asJava, ttl)
96 | }
97 |
98 | override def close(): Unit = {}
99 |
100 | private def getSecrets(
101 | client: SecretClient,
102 | keys: Set[String],
103 | ): Map[String, (String, Option[OffsetDateTime])] = {
104 | val path = client.getVaultUrl.stripPrefix("https://")
105 | keys.map { key =>
106 | logger.info(s"Looking up value at [$path] for key [$key]")
107 | val (value, expiry) = getSecretValue(rootDir, path, client, key)
108 | (key, (value, expiry))
109 | }.toMap
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/secret-provider/src/it/scala/io/lenses/connect/secrets/testcontainers/KafkaConnectContainer.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.testcontainers
2 |
3 | import com.github.dockerjava.api.model.Ulimit
4 | import io.lenses.connect.secrets.testcontainers.KafkaConnectContainer.defaultNetworkAlias
5 | import io.lenses.connect.secrets.testcontainers.KafkaConnectContainer.defaultRestPort
6 | import io.lenses.connect.secrets.testcontainers.connect.ConfigProviders
7 | import org.testcontainers.containers.GenericContainer
8 | import org.testcontainers.containers.KafkaContainer
9 | import org.testcontainers.containers.wait.strategy.Wait
10 | import org.testcontainers.utility.DockerImageName
11 |
12 | import java.time.Duration
13 | import scala.jdk.CollectionConverters.MapHasAsJava
14 |
15 | class KafkaConnectContainer(
16 | dockerImage: String,
17 | networkAlias: String = defaultNetworkAlias,
18 | restPort: Int = defaultRestPort,
19 | connectPluginPath: Map[String, String] = Map.empty,
20 | kafkaContainer: KafkaContainer,
21 | maybeConfigProviders: Option[ConfigProviders],
22 | ) extends GenericContainer[KafkaConnectContainer](DockerImageName.parse(dockerImage)) {
23 | require(kafkaContainer != null, "You must define the kafka container")
24 |
25 | withNetwork(kafkaContainer.getNetwork)
26 | withNetworkAliases(networkAlias)
27 | withExposedPorts(restPort)
28 | waitingFor(
29 | Wait
30 | .forHttp("/connectors")
31 | .forPort(restPort)
32 | .forStatusCode(200),
33 | ).withStartupTimeout(Duration.ofSeconds(120))
34 | withCreateContainerCmdModifier { cmd =>
35 | val _ =
36 | cmd.getHostConfig.withUlimits(Array(new Ulimit("nofile", 65536L, 65536L)))
37 | }
38 |
39 | connectPluginPath.foreach {
40 | case (k, v) => withFileSystemBind(v, s"/usr/share/plugins/$k")
41 | }
42 |
43 | withEnv("CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE", "false")
44 | withEnv("CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE", "false")
45 | withEnv(
46 | "CONNECT_KEY_CONVERTER",
47 | "org.apache.kafka.connect.storage.StringConverter",
48 | )
49 | withEnv(
50 | "CONNECT_VALUE_CONVERTER",
51 | "org.apache.kafka.connect.storage.StringConverter",
52 | )
53 |
54 | withEnv("CONNECT_REST_ADVERTISED_HOST_NAME", networkAlias)
55 | withEnv("CONNECT_REST_PORT", restPort.toString)
56 | withEnv("CONNECT_BOOTSTRAP_SERVERS", kafkaContainer.bootstrapServers)
57 | withEnv("CONNECT_GROUP_ID", "io/lenses/connect/secrets/integration/connect")
58 | withEnv("CONNECT_CONFIG_STORAGE_TOPIC", "connect_config")
59 | withEnv("CONNECT_OFFSET_STORAGE_TOPIC", "connect_offset")
60 | withEnv("CONNECT_STATUS_STORAGE_TOPIC", "connect_status")
61 | withEnv("CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR", "1")
62 | withEnv("CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR", "1")
63 | withEnv("CONNECT_STATUS_STORAGE_REPLICATION_FACTOR", "1")
64 | withEnv(
65 | "CONNECT_PLUGIN_PATH",
66 | "/usr/share/java,/usr/share/confluent-hub-components,/usr/share/plugins",
67 | )
68 |
69 | maybeConfigProviders.map(cpc => withEnv(cpc.toEnvMap.asJava))
70 |
71 | lazy val hostNetwork = new HostNetwork()
72 |
73 | class HostNetwork {
74 | def restEndpointUrl: String = s"http://$getHost:${getMappedPort(restPort)}"
75 | }
76 |
77 | }
78 | object KafkaConnectContainer {
79 | private val dockerImage =
80 | DockerImageName.parse("confluentinc/cp-kafka-connect")
81 | private val defaultConfluentPlatformVersion: String =
82 | sys.env.getOrElse("CONFLUENT_VERSION", "7.3.1")
83 | private val defaultNetworkAlias = "connect"
84 | private val defaultRestPort = 8083
85 |
86 | def apply(
87 | confluentPlatformVersion: String = defaultConfluentPlatformVersion,
88 | networkAlias: String = defaultNetworkAlias,
89 | restPort: Int = defaultRestPort,
90 | connectPluginPathMap: Map[String, String] = Map.empty,
91 | kafkaContainer: KafkaContainer,
92 | maybeConfigProviders: Option[ConfigProviders],
93 | ): KafkaConnectContainer =
94 | new KafkaConnectContainer(
95 | dockerImage.withTag(confluentPlatformVersion).toString,
96 | networkAlias,
97 | restPort,
98 | connectPluginPathMap,
99 | kafkaContainer,
100 | maybeConfigProviders,
101 | )
102 | }
103 |
--------------------------------------------------------------------------------
/secret-provider/src/test/scala/io/lenses/connect/secrets/providers/VaultHelperTest.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.providers
2 | import io.github.jopenlibs.vault.Vault
3 | import io.github.jopenlibs.vault.response.DatabaseResponse
4 | import io.github.jopenlibs.vault.response.LogicalResponse
5 | import io.lenses.connect.secrets.cache.Ttl
6 | import io.lenses.connect.secrets.cache.ValueWithTtl
7 | import io.lenses.connect.secrets.io.FileWriter
8 | import org.mockito.Answers
9 | import org.mockito.Mockito.when
10 | import org.scalatest.funsuite.AnyFunSuite
11 | import org.scalatest.matchers.should.Matchers
12 | import org.scalatest.BeforeAndAfterAll
13 | import org.scalatest.EitherValues
14 | import org.scalatest.OptionValues
15 | import org.scalatestplus.mockito.MockitoSugar
16 | import java.time._
17 | import java.util.{ Map => JavaMap }
18 | import scala.jdk.CollectionConverters._
19 |
20 | class VaultHelperTest
21 | extends AnyFunSuite
22 | with Matchers
23 | with BeforeAndAfterAll
24 | with MockitoSugar
25 | with OptionValues
26 | with EitherValues {
27 |
28 | private val notFoundResponse = {
29 | val emptyResponse = mock[LogicalResponse](Answers.RETURNS_DEEP_STUBS)
30 | when(emptyResponse.getRestResponse.getStatus).thenReturn(404)
31 | when(emptyResponse.getRestResponse.getBody).thenReturn("Not found".getBytes)
32 | emptyResponse
33 | }
34 |
35 | private val validResponse: LogicalResponse = {
36 | val sampleResponseData: JavaMap[String, String] = Map("key1" -> "value1", "key2" -> "value2").asJava
37 | val sampleResponse: LogicalResponse = mock[LogicalResponse](Answers.RETURNS_DEEP_STUBS)
38 | when(sampleResponse.getData).thenReturn(sampleResponseData)
39 | when(sampleResponse.getRestResponse.getStatus).thenReturn(200)
40 | sampleResponse
41 | }
42 |
43 | private val validDatabaseResponse: DatabaseResponse = {
44 | val sampleResponseData: JavaMap[String, String] = Map("key1" -> "value1", "key2" -> "value2").asJava
45 | val sampleResponse: DatabaseResponse = mock[DatabaseResponse](Answers.RETURNS_DEEP_STUBS)
46 | when(sampleResponse.getData).thenReturn(sampleResponseData)
47 | when(sampleResponse.getRestResponse.getStatus).thenReturn(200)
48 | sampleResponse
49 | }
50 |
51 | private val clock: Clock = Clock.fixed(Instant.now(), ZoneId.systemDefault())
52 | private val defaultTtl: Duration = Duration.ofMinutes(10)
53 | private val expiry: OffsetDateTime = clock.instant().plus(defaultTtl).atZone(clock.getZone).toOffsetDateTime
54 |
55 | private val fileWriterCreateFn: () => Option[FileWriter] = () => None
56 |
57 | test("VaultHelper.lookup should fetch secrets from Vault and return them as a ValueWithTtl") {
58 |
59 | val vaultClient: Vault = mock[Vault](Answers.RETURNS_DEEP_STUBS)
60 | when(vaultClient.logical().read("secret/path")).thenReturn(validResponse)
61 |
62 | val vaultHelper: VaultHelper = new VaultHelper(vaultClient, Some(defaultTtl), fileWriterCreateFn)(clock)
63 |
64 | val result = vaultHelper.lookup("secret/path")
65 |
66 | result.value shouldBe ValueWithTtl[Map[String, String]](Some(Ttl(defaultTtl, expiry)),
67 | Map("key1" -> "value1", "key2" -> "value2"),
68 | )
69 | }
70 |
71 | test(
72 | "VaultHelper.lookup should fetch secrets from Vault Database Credentials Engine and return them as a ValueWithTtl",
73 | ) {
74 |
75 | val vaultClient: Vault = mock[Vault](Answers.RETURNS_DEEP_STUBS)
76 | when(vaultClient.database().creds("sausages")).thenReturn(validDatabaseResponse)
77 |
78 | val vaultHelper: VaultHelper = new VaultHelper(vaultClient, Some(defaultTtl), fileWriterCreateFn)(clock)
79 |
80 | val result = vaultHelper.lookup("database/creds/sausages")
81 |
82 | result.value shouldBe ValueWithTtl[Map[String, String]](Some(Ttl(defaultTtl, expiry)),
83 | Map("key1" -> "value1", "key2" -> "value2"),
84 | )
85 | }
86 |
87 | test("VaultHelper.lookup should handle secrets not found at the specified path") {
88 | val vaultClient: Vault = mock[Vault](Answers.RETURNS_DEEP_STUBS)
89 | when(vaultClient.logical().read("empty/path")).thenReturn(notFoundResponse)
90 |
91 | val vaultHelper: VaultHelper = new VaultHelper(vaultClient, Some(defaultTtl), fileWriterCreateFn)(clock)
92 |
93 | val result = vaultHelper.lookup("empty/path")
94 |
95 | result.left.value.getMessage should include("No secrets found at path [empty/path]")
96 | }
97 |
98 | }
99 |
--------------------------------------------------------------------------------
/project/Dependencies.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017-2020 Lenses.io Ltd
3 | */
4 |
5 | import sbt._
6 |
7 | trait Dependencies {
8 |
9 | object Versions {
10 |
11 | val scalaLoggingVersion = "3.9.5"
12 | val kafkaVersion = "3.4.0"
13 | val vaultVersion = "5.3.0"
14 | val azureKeyVaultVersion = "4.5.2"
15 | val azureIdentityVersion = "1.8.1"
16 |
17 | val awsSdkV2Version = "2.20.26"
18 |
19 | //test
20 | val scalaTestVersion = "3.2.15"
21 | val mockitoVersion = "3.2.15.0"
22 | val byteBuddyVersion = "1.14.2"
23 | val slf4jVersion = "2.0.5"
24 | val commonsIOVersion = "1.3.2"
25 | val jettyVersion = "11.0.14"
26 | val flexmarkVersion = "0.64.0"
27 | val jsonSmartVersion = "2.5.0"
28 |
29 | val scalaCollectionCompatVersion = "2.8.1"
30 | val jakartaServletVersion = "6.0.0"
31 | val testContainersVersion = "1.17.6"
32 | val json4sVersion = "4.0.6"
33 | val catsEffectVersion = "3.4.8"
34 |
35 | }
36 |
37 | object Dependencies {
38 |
39 | import Versions._
40 |
41 | val `scala-logging` =
42 | "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion
43 | val `kafka-connect-api` = "org.apache.kafka" % "connect-api" % kafkaVersion
44 | val `vault-java-driver` =
45 | "io.github.jopenlibs" % "vault-java-driver" % vaultVersion
46 | val `azure-key-vault` =
47 | "com.azure" % "azure-security-keyvault-secrets" % azureKeyVaultVersion
48 | val `azure-identity` = "com.azure" % "azure-identity" % azureIdentityVersion
49 |
50 | lazy val awsSecretsManagerSdkV2 = "software.amazon.awssdk" % "secretsmanager" % awsSdkV2Version
51 | lazy val awsIamSdkV2 = "software.amazon.awssdk" % "iam" % awsSdkV2Version
52 | lazy val awsStsSdkV2 = "software.amazon.awssdk" % "sts" % awsSdkV2Version
53 |
54 | val `mockito` = "org.scalatestplus" %% "mockito-4-6" % mockitoVersion
55 | val `scalatest` = "org.scalatest" %% "scalatest" % scalaTestVersion
56 | val `jetty` = "org.eclipse.jetty" % "jetty-server" % jettyVersion
57 | val `commons-io` = "org.apache.commons" % "commons-io" % commonsIOVersion
58 | val `flexmark` = "com.vladsch.flexmark" % "flexmark-all" % flexmarkVersion
59 | val `slf4j-api` = "org.slf4j" % "slf4j-api" % slf4jVersion
60 | val `slf4j-simple` = "org.slf4j" % "slf4j-simple" % slf4jVersion
61 |
62 | val `byteBuddy` = "net.bytebuddy" % "byte-buddy" % byteBuddyVersion
63 | val `scalaCollectionCompat` =
64 | "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion
65 | val `jakartaServlet` =
66 | "jakarta.servlet" % "jakarta.servlet-api" % jakartaServletVersion
67 |
68 | val `testContainersCore` =
69 | "org.testcontainers" % "testcontainers" % testContainersVersion
70 | val `testContainersKafka` =
71 | "org.testcontainers" % "kafka" % testContainersVersion
72 | val `testContainersVault` =
73 | "org.testcontainers" % "vault" % testContainersVersion
74 |
75 | val `json4sNative` = "org.json4s" %% "json4s-native" % json4sVersion
76 | val `json4sJackson` = "org.json4s" %% "json4s-jackson" % json4sVersion
77 | val `cats` = "org.typelevel" %% "cats-effect" % catsEffectVersion
78 |
79 | val `jsonSmart` = "net.minidev" % "json-smart" % jsonSmartVersion
80 |
81 | }
82 |
83 | import Dependencies._
84 | val secretProviderDeps = Seq(
85 | `scala-logging`,
86 | `jsonSmart`,
87 | `kafka-connect-api` % Provided,
88 | `vault-java-driver`,
89 | `azure-key-vault` exclude ("net.minidev", "json-smart"),
90 | `azure-identity` exclude ("javax.activation", "activation") exclude ("net.minidev", "json-smart"),
91 | `awsSecretsManagerSdkV2`,
92 | `awsStsSdkV2`,
93 | `awsIamSdkV2` % IntegrationTest,
94 | `jakartaServlet` % Test,
95 | `mockito` % Test,
96 | `byteBuddy` % Test,
97 | `scalatest` % Test,
98 | `jetty` % Test,
99 | `commons-io` % Test,
100 | `flexmark` % Test,
101 | `slf4j-api` % Test,
102 | `slf4j-simple` % Test,
103 | `cats` % IntegrationTest,
104 | `testContainersCore` % IntegrationTest,
105 | `testContainersKafka` % IntegrationTest,
106 | `testContainersVault` % IntegrationTest,
107 | `json4sNative` % IntegrationTest,
108 | )
109 |
110 | val testSinkDeps = Seq(
111 | `scala-logging`,
112 | `kafka-connect-api` % Provided,
113 | `vault-java-driver`,
114 | `awsSecretsManagerSdkV2`,
115 | `json4sNative`,
116 | `scalatest` % Test,
117 | )
118 |
119 | }
120 |
--------------------------------------------------------------------------------
/secret-provider/src/it/scala/io/lenses/connect/secrets/testcontainers/connect/KafkaConnectClient.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.testcontainers.connect
2 |
3 | import com.typesafe.scalalogging.StrictLogging
4 | import io.lenses.connect.secrets.testcontainers.KafkaConnectContainer
5 | import io.lenses.connect.secrets.testcontainers.connect.KafkaConnectClient.ConnectorStatus
6 | import org.apache.http.HttpHeaders
7 | import org.apache.http.HttpResponse
8 | import org.apache.http.client.HttpClient
9 | import org.apache.http.client.methods.HttpDelete
10 | import org.apache.http.client.methods.HttpGet
11 | import org.apache.http.client.methods.HttpPost
12 | import org.apache.http.client.methods.HttpPut
13 | import org.apache.http.entity.StringEntity
14 | import org.apache.http.impl.client.HttpClientBuilder
15 | import org.apache.http.message.BasicHeader
16 | import org.apache.http.util.EntityUtils
17 | import org.json4s.DefaultFormats
18 | import org.json4s._
19 | import org.json4s.native.JsonMethods._
20 | import org.scalatest.concurrent.Eventually
21 | import org.testcontainers.shaded.org.awaitility.Awaitility.await
22 |
23 | import java.util.concurrent.TimeUnit
24 | import scala.jdk.CollectionConverters._
25 |
26 | class KafkaConnectClient(kafkaConnectContainer: KafkaConnectContainer) extends StrictLogging with Eventually {
27 |
28 | implicit val formats: DefaultFormats.type = DefaultFormats
29 |
30 | val httpClient: HttpClient = {
31 | val acceptHeader = new BasicHeader(HttpHeaders.ACCEPT, "application/json")
32 | val contentHeader =
33 | new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json")
34 | HttpClientBuilder.create
35 | .setDefaultHeaders(List(acceptHeader, contentHeader).asJava)
36 | .build()
37 | }
38 |
39 | def configureLogging(loggerFQCN: String, loggerLevel: String): Unit = {
40 | val entity = new StringEntity(s"""{ "level": "${loggerLevel.toUpperCase}" }""")
41 | entity.setContentType("application/json")
42 | val httpPut = new HttpPut(
43 | s"${kafkaConnectContainer.hostNetwork.restEndpointUrl}/admin/loggers/$loggerFQCN",
44 | )
45 | httpPut.setEntity(entity)
46 | val response = httpClient.execute(httpPut)
47 | checkRequestSuccessful(response)
48 | }
49 |
50 | def registerConnector(
51 | connector: ConnectorConfiguration,
52 | timeoutSeconds: Long = 10L,
53 | ): Unit = {
54 | val httpPost = new HttpPost(
55 | s"${kafkaConnectContainer.hostNetwork.restEndpointUrl}/connectors",
56 | )
57 | val entity = new StringEntity(connector.toJson())
58 | httpPost.setEntity(entity)
59 | val response = httpClient.execute(httpPost)
60 | checkRequestSuccessful(response)
61 | EntityUtils.consume(response.getEntity)
62 | await
63 | .atMost(timeoutSeconds, TimeUnit.SECONDS)
64 | .until(() => isConnectorConfigured(connector.name))
65 | }
66 |
67 | def deleteConnector(
68 | connectorName: String,
69 | timeoutSeconds: Long = 10L,
70 | ): Unit = {
71 | val httpDelete =
72 | new HttpDelete(
73 | s"${kafkaConnectContainer.hostNetwork.restEndpointUrl}/connectors/$connectorName",
74 | )
75 | val response = httpClient.execute(httpDelete)
76 | checkRequestSuccessful(response)
77 | EntityUtils.consume(response.getEntity)
78 | await
79 | .atMost(timeoutSeconds, TimeUnit.SECONDS)
80 | .until(() => !this.isConnectorConfigured(connectorName))
81 | }
82 |
83 | def getConnectorStatus(connectorName: String): ConnectorStatus = {
84 | val httpGet = new HttpGet(
85 | s"${kafkaConnectContainer.hostNetwork.restEndpointUrl}/connectors/$connectorName/status",
86 | )
87 | val response = httpClient.execute(httpGet)
88 | checkRequestSuccessful(response)
89 | val strResponse = EntityUtils.toString(response.getEntity)
90 | logger.info(s"Connector status: $strResponse")
91 | parse(strResponse).extract[ConnectorStatus]
92 | }
93 |
94 | def isConnectorConfigured(connectorName: String): Boolean = {
95 | val httpGet = new HttpGet(
96 | s"${kafkaConnectContainer.hostNetwork.restEndpointUrl}/connectors/$connectorName",
97 | )
98 | val response = httpClient.execute(httpGet)
99 | EntityUtils.consume(response.getEntity)
100 | response.getStatusLine.getStatusCode == 200
101 | }
102 |
103 | def waitConnectorInRunningState(
104 | connectorName: String,
105 | timeoutSeconds: Long = 10L,
106 | ): Unit =
107 | await
108 | .atMost(timeoutSeconds, TimeUnit.SECONDS)
109 | .until { () =>
110 | try {
111 | val connectorState: String =
112 | getConnectorStatus(connectorName).connector.state
113 | logger.info("Connector State: {}", connectorState)
114 | connectorState.equals("RUNNING")
115 | } catch {
116 | case e: Throwable =>
117 | logger.error("Connector Throwable: {}", e)
118 | false
119 | }
120 | }
121 |
122 | def checkRequestSuccessful(response: HttpResponse): Unit =
123 | if (!isSuccess(response.getStatusLine.getStatusCode)) {
124 | throw new IllegalStateException(
125 | s"Http request failed with response: ${EntityUtils.toString(response.getEntity)}",
126 | )
127 | }
128 |
129 | def isSuccess(code: Int): Boolean = code / 100 == 2
130 | }
131 |
132 | object KafkaConnectClient {
133 | case class ConnectorStatus(
134 | name: String,
135 | connector: Connector,
136 | tasks: Seq[Tasks],
137 | `type`: String,
138 | )
139 | case class Connector(state: String, worker_id: String)
140 | case class Tasks(
141 | id: Int,
142 | state: String,
143 | worker_id: String,
144 | trace: Option[String],
145 | )
146 | }
147 |
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/package.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets
8 |
9 | import com.typesafe.scalalogging.StrictLogging
10 | import org.apache.kafka.connect.errors.ConnectException
11 |
12 | import java.io.File
13 | import java.io.FileOutputStream
14 | import java.time.OffsetDateTime
15 | import java.util.Base64
16 | import scala.collection.mutable
17 | import scala.util.Failure
18 | import scala.util.Success
19 | import scala.util.Try
20 |
21 | package object connect extends StrictLogging {
22 |
23 | val FILE_ENCODING: String = "file-encoding"
24 | val FILE_DIR: String = "file.dir"
25 | val FILE_DIR_DESC: String =
26 | """
27 | | Location to write any files for any secrets that need to
28 | | be written to disk. For example java keystores.
29 | | Files will be written under this directory following the
30 | | pattern /file.dir/[path|keyvault]/key
31 | |""".stripMargin
32 |
33 | val WRITE_FILES: String = "file.write"
34 | val WRITE_FILES_DESC: String =
35 | """
36 | | Boolean flag whether to write secrets to disk. Defaults
37 | | to false. Should be set to true when writing Java
38 | | keystores etc.
39 | |""".stripMargin
40 |
41 | val SECRET_DEFAULT_TTL = "secret.default.ttl"
42 | val SECRET_DEFAULT_TTL_DEFAULT = 0L
43 |
44 | object AuthMode extends Enumeration {
45 | type AuthMode = Value
46 | val DEFAULT, CREDENTIALS = Value
47 | def withNameOpt(s: String): Option[Value] = values.find(_.toString == s)
48 | }
49 |
50 | object Encoding extends Enumeration {
51 | type Encoding = Value
52 | val BASE64, BASE64_FILE, UTF8, UTF8_FILE = Value
53 | def withNameOpt(s: String): Option[Value] = values.find(_.toString == s)
54 |
55 | def withoutHyphensInsensitiveOpt(s: String): Option[Value] = withNameOpt(s.replace("-", "").toUpperCase)
56 | }
57 |
58 | // get the authmethod
59 | def getAuthenticationMethod(method: String): AuthMode.Value =
60 | AuthMode.withNameOpt(method.toUpperCase) match {
61 | case Some(auth) => auth
62 | case None =>
63 | throw new ConnectException(
64 | s"Unsupported authentication method",
65 | )
66 | }
67 |
68 | // base64 decode secret
69 | def decode(key: String, value: String): String =
70 | Try(Base64.getDecoder.decode(value)) match {
71 | case Success(decoded) => decoded.map(_.toChar).mkString
72 | case Failure(exception) =>
73 | throw new ConnectException(
74 | s"Failed to decode value for key [$key]",
75 | exception,
76 | )
77 | }
78 |
79 | def decodeToBytes(key: String, value: String): Array[Byte] =
80 | Try(Base64.getDecoder.decode(value)) match {
81 | case Success(decoded) => decoded
82 | case Failure(exception) =>
83 | throw new ConnectException(
84 | s"Failed to decode value for key [$key]",
85 | exception,
86 | )
87 | }
88 |
89 | // decode a key bases on the prefix encoding
90 | def decodeKey(
91 | encoding: Option[Encoding.Value],
92 | key: String,
93 | value: String,
94 | writeFileFn: Array[Byte] => String,
95 | ): String =
96 | encoding.fold(value) {
97 | case Encoding.BASE64 => decode(key, value)
98 | case Encoding.BASE64_FILE =>
99 | val decoded = decodeToBytes(key, value)
100 | writeFileFn(decoded)
101 | case Encoding.UTF8 => value
102 | case Encoding.UTF8_FILE => writeFileFn(value.getBytes())
103 | }
104 |
105 | // write secrets to file
106 | private def writer(file: File, payload: Array[Byte], key: String): Unit =
107 | Try(file.createNewFile()) match {
108 | case Success(_) =>
109 | Try(new FileOutputStream(file)) match {
110 | case Success(fos) =>
111 | fos.write(payload)
112 | logger.info(
113 | s"Payload written to [${file.getAbsolutePath}] for key [$key]",
114 | )
115 |
116 | case Failure(exception) =>
117 | throw new ConnectException(
118 | s"Failed to write payload to file [${file.getAbsolutePath}] for key [$key]",
119 | exception,
120 | )
121 | }
122 |
123 | case Failure(exception) =>
124 | throw new ConnectException(
125 | s"Failed to create file [${file.getAbsolutePath}] for key [$key]",
126 | exception,
127 | )
128 | }
129 |
130 | // write secrets to a file
131 | def fileWriter(
132 | fileName: String,
133 | payload: Array[Byte],
134 | key: String,
135 | overwrite: Boolean = false,
136 | ): Unit = {
137 | val file = new File(fileName)
138 | file.getParentFile.mkdirs()
139 |
140 | if (file.exists()) {
141 | logger.warn(s"File [$fileName] already exists")
142 | if (overwrite) {
143 | writer(file, payload, key)
144 | }
145 | } else {
146 | writer(file, payload, key)
147 | }
148 | }
149 |
150 | //calculate the min expiry for secrets and return the configData and expiry
151 | def getSecretsAndExpiry(
152 | secrets: Map[String, (String, Option[OffsetDateTime])],
153 | ): (Option[OffsetDateTime], Map[String, String]) = {
154 | val expiryList = mutable.ListBuffer.empty[OffsetDateTime]
155 |
156 | val data = secrets
157 | .map({
158 | case (key, (value, expiry)) =>
159 | expiry.foreach(e => expiryList.append(e))
160 | (key, value)
161 | })
162 |
163 | if (expiryList.isEmpty) {
164 | (None, data)
165 | } else {
166 | val minExpiry = expiryList.min
167 | (Some(minExpiry), data)
168 | }
169 | }
170 |
171 | def getFileName(
172 | rootDir: String,
173 | path: String,
174 | key: String,
175 | separator: String,
176 | ): String =
177 | s"${rootDir.stripSuffix(separator)}$separator$path$separator${key.toLowerCase}"
178 | }
179 |
--------------------------------------------------------------------------------
/secret-provider/src/it/scala/io/lenses/connect/secrets/integration/VaultSecretProviderIT.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.integration
2 |
3 | import cats.effect.FiberIO
4 | import cats.effect.IO
5 | import cats.effect.Ref
6 | import cats.effect.Resource
7 | import cats.effect.unsafe.implicits.global
8 | import cats.implicits.catsSyntaxOptionId
9 | import io.lenses.connect.secrets.testcontainers.LeaseInfo
10 | import io.lenses.connect.secrets.testcontainers.VaultContainer
11 | import io.lenses.connect.secrets.testcontainers.VaultContainer.vaultSecretKey
12 | import io.lenses.connect.secrets.testcontainers.VaultContainer.vaultSecretPath
13 | import io.lenses.connect.secrets.testcontainers.connect.ConfigProvider
14 | import io.lenses.connect.secrets.testcontainers.connect.ConfigProviders
15 | import io.lenses.connect.secrets.testcontainers.connect.ConnectorConfiguration
16 | import io.lenses.connect.secrets.testcontainers.connect.KafkaConnectClient
17 | import io.lenses.connect.secrets.testcontainers.connect.StringCnfVal
18 | import io.lenses.connect.secrets.testcontainers.scalatest.SecretProviderContainerPerSuite
19 | import org.apache.kafka.clients.producer.KafkaProducer
20 | import org.apache.kafka.clients.producer.ProducerRecord
21 | import org.scalatest.BeforeAndAfterAll
22 | import org.scalatest.funsuite.AnyFunSuite
23 | import org.scalatest.matchers.should.Matchers
24 |
25 | import java.util.UUID
26 | import java.util.concurrent.TimeUnit
27 | import scala.concurrent.duration.FiniteDuration
28 |
29 | class VaultSecretProviderIT
30 | extends AnyFunSuite
31 | with SecretProviderContainerPerSuite
32 | with Matchers
33 | with BeforeAndAfterAll {
34 |
35 | lazy val secretTtlSeconds = 5
36 |
37 | lazy val container: VaultContainer =
38 | new VaultContainer(
39 | LeaseInfo(secretTtlSeconds, secretTtlSeconds).some,
40 | ).withNetwork(network)
41 |
42 | override def modules: Seq[String] = Seq("secret-provider", "test-sink")
43 |
44 | val topicName = "vaultVandal"
45 | val connectorName = "testSink"
46 | override def beforeAll(): Unit = {
47 | container.start()
48 | super.beforeAll()
49 | }
50 |
51 | override def afterAll(): Unit = {
52 | super.afterAll()
53 | container.stop()
54 | }
55 |
56 | test("does not fail with secret TTLs") {
57 | (for {
58 | _ <- container.configureMount(secretTtlSeconds)
59 | initialSecret <- container.rotateSecret(secretTtlSeconds)
60 | latestSecretRef <- Ref.of[IO, String](initialSecret)
61 | _ <- secretUpdater(latestSecretRef)
62 | _ <- startConnector()
63 | } yield ()).unsafeRunSync()
64 | }
65 |
66 | private def startConnector(): IO[Unit] =
67 | makeStringStringProducerResource().use { prod =>
68 | withConnectorResource(sinkConfig()).use(_ => writeRecords(prod))
69 | }
70 |
71 | private def writeRecords(prod: KafkaProducer[String, String]): IO[Unit] =
72 | IO {
73 | (1 to 100).foreach { i =>
74 | writeRecord(prod, i)
75 | Thread.sleep(250)
76 | }
77 | }
78 |
79 | private def writeRecord(
80 | producer: KafkaProducer[String, String],
81 | i: Int,
82 | ): Unit = {
83 | val record: ProducerRecord[String, String] =
84 | new ProducerRecord[String, String](
85 | topicName,
86 | s"${i}_${UUID.randomUUID().toString}",
87 | )
88 | producer.send(record).get
89 | producer.flush()
90 | }
91 |
92 | private def withConnectorResource(
93 | connectorConfig: ConnectorConfiguration,
94 | timeoutSeconds: Long = 10L,
95 | )(
96 | implicit
97 | kafkaConnectClient: KafkaConnectClient,
98 | ): Resource[IO, String] =
99 | Resource.make(
100 | IO {
101 | val connectorName = connectorConfig.name
102 | kafkaConnectClient.registerConnector(connectorConfig)
103 | kafkaConnectClient.configureLogging("org.apache.kafka.connect.runtime.WorkerConfigTransformer", "DEBUG")
104 | kafkaConnectClient.waitConnectorInRunningState(
105 | connectorName,
106 | timeoutSeconds,
107 | )
108 | connectorName
109 | },
110 | )(connectorName => IO(kafkaConnectClient.deleteConnector(connectorName)))
111 |
112 | private def secretUpdater(ref: Ref[IO, String]): IO[FiberIO[Unit]] =
113 | (for {
114 | _ <- IO.sleep(FiniteDuration(secretTtlSeconds, TimeUnit.SECONDS))
115 | rotated <- container.rotateSecret(secretTtlSeconds)
116 | _ <- ref.set(rotated)
117 | updateAgain <- secretUpdater(ref)
118 | _ <- updateAgain.join
119 | } yield ()).start
120 |
121 | private def sinkConfig(): ConnectorConfiguration =
122 | ConnectorConfiguration(
123 | connectorName,
124 | Map(
125 | "name" -> StringCnfVal(connectorName),
126 | "connector.class" -> StringCnfVal(
127 | "io.lenses.connect.secrets.test.vault.TestSinkConnector",
128 | ),
129 | "topics" -> StringCnfVal(topicName),
130 | "key.converter" -> StringCnfVal(
131 | "org.apache.kafka.connect.storage.StringConverter",
132 | ),
133 | "value.converter" -> StringCnfVal(
134 | "org.apache.kafka.connect.storage.StringConverter",
135 | ),
136 | "test.sink.vault.host" -> StringCnfVal(container.networkVaultAddress),
137 | "test.sink.vault.token" -> StringCnfVal(container.vaultToken),
138 | "test.sink.secret.path" -> StringCnfVal(vaultSecretPath),
139 | "test.sink.secret.key" -> StringCnfVal(vaultSecretKey),
140 | "test.sink.secret.value" -> StringCnfVal(
141 | s"$${vault:$vaultSecretPath:$vaultSecretKey}",
142 | ),
143 | ),
144 | )
145 |
146 | override def getConfigProviders(): Option[ConfigProviders] =
147 | ConfigProviders(
148 | Seq(
149 | ConfigProvider(
150 | "vault",
151 | "io.lenses.connect.secrets.providers.VaultSecretProvider",
152 | Map(
153 | "vault.addr" -> container.networkVaultAddress,
154 | "vault.engine.version" -> "2",
155 | "vault.auth.method" -> "token",
156 | "vault.token" -> container.vaultToken,
157 | "default.ttl" -> "30000",
158 | "file.write" -> "false",
159 | ),
160 | ),
161 | ),
162 | ).some
163 | }
164 |
--------------------------------------------------------------------------------
/secret-provider/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingProviderTest.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.providers
2 |
3 | import io.lenses.connect.secrets.config.Aes256ProviderConfig
4 | import io.lenses.connect.secrets.connect.Encoding
5 | import io.lenses.connect.secrets.connect.FILE_DIR
6 | import io.lenses.connect.secrets.utils.EncodingAndId
7 | import org.apache.kafka.common.config.provider.ConfigProvider
8 | import org.apache.kafka.common.config.ConfigException
9 | import org.apache.kafka.common.config.ConfigTransformer
10 | import org.apache.kafka.connect.errors.ConnectException
11 | import org.scalatest.matchers.should.Matchers
12 | import org.scalatest.prop.TableDrivenPropertyChecks
13 | import org.scalatest.wordspec.AnyWordSpec
14 |
15 | import java.io.FileInputStream
16 | import java.nio.file.Files
17 | import java.util
18 | import java.util.Base64
19 | import java.util.UUID.randomUUID
20 | import scala.io.Source
21 | import scala.jdk.CollectionConverters._
22 | import scala.util.Random
23 | import scala.util.Success
24 | import scala.util.Using
25 |
26 | class Aes256DecodingProviderTest extends AnyWordSpec with TableDrivenPropertyChecks with Matchers {
27 | import AesDecodingTestHelper.encrypt
28 |
29 | "aes256 provider" should {
30 | "decrypt aes 256 utf-8 encoded value" in new TestContext with ConfiguredProvider {
31 | val encrypted = encrypt(value, key)
32 |
33 | forAll(Table("encoding", "", "utf-8")) { encoding =>
34 | val decrypted =
35 | provider.get(encoding, Set(encrypted).asJava).data().asScala
36 |
37 | decrypted.get(encrypted) shouldBe Some(value)
38 | }
39 | }
40 |
41 | "decrypt aes 256 base64 encoded value" in new TestContext with ConfiguredProvider {
42 | val encrypted =
43 | encrypt(Base64.getEncoder.encodeToString(value.getBytes()), key)
44 |
45 | val decrypted =
46 | provider.get("base64", Set(encrypted).asJava).data().asScala
47 |
48 | decrypted.get(encrypted) shouldBe Some(value)
49 | }
50 |
51 | "decrypt aes 256 encoded value stored in file with utf-8 encoding" in new TestContext with ConfiguredProvider {
52 | val encrypted = encrypt(value, key)
53 |
54 | val providerData = provider
55 | .get(s"utf8_file${EncodingAndId.Separator}id1", Set(encrypted).asJava)
56 | .data()
57 | .asScala
58 | val decryptedPath = providerData(encrypted)
59 |
60 | decryptedPath should startWith(s"$tmpDir/secrets/")
61 | decryptedPath.toLowerCase.contains(encrypted.toLowerCase) shouldBe false
62 | Using(Source.fromFile(decryptedPath))(_.getLines().mkString) shouldBe Success(
63 | value,
64 | )
65 | }
66 |
67 | "decrypt aes 256 encoded value stored in file with base64 encoding" in new TestContext with ConfiguredProvider {
68 | val bytesAmount = 100
69 | val bytesInput = Array.fill[Byte](bytesAmount)(0)
70 | Random.nextBytes(bytesInput)
71 | val encrypted = encrypt(Base64.getEncoder.encodeToString(bytesInput), key)
72 |
73 | val providerData = provider
74 | .get(
75 | s"${Encoding.BASE64_FILE}${EncodingAndId.Separator}fileId1",
76 | Set(encrypted).asJava,
77 | )
78 | .data()
79 | .asScala
80 | val decryptedPath = providerData(encrypted)
81 |
82 | decryptedPath should startWith(s"$tmpDir/secrets/")
83 | decryptedPath.toLowerCase.contains(encrypted.toLowerCase) shouldBe false
84 | val bytesConsumed = Array.fill[Byte](bytesAmount)(0)
85 | new FileInputStream(decryptedPath).read(bytesConsumed)
86 |
87 | bytesConsumed.toList shouldBe bytesInput.toList
88 | }
89 |
90 | "transform value referencing to the provider" in new TestContext {
91 | val value = "hi!"
92 | val encrypted = encrypt(value, key)
93 | provider.configure(config)
94 |
95 | val transformer = new ConfigTransformer(
96 | Map[String, ConfigProvider]("aes256" -> provider).asJava,
97 | )
98 | val props = Map("mykey" -> ("$" + s"{aes256::$encrypted}")).asJava
99 | val data = transformer.transform(props)
100 | data.data().containsKey(encrypted)
101 | data.data().get("mykey") shouldBe value
102 | }
103 | }
104 |
105 | forAll(encodings) { encoding =>
106 | s"aes256 provider for encoding $encoding" should {
107 | "fail decoding when unable to decode" in new TestContext {
108 | provider.configure(config)
109 |
110 | assertThrows[ConnectException] {
111 | provider.get(encoding, Set("abc").asJava)
112 | }
113 | }
114 |
115 | "fail decoding when secret key is not configured" in new TestContext {
116 | assertThrows[ConnectException] {
117 | provider.get(encoding, Set(encrypt("hi!", key)).asJava)
118 | }
119 | }
120 |
121 | "fail decoding when file dir is not configured" in new TestContext {
122 | assertThrows[ConnectException] {
123 | provider.get(encoding, Set(encrypt("hi!", key)).asJava)
124 | }
125 | }
126 | }
127 | }
128 |
129 | "configuring" should {
130 | "fail for invalid key" in new TestContext {
131 | assertThrows[ConfigException] {
132 | provider.configure(
133 | Map(
134 | Aes256ProviderConfig.SECRET_KEY -> "too-short",
135 | FILE_DIR -> "/tmp",
136 | ).asJava,
137 | )
138 | }
139 | }
140 |
141 | "fail for missing secret key" in new TestContext {
142 | assertThrows[ConfigException] {
143 | provider.configure(Map(FILE_DIR -> "/tmp").asJava)
144 | }
145 | }
146 | }
147 |
148 | trait ConfiguredProvider {
149 | self: TestContext =>
150 |
151 | val value: String = randomUUID().toString
152 |
153 | provider.configure(config)
154 | }
155 |
156 | trait TestContext {
157 | val key: String = randomUUID.toString.take(32)
158 | val tmpDir: String = Files
159 | .createTempDirectory(randomUUID().toString)
160 | .toFile
161 | .getAbsolutePath
162 | val provider = new Aes256DecodingProvider()
163 | val config: util.Map[String, String] = Map(
164 | Aes256ProviderConfig.SECRET_KEY -> key,
165 | FILE_DIR -> tmpDir,
166 | ).asJava
167 | }
168 |
169 | private def encodings = Table(
170 | "encoding",
171 | "",
172 | "utf8",
173 | "utf_file",
174 | "base64",
175 | "base64_file",
176 | )
177 | }
178 |
--------------------------------------------------------------------------------
/secret-provider/src/it/scala/io/lenses/connect/secrets/testcontainers/scalatest/SecretProviderContainerPerSuite.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.testcontainers.scalatest
2 |
3 | import cats.effect.IO
4 | import cats.effect.Resource
5 | import com.typesafe.scalalogging.LazyLogging
6 | import io.lenses.connect.secrets.testcontainers.connect.ConfigProviders
7 | import io.lenses.connect.secrets.testcontainers.connect.KafkaConnectClient
8 | import io.lenses.connect.secrets.testcontainers.KafkaConnectContainer
9 | import org.apache.kafka.clients.consumer.ConsumerConfig
10 | import org.apache.kafka.clients.consumer.ConsumerRecord
11 | import org.apache.kafka.clients.consumer.KafkaConsumer
12 | import org.apache.kafka.clients.producer.KafkaProducer
13 | import org.apache.kafka.clients.producer.ProducerConfig
14 | import org.apache.kafka.common.serialization.StringDeserializer
15 | import org.apache.kafka.common.serialization.StringSerializer
16 | import org.scalatest.BeforeAndAfterAll
17 | import org.scalatest.TestSuite
18 | import org.scalatest.concurrent.Eventually
19 | import org.scalatest.time.Minute
20 | import org.scalatest.time.Span
21 | import org.testcontainers.containers.KafkaContainer
22 | import org.testcontainers.containers.Network
23 | import org.testcontainers.containers.output.Slf4jLogConsumer
24 | import org.testcontainers.utility.DockerImageName
25 |
26 | import java.io.File
27 | import java.nio.file.Files
28 | import java.nio.file.Path
29 | import java.nio.file.Paths
30 | import java.time.Duration
31 | import java.util
32 | import java.util.Properties
33 | import java.util.UUID
34 | import java.util.stream.Collectors
35 | import scala.collection.mutable.ListBuffer
36 | import scala.util.Try
37 |
38 | trait SecretProviderContainerPerSuite extends BeforeAndAfterAll with Eventually with LazyLogging { this: TestSuite =>
39 |
40 | def getConfigProviders(): Option[ConfigProviders] = Option.empty
41 |
42 | override implicit def patienceConfig: PatienceConfig =
43 | PatienceConfig(timeout = Span(1, Minute))
44 |
45 | private val confluentPlatformVersion: String = {
46 | val (vers, from) = sys.env.get("CONFLUENT_VERSION") match {
47 | case Some(value) => (value, "env")
48 | case None => ("7.3.1", "default")
49 | }
50 | logger.info("Selected confluent version {} from {}", vers, from)
51 | vers
52 | }
53 |
54 | val network: Network = Network.SHARED
55 |
56 | def modules: Seq[String]
57 |
58 | lazy val kafkaContainer: KafkaContainer =
59 | new KafkaContainer(
60 | DockerImageName.parse(s"confluentinc/cp-kafka:$confluentPlatformVersion"),
61 | ).withNetwork(network)
62 | .withNetworkAliases("kafka")
63 | .withLogConsumer(new Slf4jLogConsumer(logger.underlying))
64 |
65 | lazy val kafkaConnectContainer: KafkaConnectContainer = {
66 | KafkaConnectContainer(
67 | kafkaContainer = kafkaContainer,
68 | connectPluginPathMap = connectPluginPath(),
69 | maybeConfigProviders = getConfigProviders(),
70 | ).withNetwork(network)
71 | .withLogConsumer(new Slf4jLogConsumer(logger.underlying))
72 | }
73 |
74 | implicit lazy val kafkaConnectClient: KafkaConnectClient =
75 | new KafkaConnectClient(kafkaConnectContainer)
76 |
77 | override def beforeAll(): Unit = {
78 | kafkaContainer.start()
79 | kafkaConnectContainer.start()
80 | super.beforeAll()
81 | }
82 |
83 | override def afterAll(): Unit =
84 | try super.afterAll()
85 | finally {
86 | kafkaConnectContainer.stop()
87 | kafkaContainer.stop()
88 | }
89 |
90 | def connectPluginPath(): Map[String, String] = {
91 |
92 | val dir: String = detectUserOrGithubDir
93 |
94 | modules.map { module: String =>
95 | val regex = s".*$module.*.jar"
96 | val files: util.List[Path] = Files
97 | .find(
98 | Paths.get(
99 | String.join(
100 | File.separator,
101 | dir,
102 | module,
103 | "target",
104 | ),
105 | ),
106 | 3,
107 | (p, _) => p.toFile.getName.matches(regex),
108 | )
109 | .collect(Collectors.toList())
110 | if (files.isEmpty)
111 | throw new RuntimeException(
112 | s"""Please run `sbt "project $module" assembly""",
113 | )
114 | module -> files.get(0).getParent.toString
115 | }.toMap
116 |
117 | }
118 |
119 | private def detectUserOrGithubDir = {
120 | val userDir = sys.props("user.dir")
121 | val githubWs = Try(Option(sys.env("GITHUB_WORKSPACE"))).toOption.flatten
122 |
123 | logger.info("userdir: {} githubWs: {}", userDir, githubWs)
124 |
125 | githubWs.getOrElse(userDir)
126 | }
127 |
128 | def makeStringStringProducerResource(): Resource[IO, KafkaProducer[String, String]] =
129 | makeProducerResource(classOf[StringSerializer], classOf[StringSerializer])
130 |
131 | def makeProducerResource[K, V](
132 | keySer: Class[_],
133 | valueSer: Class[_],
134 | ): Resource[IO, KafkaProducer[K, V]] = {
135 | val props = new Properties
136 | props.put(
137 | ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,
138 | kafkaContainer.getBootstrapServers,
139 | )
140 | props.put(ProducerConfig.ACKS_CONFIG, "all")
141 | props.put(ProducerConfig.RETRIES_CONFIG, 0)
142 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, keySer)
143 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSer)
144 | Resource.make(IO(new KafkaProducer[K, V](props)))(r => IO(r.close()))
145 | }
146 |
147 | def createConsumer(): KafkaConsumer[String, String] = {
148 | val props = new Properties
149 | props.put(
150 | ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
151 | kafkaContainer.getBootstrapServers,
152 | )
153 | props.put(ConsumerConfig.GROUP_ID_CONFIG, "cg-" + UUID.randomUUID())
154 | props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
155 | new KafkaConsumer[String, String](
156 | props,
157 | new StringDeserializer(),
158 | new StringDeserializer(),
159 | )
160 | }
161 |
162 | /**
163 | * Drain a kafka topic.
164 | *
165 | * @param consumer the kafka consumer
166 | * @param expectedRecordCount the expected record count
167 | * @tparam K the key type
168 | * @tparam V the value type
169 | * @return the records
170 | */
171 | def drain[K, V](
172 | consumer: KafkaConsumer[K, V],
173 | expectedRecordCount: Int,
174 | ): List[ConsumerRecord[K, V]] = {
175 | val allRecords = ListBuffer[ConsumerRecord[K, V]]()
176 |
177 | eventually {
178 | consumer
179 | .poll(Duration.ofMillis(50))
180 | .iterator()
181 | .forEachRemaining(e => allRecords :+ e)
182 | assert(allRecords.size == expectedRecordCount)
183 | }
184 | allRecords.toList
185 | }
186 | }
187 |
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/AWSHelper.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.providers
8 | import com.fasterxml.jackson.core.JsonParseException
9 | import com.fasterxml.jackson.databind.ObjectMapper
10 | import com.typesafe.scalalogging.LazyLogging
11 | import com.typesafe.scalalogging.StrictLogging
12 | import io.lenses.connect.secrets.cache.Ttl
13 | import io.lenses.connect.secrets.cache.ValueWithTtl
14 | import io.lenses.connect.secrets.config.AWSProviderSettings
15 | import io.lenses.connect.secrets.config.SecretType
16 | import io.lenses.connect.secrets.config.SecretType.SecretType
17 | import io.lenses.connect.secrets.connect.AuthMode
18 | import io.lenses.connect.secrets.connect.decodeKey
19 | import io.lenses.connect.secrets.io.FileWriter
20 | import io.lenses.connect.secrets.utils.EncodingAndId
21 | import org.apache.kafka.connect.errors.ConnectException
22 | import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
23 | import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
24 | import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
25 | import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
26 | import software.amazon.awssdk.regions.Region
27 | import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient
28 | import software.amazon.awssdk.services.secretsmanager.model.DescribeSecretRequest
29 | import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest
30 |
31 | import java.net.URI
32 | import java.time.Clock
33 | import java.time.Duration
34 | import java.time.OffsetDateTime
35 | import java.time.ZoneId
36 | import java.time.temporal.ChronoUnit
37 | import java.util
38 | import scala.jdk.CollectionConverters.MapHasAsScala
39 | import scala.util.Failure
40 | import scala.util.Success
41 | import scala.util.Try
42 |
43 | class AWSHelper(
44 | client: SecretsManagerClient,
45 | defaultTtl: Option[Duration],
46 | fileWriterCreateFn: () => Option[FileWriter],
47 | secretType: SecretType,
48 | )(
49 | implicit
50 | clock: Clock,
51 | ) extends SecretHelper
52 | with LazyLogging {
53 |
54 | private val objectMapper = new ObjectMapper()
55 |
56 | // get the key value and ttl in the specified secret
57 | override def lookup(secretId: String): Either[Throwable, ValueWithTtl[Map[String, String]]] =
58 | for {
59 | secretTtl <- getTTL(secretId)
60 | stringSecretValue <- getStringSecretValue(secretId)
61 | secretValue <- secretType match {
62 | case SecretType.STRING =>
63 | Try(ValueWithTtl[Map[String, String]](secretTtl, Map("value" -> stringSecretValue))).toEither
64 | case _ =>
65 | for {
66 | mapSecretValue <- secretValueToMap(secretId, stringSecretValue)
67 | parsedSecretValue <- parseSecretValue(mapSecretValue)
68 | } yield ValueWithTtl[Map[String, String]](secretTtl, parsedSecretValue)
69 | }
70 | } yield secretValue
71 |
72 | // determine the ttl for the secret
73 | private def getTTL(
74 | secretId: String,
75 | )(
76 | implicit
77 | clock: Clock,
78 | ): Either[Throwable, Option[Ttl]] = {
79 |
80 | // describe to get the ttl
81 | val descRequest: DescribeSecretRequest =
82 | DescribeSecretRequest.builder().secretId(secretId).build()
83 |
84 | for {
85 | secretResponse <- Try(client.describeSecret(descRequest)).toEither
86 | } yield {
87 | if (secretResponse.rotationEnabled()) {
88 | //val lastRotation = secretResponse.lastRotatedDate()
89 | val ttlExpires = secretResponse.nextRotationDate()
90 | val rotationDuration = Duration.of(secretResponse.rotationRules().automaticallyAfterDays(), ChronoUnit.DAYS)
91 | Some(Ttl(
92 | rotationDuration,
93 | OffsetDateTime.ofInstant(ttlExpires, ZoneId.systemDefault()),
94 | ))
95 | } else {
96 | Ttl(Option.empty, defaultTtl)
97 | }
98 | }
99 |
100 | }
101 |
102 | private def parseSecretValue(
103 | secretValues: Map[String, String],
104 | ): Either[Throwable, Map[String, String]] = {
105 | val fileWriterMaybe = fileWriterCreateFn()
106 | Try(
107 | secretValues.map {
108 | case (k, v) =>
109 | (k,
110 | decodeKey(
111 | key = k,
112 | value = v,
113 | encoding = EncodingAndId.from(k).encoding,
114 | writeFileFn = content => {
115 | fileWriterMaybe.fold("nofile")(_.write(k.toLowerCase, content, k).toString)
116 | },
117 | ),
118 | )
119 | },
120 | ).toEither
121 | }
122 |
123 | private def getStringSecretValue(
124 | secretId: String,
125 | ): Either[Throwable, String] = {
126 | for {
127 | secretValueResult <- Try(
128 | client.getSecretValue(GetSecretValueRequest.builder().secretId(secretId).build()),
129 | )
130 | secretString <- Try(secretValueResult.secretString())
131 | } yield secretString
132 | }.toEither
133 |
134 | private def secretValueToMap(
135 | secretId: String,
136 | secretValueString: String,
137 | ): Either[Throwable, Map[String, String]] = {
138 | val a = for {
139 | mapFromResponse <-
140 | Try(
141 | objectMapper
142 | .readValue(
143 | secretValueString,
144 | classOf[util.HashMap[String, String]],
145 | )
146 | .asScala.toMap,
147 | )
148 | } yield mapFromResponse
149 | a match {
150 | case Failure(_: JsonParseException) =>
151 | Left(new IllegalStateException(s"Unable to parse JSON in secret [$secretId}]"))
152 | case Failure(exception) => Left(exception)
153 | case Success(value) => Right(value)
154 | }
155 |
156 | }
157 | }
158 |
159 | object AWSHelper extends StrictLogging {
160 |
161 | // initialize the AWS client based on the auth mode
162 | def createClient(settings: AWSProviderSettings): SecretsManagerClient = {
163 |
164 | logger.info(
165 | s"Initializing client with mode [${settings.authMode}]",
166 | )
167 |
168 | val credentialProvider: AwsCredentialsProvider = settings.authMode match {
169 | case AuthMode.CREDENTIALS =>
170 | settings.credentials.map {
171 | creds =>
172 | StaticCredentialsProvider.create(
173 | AwsBasicCredentials.create(
174 | creds.accessKey,
175 | creds.secretKey.value(),
176 | ),
177 | )
178 | }.getOrElse(throw new ConnectException(
179 | "No access key and secret key credentials available for CREDENTIALS mode",
180 | ))
181 |
182 | case _ =>
183 | DefaultCredentialsProvider.create()
184 | }
185 |
186 | val builder = SecretsManagerClient.builder()
187 | .credentialsProvider(credentialProvider)
188 | .region(Region.of(settings.region))
189 |
190 | settings.endpointOverride.foreach(eo => builder.endpointOverride(URI.create(eo)))
191 |
192 | builder.build()
193 | }
194 | }
195 |
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/providers/VaultHelper.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.providers
8 |
9 | import io.github.jopenlibs.vault.SslConfig
10 | import io.github.jopenlibs.vault.Vault
11 | import io.github.jopenlibs.vault.VaultConfig
12 | import io.github.jopenlibs.vault.response.LogicalResponse
13 | import com.typesafe.scalalogging.LazyLogging
14 | import com.typesafe.scalalogging.StrictLogging
15 | import io.lenses.connect.secrets.cache.ValueWithTtl
16 | import io.lenses.connect.secrets.config.VaultAuthMethod
17 | import io.lenses.connect.secrets.config.VaultSettings
18 | import io.lenses.connect.secrets.connect.decodeKey
19 | import io.lenses.connect.secrets.io.FileWriter
20 | import io.lenses.connect.secrets.utils.EncodingAndId
21 | import io.lenses.connect.secrets.utils.ExceptionUtils.failWithEx
22 | import org.apache.kafka.connect.errors.ConnectException
23 |
24 | import java.io.File
25 | import java.time.Clock
26 | import java.time.Duration
27 | import java.time.temporal.ChronoUnit
28 | import scala.jdk.CollectionConverters.MapHasAsScala
29 | import scala.util.Try
30 |
31 | class VaultHelper(
32 | vaultClient: Vault,
33 | defaultTtl: Option[Duration],
34 | fileWriterCreateFn: () => Option[FileWriter],
35 | )(
36 | implicit
37 | clock: Clock,
38 | ) extends SecretHelper
39 | with LazyLogging {
40 |
41 | override def lookup(path: String): Either[Throwable, ValueWithTtl[Map[String, String]]] = {
42 | logger.debug(s"Looking up value from Vault at [$path]")
43 | val lookupFn = if (path.startsWith("database/creds/")) lookupFromDatabaseEngine _ else lookupFromVault _
44 | lookupFn(path) match {
45 | case Left(ex) =>
46 | failWithEx(s"Failed to fetch secrets from path [$path]", ex)
47 | case Right(response) if response.getRestResponse.getStatus != 200 =>
48 | failWithEx(
49 | s"No secrets found at path [$path]. Vault response: ${new String(response.getRestResponse.getBody)}",
50 | )
51 | case Right(response) if response.getData.isEmpty =>
52 | failWithEx(s"No secrets found at path [$path]")
53 | case Right(response) =>
54 | val ttl =
55 | Option(response.getLeaseDuration).filterNot(_ == 0L).map(Duration.of(_, ChronoUnit.SECONDS))
56 | Right(
57 | ValueWithTtl(ttl, defaultTtl, parseSuccessfulResponse(response)),
58 | )
59 | }
60 | }
61 |
62 | private def lookupFromVault(path: String): Either[Throwable, LogicalResponse] =
63 | Try(vaultClient.logical().read(path)).toEither
64 |
65 | private def lookupFromDatabaseEngine(path: String): Either[Throwable, LogicalResponse] = {
66 | val parts = path.split("/")
67 | Either.cond(
68 | parts.length == 3,
69 | vaultClient.database().creds(parts(2)),
70 | new ConnectException("Database path is invalid. Path must be in the form 'database/creds/'"),
71 | )
72 | }
73 |
74 | private def parseSuccessfulResponse(
75 | response: LogicalResponse,
76 | ) = {
77 | val secretValues = response.getData.asScala
78 | val fileWriterMaybe = fileWriterCreateFn()
79 | secretValues.map {
80 | case (k, v) =>
81 | (k,
82 | decodeKey(
83 | encoding = EncodingAndId.from(k).encoding,
84 | key = k,
85 | value = v,
86 | writeFileFn = { content =>
87 | fileWriterMaybe.fold("nofile")(_.write(k.toLowerCase, content, k).toString)
88 | },
89 | ),
90 | )
91 | }.toMap
92 | }
93 | }
94 |
95 | object VaultHelper extends StrictLogging {
96 |
97 | // initialize the vault client
98 | def createClient(settings: VaultSettings): Vault = {
99 | val config =
100 | new VaultConfig().address(settings.addr)
101 |
102 | // set ssl if configured
103 | config.sslConfig(configureSSL(settings))
104 |
105 | if (settings.namespace.nonEmpty) {
106 | logger.info(s"Setting namespace to ${settings.namespace}")
107 | config.nameSpace(settings.namespace)
108 | }
109 |
110 | logger.info(s"Setting engine version to ${settings.engineVersion}")
111 | config.engineVersion(settings.engineVersion)
112 |
113 | logger.info(s"Setting prefix path depth to ${settings.prefixDepth}")
114 | config.prefixPathDepth(settings.prefixDepth)
115 |
116 | val vault = new Vault(config.build())
117 |
118 | logger.info(
119 | s"Initializing client with mode [${settings.authMode.toString}]",
120 | )
121 |
122 | val token = settings.authMode match {
123 | case VaultAuthMethod.USERPASS =>
124 | settings.userPass
125 | .map(up =>
126 | vault
127 | .auth()
128 | .loginByUserPass(up.username, up.password.value(), up.mount)
129 | .getAuthClientToken,
130 | )
131 |
132 | case VaultAuthMethod.APPROLE =>
133 | settings.appRole
134 | .map(ar =>
135 | vault
136 | .auth()
137 | .loginByAppRole(ar.path, ar.role, ar.secretId.value())
138 | .getAuthClientToken,
139 | )
140 |
141 | case VaultAuthMethod.CERT =>
142 | settings.cert
143 | .map(c => vault.auth().loginByCert(c.mount).getAuthClientToken)
144 |
145 | case VaultAuthMethod.AWSIAM =>
146 | settings.awsIam
147 | .map(aws =>
148 | vault
149 | .auth()
150 | .loginByAwsIam(
151 | aws.role,
152 | aws.url,
153 | aws.body.value(),
154 | aws.headers.value(),
155 | aws.mount,
156 | )
157 | .getAuthClientToken,
158 | )
159 |
160 | case VaultAuthMethod.KUBERNETES =>
161 | settings.k8s
162 | .map(k8s =>
163 | vault
164 | .auth()
165 | .loginByJwt("kubernetes", k8s.role, k8s.jwt.value(), k8s.authPath)
166 | .getAuthClientToken,
167 | )
168 | case VaultAuthMethod.GCP =>
169 | settings.gcp
170 | .map(gcp =>
171 | vault
172 | .auth()
173 | .loginByGCP(gcp.role, gcp.jwt.value())
174 | .getAuthClientToken,
175 | )
176 |
177 | case VaultAuthMethod.LDAP =>
178 | settings.ldap
179 | .map(l =>
180 | vault
181 | .auth()
182 | .loginByLDAP(l.username, l.password.value(), l.mount)
183 | .getAuthClientToken,
184 | )
185 |
186 | case VaultAuthMethod.JWT =>
187 | settings.jwt
188 | .map(j =>
189 | vault
190 | .auth()
191 | .loginByJwt(j.provider, j.role, j.jwt.value())
192 | .getAuthClientToken,
193 | )
194 |
195 | case VaultAuthMethod.TOKEN =>
196 | Some(settings.token.value())
197 |
198 | case VaultAuthMethod.GITHUB =>
199 | settings.github
200 | .map(gh =>
201 | vault
202 | .auth()
203 | .loginByGithub(gh.token.value(), gh.mount)
204 | .getAuthClientToken,
205 | )
206 |
207 | case _ =>
208 | throw new ConnectException(
209 | s"Unsupported auth method [${settings.authMode.toString}]",
210 | )
211 | }
212 |
213 | config.token(token.get)
214 | config.build()
215 | new Vault(config)
216 | }
217 |
218 | // set up tls
219 | private def configureSSL(settings: VaultSettings): SslConfig = {
220 | val ssl = new SslConfig()
221 |
222 | if (settings.keystoreLoc != "") {
223 | logger.info(s"Configuring keystore at [${settings.keystoreLoc}]")
224 | ssl.keyStoreFile(
225 | new File(settings.keystoreLoc),
226 | settings.keystorePass.value(),
227 | )
228 | }
229 |
230 | if (settings.truststoreLoc != "") {
231 | logger.info(s"Configuring keystore at [${settings.truststoreLoc}]")
232 | ssl.trustStoreFile(new File(settings.truststoreLoc))
233 | }
234 |
235 | if (settings.clientPem != "") {
236 | logger.info(s"Configuring client PEM. Ignored if JKS set.")
237 | ssl.clientKeyPemFile(new File(settings.clientPem))
238 | }
239 |
240 | if (settings.pem != "") {
241 | logger.info(s"Configuring Vault Server PEM. Ignored if JKS set.")
242 | ssl.pemFile(new File(settings.pem))
243 | }
244 |
245 | ssl.build()
246 | }
247 | }
248 |
--------------------------------------------------------------------------------
/project/Settings.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017-2020 Lenses.io Ltd
3 | */
4 |
5 | import com.eed3si9n.jarjarabrams.ShadeRule
6 | import sbt.*
7 | import sbt.Keys.*
8 | import sbt.internal.util.ManagedLogger
9 | import sbtassembly.Assembly.JarEntry
10 | import sbtassembly.AssemblyKeys.*
11 | import sbtassembly.CustomMergeStrategy
12 | import sbtassembly.MergeStrategy
13 | import sbtassembly.PathList
14 |
15 | import java.io.*
16 | import java.net.*
17 | import scala.util.Try
18 | import java.nio.file.Files
19 | import java.nio.file.Paths
20 |
21 | object Settings extends Dependencies {
22 |
23 | val scala213 = "2.13.10"
24 | val scala3 = "3.2.2"
25 |
26 | val nextVersion = "2.3.1"
27 | val artifactVersion: String = {
28 | sys.env.get("LENSES_TAG_NAME") match {
29 | case Some(tag) => tag
30 | case _ => s"$nextVersion-SNAPSHOT"
31 | }
32 | }
33 |
34 | object ScalacFlags {
35 | val FatalWarnings212 = "-Xfatal-warnings"
36 | val FatalWarnings213 = "-Werror"
37 | val WarnUnusedImports212 = "-Ywarn-unused-import"
38 | val WarnUnusedImports213 = "-Ywarn-unused:imports"
39 | }
40 |
41 | val modulesSettings: Seq[Setting[_]] = Seq(
42 | organization := "io.lenses",
43 | version := artifactVersion,
44 | scalaOrganization := "org.scala-lang",
45 | resolvers ++= Seq(
46 | Resolver.mavenLocal,
47 | Resolver.mavenCentral,
48 | ),
49 | crossScalaVersions := List( /*scala3, */ scala213),
50 | Compile / scalacOptions ++= Seq(
51 | "-release:11",
52 | "-encoding",
53 | "utf8",
54 | "-deprecation",
55 | "-unchecked",
56 | "-feature",
57 | "11",
58 | ),
59 | Compile / scalacOptions ++= {
60 | Seq(
61 | ScalacFlags.WarnUnusedImports213,
62 | )
63 | },
64 | Compile / console / scalacOptions --= Seq(
65 | ScalacFlags.FatalWarnings212,
66 | ScalacFlags.FatalWarnings213,
67 | ScalacFlags.WarnUnusedImports212,
68 | ScalacFlags.WarnUnusedImports213,
69 | ),
70 | // TODO remove for cross build
71 | scalaVersion := scala213,
72 | Test / fork := true,
73 | )
74 |
75 | implicit final class AssemblyConfigurator(project: Project) {
76 |
77 | /*
78 | Exclude the jar signing files from dependencies. They come from other jars,
79 | and including these would break assembly (due to duplicates) or classloading
80 | (due to mismatching jar checksum/signature).
81 | */
82 | val excludeFilePatterns = Set(".MF", ".RSA", ".DSA", ".SF")
83 |
84 | def excludeFileFilter(p: String): Boolean =
85 | excludeFilePatterns.exists(p.endsWith)
86 |
87 | val excludePatterns = Set(
88 | "kafka-client",
89 | "kafka-connect-json",
90 | "hadoop-yarn",
91 | "org.apache.kafka",
92 | "zookeeper",
93 | "log4j",
94 | "junit",
95 | )
96 |
97 | def configureAssembly(): Project =
98 | project.settings(
99 | Seq(
100 | assembly / test := {},
101 | assembly / assemblyOutputPath := {
102 | createLibsDir((Compile / target).value)
103 | file(
104 | target.value + "/libs/" + (assembly / assemblyJarName).value,
105 | )
106 | },
107 | assembly / assemblyExcludedJars := {
108 | val cp: Classpath = (assembly / fullClasspath).value
109 | cp filter { f =>
110 | excludePatterns
111 | .exists(f.data.getName.contains) && (!f.data.getName
112 | .contains("slf4j"))
113 | }
114 | },
115 | assembly / assemblyMergeStrategy := {
116 | case PathList("META-INF", "maven", _ @_*) =>
117 | CustomMergeStrategy("keep-only-fresh-maven-descriptors", 1) {
118 | assemblyDependency =>
119 | val keepDeps = assemblyDependency.collect {
120 | case dependency @ (_: sbtassembly.Assembly.Project) =>
121 | JarEntry(dependency.target, dependency.stream)
122 | }
123 | Right(keepDeps)
124 | }
125 | case PathList("META-INF", "maven", _ @_*) =>
126 | MergeStrategy.discard
127 | case PathList("META-INF", "MANIFEST.MF") =>
128 | MergeStrategy.discard
129 | case p if excludeFileFilter(p) =>
130 | MergeStrategy.discard
131 | case PathList(ps @ _*) if ps.last == "module-info.class" =>
132 | MergeStrategy.discard
133 | case _ => MergeStrategy.first
134 | },
135 | assembly / assemblyShadeRules ++= Seq(
136 | ShadeRule.rename("io.confluent.**" -> "lshaded.confluent.@1").inAll,
137 | ShadeRule.rename("com.fasterxml.**" -> "lshaded.fasterxml.@1").inAll,
138 | ),
139 | ),
140 | )
141 |
142 | def createLibsDir(targetDir: File): Unit = {
143 |
144 | val newDirectory = targetDir / "libs"
145 |
146 | if (!Files.exists(Paths.get(newDirectory.toString))) {
147 | Files.createDirectories(Paths.get(newDirectory.toString))
148 | }
149 | }
150 |
151 | }
152 |
153 | lazy val IntegrationTest = config("it") extend Test
154 |
155 | implicit final class MavenDescriptorConfigurator(project: Project) {
156 |
157 | val generateMetaInfMaven = taskKey[Unit]("Generate META-INF/maven directory")
158 |
159 | def configureMavenDescriptor(): Project =
160 | project
161 | .settings(
162 | generateMetaInfMaven := {
163 | val log = streams.value.log
164 |
165 | val targetDirBase = (Compile / crossTarget).value / "classes" / "META-INF" / "maven"
166 |
167 | val allModuleIds: Map[ModuleID, String] = update
168 | .value
169 | .configuration(Compile)
170 | .toVector
171 | .flatMap(_.modules)
172 | .map {
173 | e: ModuleReport => e.module -> e.artifacts.headOption
174 | }
175 | .collect {
176 | case (moduleId, Some((moduleJar, moduleFile))) =>
177 | moduleId ->
178 | moduleJar.url.get.toString
179 | .reverse
180 | .replaceFirst(".jar".reverse, ".pom".reverse)
181 | .reverse
182 | }.toMap
183 |
184 | for ((moduleId, pomUrl) <- allModuleIds) {
185 |
186 | log.info(s"Processing ${moduleId.name}")
187 |
188 | val groupId = moduleId.organization
189 | val artifactId = moduleId.name
190 | val version = moduleId.revision
191 | val targetDir = targetDirBase / groupId / artifactId
192 | targetDir.mkdirs()
193 |
194 | val propertiesFileChanged = createPomPropertiesIfChanged(groupId, artifactId, version, targetDir)
195 | if (propertiesFileChanged) {
196 | createPomXml(log, targetDir, pomUrl)
197 | }
198 | }
199 |
200 | },
201 | (Compile / compile) := ((Compile / compile) dependsOn generateMetaInfMaven).value,
202 | )
203 |
204 | private def createPomXml(log: ManagedLogger, targetDir: File, pomUrl: String): Option[File] = {
205 | val pomFile = targetDir / "pom.xml"
206 |
207 | try {
208 | val url = new URL(pomUrl)
209 | val connection = url.openConnection().asInstanceOf[HttpURLConnection]
210 | connection.setRequestMethod("GET")
211 |
212 | if (connection.getResponseCode == HttpURLConnection.HTTP_OK && connection.getContentType == "text/xml") {
213 | val inputStream = connection.getInputStream
214 | try {
215 | val pomContent = new String(inputStream.readAllBytes())
216 | IO.write(pomFile, pomContent)
217 | log.info(s"Successfully retrieved and saved POM from $pomUrl to $pomFile")
218 | Some(pomFile)
219 |
220 | } finally {
221 | inputStream.close()
222 | }
223 | } else {
224 | log.error(
225 | s"Failed to retrieve POM from $pomUrl. HTTP Status: ${connection.getResponseCode}, Content Type: ${connection.getContentType}",
226 | )
227 | Option.empty
228 | }
229 | } catch {
230 | case e: MalformedURLException =>
231 | log.error(s"Invalid URL: $pomUrl")
232 | Option.empty
233 | case e: IOException =>
234 | log.error(s"Error while retrieving POM from $pomUrl: ${e.getMessage}")
235 | Option.empty
236 | }
237 |
238 | }
239 |
240 | private def createPomPropertiesIfChanged(
241 | groupId: String,
242 | artifactId: String,
243 | version: String,
244 | targetDir: File,
245 | ): Boolean = {
246 | val propertiesFile = targetDir / "pom.properties"
247 | val propertiesContent =
248 | s"""version=$version
249 | |groupId=$groupId
250 | |artifactId=$artifactId
251 | """.stripMargin
252 |
253 | val alreadyExists = Try(IO.read(propertiesFile))
254 | .toOption
255 | .contains(propertiesContent)
256 |
257 | if (!alreadyExists) {
258 | IO.write(propertiesFile, propertiesContent)
259 | }
260 |
261 | !alreadyExists
262 |
263 | }
264 | }
265 | }
266 |
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/config/VaultProviderSettings.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.config
8 |
9 | import com.typesafe.scalalogging.StrictLogging
10 | import io.lenses.connect.secrets.config.AbstractConfigExtensions._
11 | import io.lenses.connect.secrets.config.VaultAuthMethod.VaultAuthMethod
12 | import io.lenses.connect.secrets.config.VaultProviderConfig.TOKEN_RENEWAL
13 | import io.lenses.connect.secrets.connect._
14 | import org.apache.kafka.common.config.types.Password
15 | import org.apache.kafka.connect.errors.ConnectException
16 |
17 | import java.time.Duration
18 | import java.time.temporal.ChronoUnit
19 | import scala.concurrent.duration.DurationInt
20 | import scala.concurrent.duration.FiniteDuration
21 | import scala.io.Source
22 | import scala.util.Failure
23 | import scala.util.Success
24 | import scala.util.Using
25 |
26 | case class AwsIam(
27 | role: String,
28 | url: String,
29 | headers: Password,
30 | body: Password,
31 | mount: String,
32 | )
33 | case class Gcp(role: String, jwt: Password)
34 | case class Jwt(role: String, provider: String, jwt: Password)
35 | case class UserPass(username: String, password: Password, mount: String)
36 | case class Ldap(username: String, password: Password, mount: String)
37 | case class AppRole(path: String, role: String, secretId: Password)
38 | case class K8s(role: String, jwt: Password, authPath: String)
39 | case class Cert(mount: String)
40 | case class Github(token: Password, mount: String)
41 |
42 | case class VaultSettings(
43 | addr: String,
44 | namespace: String,
45 | token: Password,
46 | authMode: VaultAuthMethod,
47 | keystoreLoc: String,
48 | keystorePass: Password,
49 | truststoreLoc: String,
50 | pem: String,
51 | clientPem: String,
52 | engineVersion: Int = 2,
53 | prefixDepth: Int = 1,
54 | appRole: Option[AppRole],
55 | awsIam: Option[AwsIam],
56 | gcp: Option[Gcp],
57 | jwt: Option[Jwt],
58 | userPass: Option[UserPass],
59 | ldap: Option[Ldap],
60 | k8s: Option[K8s],
61 | cert: Option[Cert],
62 | github: Option[Github],
63 | tokenRenewal: FiniteDuration,
64 | fileWriterOpts: Option[FileWriterOptions],
65 | defaultTtl: Option[Duration],
66 | )
67 |
68 | object VaultSettings extends StrictLogging {
69 | def apply(config: VaultProviderConfig): VaultSettings = {
70 | val addr = config.getString(VaultProviderConfig.VAULT_ADDR)
71 | val token = config.getPassword(VaultProviderConfig.VAULT_TOKEN)
72 | val namespace = config.getString(VaultProviderConfig.VAULT_NAMESPACE)
73 | val keystoreLoc = config.getString(VaultProviderConfig.VAULT_KEYSTORE_LOC)
74 | val keystorePass =
75 | config.getPassword(VaultProviderConfig.VAULT_KEYSTORE_PASS)
76 | val truststoreLoc =
77 | config.getString(VaultProviderConfig.VAULT_TRUSTSTORE_LOC)
78 | val pem = config.getString(VaultProviderConfig.VAULT_PEM)
79 | val clientPem = config.getString(VaultProviderConfig.VAULT_CLIENT_PEM)
80 | val engineVersion = config.getInt(VaultProviderConfig.VAULT_ENGINE_VERSION)
81 | val prefixDepth = config.getInt(VaultProviderConfig.VAULT_PREFIX_DEPTH)
82 |
83 | val authMode = VaultAuthMethod.withNameOpt(
84 | config.getString(VaultProviderConfig.AUTH_METHOD).toUpperCase,
85 | ) match {
86 | case Some(auth) => auth
87 | case None =>
88 | throw new ConnectException(
89 | s"Unsupported ${VaultProviderConfig.AUTH_METHOD}",
90 | )
91 | }
92 |
93 | val awsIam =
94 | if (authMode.equals(VaultAuthMethod.AWSIAM)) Some(getAWS(config))
95 | else None
96 | val gcp =
97 | if (authMode.equals(VaultAuthMethod.GCP)) Some(getGCP(config)) else None
98 | val appRole =
99 | if (authMode.equals(VaultAuthMethod.APPROLE)) Some(getAppRole(config))
100 | else None
101 | val jwt =
102 | if (authMode.equals(VaultAuthMethod.JWT)) Some(getJWT(config)) else None
103 | val k8s =
104 | if (authMode.equals(VaultAuthMethod.KUBERNETES)) Some(getK8s(config))
105 | else None
106 | val userpass =
107 | if (authMode.equals(VaultAuthMethod.USERPASS)) Some(getUserPass(config))
108 | else None
109 | val ldap =
110 | if (authMode.equals(VaultAuthMethod.LDAP)) Some(getLDAP(config)) else None
111 | val cert =
112 | if (authMode.equals(VaultAuthMethod.CERT)) Some(getCert(config)) else None
113 | val github =
114 | if (authMode.equals(VaultAuthMethod.GITHUB)) Some(getGitHub(config))
115 | else None
116 |
117 | val tokenRenewal = config.getInt(TOKEN_RENEWAL).toInt.milliseconds
118 | VaultSettings(
119 | addr = addr,
120 | namespace = namespace,
121 | token = token,
122 | authMode = authMode,
123 | keystoreLoc = keystoreLoc,
124 | keystorePass = keystorePass,
125 | truststoreLoc = truststoreLoc,
126 | pem = pem,
127 | clientPem = clientPem,
128 | engineVersion = engineVersion,
129 | prefixDepth = prefixDepth,
130 | appRole = appRole,
131 | awsIam = awsIam,
132 | gcp = gcp,
133 | jwt = jwt,
134 | userPass = userpass,
135 | ldap = ldap,
136 | k8s = k8s,
137 | cert = cert,
138 | github = github,
139 | tokenRenewal = tokenRenewal,
140 | fileWriterOpts = FileWriterOptions(config),
141 | defaultTtl =
142 | Option(config.getLong(SECRET_DEFAULT_TTL).toLong).filterNot(_ == 0L).map(Duration.of(_, ChronoUnit.MILLIS)),
143 | )
144 | }
145 |
146 | def getCert(config: VaultProviderConfig): Cert =
147 | Cert(config.getString(VaultProviderConfig.CERT_MOUNT))
148 |
149 | def getGitHub(config: VaultProviderConfig): Github = {
150 | val token =
151 | config.getPasswordOrThrowOnNull(VaultProviderConfig.GITHUB_TOKEN)
152 | val mount = config.getStringOrThrowOnNull(VaultProviderConfig.GITHUB_MOUNT)
153 | Github(token = token, mount = mount)
154 | }
155 |
156 | def getAWS(config: VaultProviderConfig): AwsIam = {
157 | val role = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_ROLE)
158 | val url = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_URL)
159 | val headers =
160 | config.getPasswordOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_HEADERS)
161 | val body =
162 | config.getPasswordOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_BODY)
163 | val mount = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_MOUNT)
164 | AwsIam(
165 | role = role,
166 | url = url,
167 | headers = headers,
168 | body = body,
169 | mount = mount,
170 | )
171 | }
172 |
173 | def getAppRole(config: VaultProviderConfig): AppRole = {
174 | val path = config.getStringOrThrowOnNull(VaultProviderConfig.APP_ROLE_PATH)
175 | val role = config.getStringOrThrowOnNull(VaultProviderConfig.APP_ROLE)
176 | val secretId =
177 | config.getPasswordOrThrowOnNull(VaultProviderConfig.APP_ROLE_SECRET_ID)
178 | AppRole(path = path, role = role, secretId = secretId)
179 | }
180 |
181 | def getK8s(config: VaultProviderConfig): K8s = {
182 | val role =
183 | config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_ROLE)
184 | val path =
185 | config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_TOKEN_PATH)
186 | val authPath = config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_AUTH_PATH)
187 | Using(Source.fromFile(path))(_.getLines().mkString) match {
188 | case Failure(exception) =>
189 | throw new ConnectException(
190 | s"Failed to load kubernetes token file [$path]",
191 | exception,
192 | )
193 | case Success(fileContents) =>
194 | K8s(role = role, jwt = new Password(fileContents), authPath)
195 | }
196 | }
197 |
198 | def getUserPass(config: VaultProviderConfig): UserPass = {
199 | val user = config.getStringOrThrowOnNull(VaultProviderConfig.USERNAME)
200 | val pass = config.getPasswordOrThrowOnNull(VaultProviderConfig.PASSWORD)
201 | val mount = config.getStringOrThrowOnNull(VaultProviderConfig.UP_MOUNT)
202 | UserPass(username = user, password = pass, mount = mount)
203 | }
204 |
205 | def getLDAP(config: VaultProviderConfig): Ldap = {
206 | val user = config.getStringOrThrowOnNull(VaultProviderConfig.LDAP_USERNAME)
207 | val pass =
208 | config.getPasswordOrThrowOnNull(VaultProviderConfig.LDAP_PASSWORD)
209 | val mount = config.getStringOrThrowOnNull(VaultProviderConfig.LDAP_MOUNT)
210 | Ldap(username = user, password = pass, mount = mount)
211 | }
212 |
213 | def getGCP(config: VaultProviderConfig): Gcp = {
214 | val role = config.getStringOrThrowOnNull(VaultProviderConfig.GCP_ROLE)
215 | val jwt = config.getPasswordOrThrowOnNull(VaultProviderConfig.GCP_JWT)
216 | Gcp(role = role, jwt = jwt)
217 | }
218 |
219 | def getJWT(config: VaultProviderConfig): Jwt = {
220 | val role = config.getStringOrThrowOnNull(VaultProviderConfig.JWT_ROLE)
221 | val provider =
222 | config.getStringOrThrowOnNull(VaultProviderConfig.JWT_PROVIDER)
223 | val jwt = config.getPasswordOrThrowOnNull(VaultProviderConfig.JWT)
224 | Jwt(role = role, provider = provider, jwt = jwt)
225 | }
226 | }
227 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
--------------------------------------------------------------------------------
/secret-provider/src/main/scala/io/lenses/connect/secrets/config/VaultProviderConfig.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * * Copyright 2017-2020 Lenses.io Ltd
4 | *
5 | */
6 |
7 | package io.lenses.connect.secrets.config
8 |
9 | import io.lenses.connect.secrets.connect.FILE_DIR
10 | import io.lenses.connect.secrets.connect.FILE_DIR_DESC
11 | import io.lenses.connect.secrets.connect.SECRET_DEFAULT_TTL
12 | import io.lenses.connect.secrets.connect.SECRET_DEFAULT_TTL_DEFAULT
13 | import io.lenses.connect.secrets.connect.WRITE_FILES
14 | import io.lenses.connect.secrets.connect.WRITE_FILES_DESC
15 | import org.apache.kafka.common.config.ConfigDef.Importance
16 | import org.apache.kafka.common.config.ConfigDef.Type
17 | import org.apache.kafka.common.config.AbstractConfig
18 | import org.apache.kafka.common.config.ConfigDef
19 | import org.apache.kafka.common.config.SslConfigs
20 |
21 | import java.util
22 |
23 | object VaultAuthMethod extends Enumeration {
24 | type VaultAuthMethod = Value
25 | val KUBERNETES, AWSIAM, GCP, USERPASS, LDAP, JWT, CERT, APPROLE, TOKEN, GITHUB = Value
26 |
27 | def withNameOpt(s: String): Option[Value] = values.find(_.toString == s)
28 | }
29 |
30 | object VaultProviderConfig {
31 | val VAULT_ADDR: String = "vault.addr"
32 | val VAULT_TOKEN: String = "vault.token"
33 | val VAULT_NAMESPACE: String = "vault.namespace"
34 | val VAULT_CLIENT_PEM: String = "vault.client.pem"
35 | val VAULT_PEM: String = "vault.pem"
36 | val VAULT_ENGINE_VERSION = "vault.engine.version"
37 | val VAULT_PREFIX_DEPTH = "vault.prefix.depth"
38 | val AUTH_METHOD: String = "vault.auth.method"
39 |
40 | val VAULT_TRUSTSTORE_LOC: String =
41 | s"vault.${SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG}"
42 | val VAULT_KEYSTORE_LOC: String =
43 | s"vault.${SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG}"
44 | val VAULT_KEYSTORE_PASS: String =
45 | s"vault.${SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG}"
46 |
47 | val KUBERNETES_ROLE: String = "kubernetes.role"
48 | val KUBERNETES_TOKEN_PATH: String = "kubernetes.token.path"
49 | val KUBERNETES_DEFAULT_TOKEN_PATH: String =
50 | "/var/run/secrets/kubernetes.io/serviceaccount/token"
51 | val KUBERNETES_AUTH_PATH: String = "kubernetes.auth.path"
52 | val KUBERNETES_AUTH_PATH_DEFAULT: String = "auth/kubernetes"
53 |
54 | val APP_ROLE_PATH: String = "app.role.path"
55 | val APP_ROLE: String = "app.role.id"
56 | val APP_ROLE_SECRET_ID: String = "app.role.secret.id"
57 |
58 | val AWS_ROLE: String = "aws.role"
59 | val AWS_REQUEST_URL: String = "aws.request.url"
60 | val AWS_REQUEST_HEADERS: String = "aws.request.headers"
61 | val AWS_REQUEST_BODY: String = "aws.request.body"
62 | val AWS_MOUNT: String = "aws.mount"
63 |
64 | val GCP_ROLE: String = "gcp.role"
65 | val GCP_JWT: String = "gcp.jwt"
66 |
67 | val LDAP_USERNAME: String = "ldap.username"
68 | val LDAP_PASSWORD: String = "ldap.password"
69 | val LDAP_MOUNT: String = "ldap.mount"
70 |
71 | val USERNAME: String = "username"
72 | val PASSWORD: String = "password"
73 | val UP_MOUNT: String = "mount"
74 |
75 | val JWT_ROLE: String = "jwt.role"
76 | val JWT_PROVIDER: String = "jwt.provider"
77 | val JWT: String = "jwt"
78 |
79 | val CERT_MOUNT: String = "cert.mount"
80 |
81 | val GITHUB_TOKEN: String = "github.token"
82 | val GITHUB_MOUNT: String = "github.mount"
83 |
84 | val TOKEN_RENEWAL: String = "token.renewal.ms"
85 | val TOKEN_RENEWAL_DEFAULT: Int = 600000
86 |
87 | val config: ConfigDef = new ConfigDef()
88 | .define(
89 | VAULT_ADDR,
90 | ConfigDef.Type.STRING,
91 | "http://localhost:8200",
92 | Importance.HIGH,
93 | "Address of the Vault server",
94 | )
95 | .define(
96 | VAULT_TOKEN,
97 | ConfigDef.Type.PASSWORD,
98 | null,
99 | Importance.HIGH,
100 | s"Vault app role token. $AUTH_METHOD must be 'token'",
101 | )
102 | .define(
103 | VAULT_NAMESPACE,
104 | Type.STRING,
105 | "",
106 | Importance.MEDIUM,
107 | "Sets a global namespace to the Vault server instance. Required Vault Enterprize Pro",
108 | )
109 | .define(
110 | VAULT_KEYSTORE_LOC,
111 | ConfigDef.Type.STRING,
112 | "",
113 | ConfigDef.Importance.HIGH,
114 | SslConfigs.SSL_KEYSTORE_LOCATION_DOC,
115 | )
116 | .define(
117 | VAULT_KEYSTORE_PASS,
118 | ConfigDef.Type.PASSWORD,
119 | "",
120 | ConfigDef.Importance.HIGH,
121 | SslConfigs.SSL_KEYSTORE_PASSWORD_DOC,
122 | )
123 | .define(
124 | VAULT_TRUSTSTORE_LOC,
125 | ConfigDef.Type.STRING,
126 | "",
127 | ConfigDef.Importance.HIGH,
128 | SslConfigs.SSL_TRUSTSTORE_LOCATION_DOC,
129 | )
130 | .define(
131 | VAULT_PEM,
132 | Type.STRING,
133 | "",
134 | Importance.HIGH,
135 | "File containing the Vault server certificate string contents",
136 | )
137 | .define(
138 | VAULT_CLIENT_PEM,
139 | Type.STRING,
140 | "",
141 | Importance.HIGH,
142 | "File containing the Client certificate string contents",
143 | )
144 | .define(
145 | VAULT_ENGINE_VERSION,
146 | Type.INT,
147 | 2,
148 | Importance.HIGH,
149 | "KV Secrets Engine version of the Vault server instance. Defaults to 2",
150 | )
151 | .define(
152 | VAULT_PREFIX_DEPTH,
153 | Type.INT,
154 | 1,
155 | Importance.HIGH,
156 | """
157 | |Set the path depth of the KV Secrets Engine prefix path.
158 | |Normally this is just 1, to correspond to one path element in the prefix path.
159 | |To use a longer prefix path, set this value
160 | |
161 | |""".stripMargin,
162 | )
163 | // auth mode
164 | .define(
165 | AUTH_METHOD,
166 | Type.STRING,
167 | "token",
168 | Importance.HIGH,
169 | """
170 | |The authentication mode for Vault.
171 | |Available values are approle, userpass, kubernetes, cert, token, ldap, gcp, awsiam, jwt, github
172 | |
173 | |""".stripMargin,
174 | )
175 | // app role auth mode
176 | .define(
177 | APP_ROLE_PATH,
178 | Type.STRING,
179 | "approle",
180 | Importance.HIGH,
181 | s"Vault App role path. $AUTH_METHOD must be 'approle'",
182 | )
183 | .define(
184 | APP_ROLE,
185 | Type.STRING,
186 | null,
187 | Importance.HIGH,
188 | s"Vault App role id. $AUTH_METHOD must be 'approle'",
189 | )
190 | .define(
191 | APP_ROLE_SECRET_ID,
192 | Type.PASSWORD,
193 | null,
194 | Importance.HIGH,
195 | s"Vault App role name secret id. $AUTH_METHOD must be 'approle'",
196 | )
197 | // userpass
198 | .define(
199 | USERNAME,
200 | Type.STRING,
201 | null,
202 | Importance.HIGH,
203 | s"Username to connect to Vault with. $AUTH_METHOD must be 'userpass'",
204 | )
205 | .define(
206 | PASSWORD,
207 | Type.PASSWORD,
208 | null,
209 | Importance.HIGH,
210 | s"Password for the username. $AUTH_METHOD must be 'uerspass'",
211 | )
212 | .define(
213 | UP_MOUNT,
214 | Type.STRING,
215 | "userpass",
216 | Importance.HIGH,
217 | s"The mount name of the userpass authentication back end. Defaults to 'userpass'. $AUTH_METHOD must be 'userpass'",
218 | )
219 | // kubernetes
220 | .define(
221 | KUBERNETES_ROLE,
222 | Type.STRING,
223 | null,
224 | Importance.HIGH,
225 | s"The kubernetes role used for authentication. $AUTH_METHOD must be 'kubernetes'",
226 | )
227 | .define(
228 | KUBERNETES_TOKEN_PATH,
229 | Type.STRING,
230 | KUBERNETES_DEFAULT_TOKEN_PATH,
231 | Importance.HIGH,
232 | s"""
233 | | s"Path to the service account token. $AUTH_METHOD must be 'kubernetes'.
234 | | Defaults to $KUBERNETES_DEFAULT_TOKEN_PATH
235 |
236 | | $AUTH_METHOD must be '
237 |
238 | |""".stripMargin,
239 | )
240 | // awsiam
241 | .define(
242 | AWS_ROLE,
243 | Type.STRING,
244 | null,
245 | Importance.HIGH,
246 | s"""
247 | |Name of the role against which the login is being attempted. If role is not specified, t
248 | in endpoint
249 | |looks for a role bearing the name of the AMI ID of the EC2 instance that is trying to
250 | ing the ec2
251 | |auth method, or the "friendly name" (i.e., role name or username) of the IAM pr
252 | henticated.
253 | |If a matching role is not found, login fails. $AUTH_METHOD
254 | must be 'awsiam'
255 | |""".stripMargin,
256 | )
257 | .define(
258 | AWS_REQUEST_URL,
259 | Type.STRING,
260 | null,
261 | Importance.HIGH,
262 | s"""
263 | |PKCS7 signature of the identity document with all \n characters removed.Base64-encoded Hsed in the signed request.
264 | |Most likely just aHR0cHM6Ly9zdHMuYW1hem9uYXdzLmNvbS8= (base64-encoding of https://sts.amom/) as most requests will
265 | |probably use POST with an empty URI. $AUTH_METHOD must be 'awsiam'
266 | |""".stripMargin,
267 | )
268 | .define(
269 | AWS_REQUEST_HEADERS,
270 | Type.PASSWORD,
271 | null,
272 | Importance.HIGH,
273 | s"Request headers. $AUTH_METHOD must be 'awsiam'",
274 | )
275 | .define(
276 | AWS_REQUEST_BODY,
277 | Type.PASSWORD,
278 | null,
279 | Importance.HIGH,
280 | s"""
281 |
282 | ded body of the signed request.
283 | |Most likely QWN0aW9uPUdldENhbGxlcklkZ
284 | nNpb249MjAxMS0wNi0xNQ== which is
285 | |the base64 encoding of Action=GetCallerIdentity&Versi
286 | 5. $AUTH_METHOD must be 'awsiam'
287 | |""".stripMargin,
288 | )
289 | .define(
290 | AWS_MOUNT,
291 | Type.STRING,
292 | "aws",
293 | Importance.HIGH,
294 | s"AWS auth mount. $AUTH_METHOD must be 'awsiam'. Default 'aws'",
295 | )
296 | //ldap
297 | .define(
298 | LDAP_USERNAME,
299 | Type.STRING,
300 | null,
301 | Importance.HIGH,
302 | s"LDAP username to connect to Vault with. $AUTH_METHOD must be 'ldap'",
303 | )
304 | .define(
305 | LDAP_PASSWORD,
306 | Type.PASSWORD,
307 | null,
308 | Importance.HIGH,
309 | s"LDAP Password for the username. $AUTH_METHOD must be 'ldap'",
310 | )
311 | .define(
312 | LDAP_MOUNT,
313 | Type.STRING,
314 | "ldap",
315 | Importance.HIGH,
316 | s"The mount name of the ldap authentication back end. Defaults to 'ldap'. $AUTH_METHOD must be 'ldap'",
317 | )
318 | //jwt
319 | .define(
320 | JWT_ROLE,
321 | Type.STRING,
322 | null,
323 | Importance.HIGH,
324 | s"Role the JWT token belongs to. $AUTH_METHOD must be 'jwt'",
325 | )
326 | .define(
327 | JWT_PROVIDER,
328 | Type.STRING,
329 | null,
330 | Importance.HIGH,
331 | s"Provider of JWT token. $AUTH_METHOD must be 'jwt'",
332 | )
333 | .define(
334 | JWT,
335 | Type.PASSWORD,
336 | null,
337 | Importance.HIGH,
338 | s"JWT token. $AUTH_METHOD must be 'jwt'",
339 | )
340 | //gcp
341 | .define(
342 | GCP_ROLE,
343 | Type.STRING,
344 | null,
345 | Importance.HIGH,
346 | s"The gcp role used for authentication. $AUTH_METHOD must be 'gcp'",
347 | )
348 | .define(
349 | GCP_JWT,
350 | Type.PASSWORD,
351 | null,
352 | Importance.HIGH,
353 | s"JWT token. $AUTH_METHOD must be 'gcp'",
354 | )
355 | // cert mount
356 | .define(
357 | CERT_MOUNT,
358 | Type.STRING,
359 | "cert",
360 | Importance.HIGH,
361 | s"The mount name of the cert authentication back end. Defaults to 'cert'. $AUTH_METHOD must be 'cert'",
362 | )
363 | .define(
364 | GITHUB_TOKEN,
365 | Type.PASSWORD,
366 | null,
367 | Importance.HIGH,
368 | s"The github app-id used for authentication. $AUTH_METHOD must be 'github'",
369 | )
370 | .define(
371 | GITHUB_MOUNT,
372 | Type.STRING,
373 | "github",
374 | Importance.HIGH,
375 | s"The mount name of the github authentication back end. Defaults to 'cert'. $AUTH_METHOD must be 'github'",
376 | )
377 | .define(
378 | FILE_DIR,
379 | Type.STRING,
380 | "",
381 | Importance.MEDIUM,
382 | FILE_DIR_DESC,
383 | )
384 | .define(
385 | WRITE_FILES,
386 | Type.BOOLEAN,
387 | false,
388 | Importance.MEDIUM,
389 | WRITE_FILES_DESC,
390 | )
391 | .define(
392 | TOKEN_RENEWAL,
393 | Type.INT,
394 | TOKEN_RENEWAL_DEFAULT,
395 | Importance.MEDIUM,
396 | "The time in milliseconds to renew the Vault token",
397 | ).define(
398 | KUBERNETES_AUTH_PATH,
399 | Type.STRING,
400 | KUBERNETES_AUTH_PATH_DEFAULT,
401 | Importance.MEDIUM,
402 | "The mount path of the Vault Kubernetes Auth Method",
403 | )
404 | .define(
405 | SECRET_DEFAULT_TTL,
406 | Type.LONG,
407 | SECRET_DEFAULT_TTL_DEFAULT,
408 | Importance.MEDIUM,
409 | "Default TTL to apply in case a secret has no TTL",
410 | )
411 |
412 | }
413 | case class VaultProviderConfig(props: util.Map[String, _]) extends AbstractConfig(VaultProviderConfig.config, props)
414 |
--------------------------------------------------------------------------------
](https://docs.lenses.io/4.0/integrations/connectors/secret-providers/)
4 | [](https://opensource.org/licenses/Apache-2.0)
5 |
6 | # Secret Provider
7 |
8 | Secret provider for Kafka to provide indirect look up of configuration values.
9 |
10 | ## Description
11 |
12 | External secret providers allow for indirect references to be placed in an
13 | applications' configuration, so for example, that secrets are not exposed in the
14 | Worker API endpoints of Kafka Connect.
15 |
16 | For [Documentation](https://docs.lenses.io/5.3/connectors/connect-secrets/).
17 |
18 |
19 | ## Contributing
20 |
21 | We'd love to accept your contributions! Please use GitHub pull requests: fork
22 | the repo, develop and test your code,
23 | [semantically commit](http://karma-runner.github.io/1.0/dev/git-commit-msg.html)
24 | and submit a pull request. Thanks!
25 |
26 | ### Building
27 |
28 | ***Requires SBT to build.***
29 |
30 | To build the (scala 2.12 and 2.13) assemblies for use with Kafka Connect (also runs tests):
31 |
32 | ```bash
33 | sbt +assembly
34 | ```
35 |
36 | To run tests:
37 |
38 | ```bash
39 | sbt +test
40 | ```
41 |
42 |
43 | ## License
44 | [](https://app.fossa.com/projects/git%2Bgithub.com%2Flensesio%2Fsecret-provider?ref=badge_large)
45 |
--------------------------------------------------------------------------------
/test-sink/src/main/scala/io/lenses/connect/secrets/test/vault/TestSinkConnector.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.test.vault
2 |
3 | import com.typesafe.scalalogging.LazyLogging
4 | import org.apache.kafka.common.config.ConfigDef
5 | import org.apache.kafka.common.config.ConfigDef.Importance
6 | import org.apache.kafka.connect.connector.Task
7 | import org.apache.kafka.connect.sink.SinkConnector
8 |
9 | import java.util
10 | import scala.jdk.CollectionConverters.MapHasAsScala
11 | import scala.jdk.CollectionConverters.SeqHasAsJava
12 |
13 | class TestSinkConnector extends SinkConnector with LazyLogging {
14 |
15 | private val storedProps: util.Map[String, String] =
16 | new util.HashMap[String, String]()
17 |
18 | override def start(props: util.Map[String, String]): Unit = {
19 | logger.info("start(props:{})", props.asScala)
20 | storedProps.putAll(props)
21 | }
22 |
23 | override def taskClass(): Class[_ <: Task] = classOf[TestSinkTask]
24 |
25 | override def taskConfigs(
26 | maxTasks: Int,
27 | ): util.List[util.Map[String, String]] = {
28 | logger.info("testConfigs(maxTasks:{})", maxTasks)
29 | List.fill(1)(storedProps).asJava
30 | }
31 |
32 | override def stop(): Unit = {
33 | logger.info("stop()")
34 | ()
35 | }
36 |
37 | override def config(): ConfigDef = new ConfigDef()
38 | .define("test.sink.vault.host", ConfigDef.Type.STRING, Importance.LOW, "host")
39 | .define("test.sink.vault.token", ConfigDef.Type.STRING, Importance.LOW, "token")
40 | .define("test.sink.secret.value", ConfigDef.Type.STRING, Importance.LOW, "value")
41 | .define("test.sink.secret.path", ConfigDef.Type.STRING, Importance.LOW, "path")
42 | .define("test.sink.secret.key", ConfigDef.Type.PASSWORD, Importance.LOW, "key")
43 |
44 | override def version(): String = "0.0.1a"
45 | }
46 |
--------------------------------------------------------------------------------
/secret-provider/src/test/scala/io/lenses/connect/secrets/io/FileWriterOnceTest.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.io
2 |
3 | import org.scalatest.BeforeAndAfterAll
4 | import org.scalatest.funsuite.AnyFunSuite
5 | import org.scalatest.matchers.should.Matchers
6 |
7 | import java.io.File
8 | import java.nio.file.Paths
9 | import java.util.UUID
10 | import scala.io.Source
11 | import scala.util.Success
12 | import scala.util.Try
13 | import scala.util.Using
14 |
15 | class FileWriterOnceTest extends AnyFunSuite with Matchers with BeforeAndAfterAll {
16 | private val folder = new File(UUID.randomUUID().toString)
17 | folder.deleteOnExit()
18 | private val writer = new FileWriterOnce(folder.toPath)
19 |
20 | override protected def beforeAll(): Unit =
21 | folder.mkdir()
22 |
23 | override protected def afterAll(): Unit = {
24 | folder.listFiles().foreach(f => Try(f.delete()))
25 | folder.delete()
26 | }
27 |
28 | test("writes the file") {
29 | val content = Array(1, 2, 3).map(_.toByte)
30 | writer.write("thisone", content, "key1")
31 | val file = Paths.get(folder.toPath.toString, "thisone").toFile
32 | file.exists() shouldBe true
33 |
34 | Using(Source.fromFile(file))(_.size) shouldBe Success(content.length)
35 | }
36 |
37 | test("does not write the file twice") {
38 | val content1 = Array(1, 2, 3).map(_.toByte)
39 | val fileName = "thissecond"
40 | writer.write(fileName, content1, "key1")
41 | val file = Paths.get(folder.toPath.toString, fileName).toFile
42 | file.exists() shouldBe true
43 |
44 | Using(Source.fromFile(file))(_.size) shouldBe Success(content1.length)
45 |
46 | val content2 = Array(1, 2, 3, 4, 5, 6, 7, 8).map(_.toByte)
47 | writer.write(fileName, content2, "key1")
48 | Using(Source.fromFile(file))(_.size) shouldBe Success(content1.length)
49 |
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/secret-provider/src/test/scala/io/lenses/connect/secrets/config/SecretTypeConfigTest.scala:
--------------------------------------------------------------------------------
1 | package io.lenses.connect.secrets.config
2 |
3 | import org.apache.kafka.connect.errors.ConnectException
4 | import org.mockito.Mockito.when
5 | import org.scalatest.OptionValues
6 | import org.scalatest.funsuite.AnyFunSuite
7 | import org.scalatest.matchers.should.Matchers
8 | import org.scalatestplus.mockito.MockitoSugar
9 |
10 | class SecretTypeConfigTest extends AnyFunSuite with Matchers with OptionValues with MockitoSugar {
11 |
12 | test("lookupAndValidateSecretTypeValue should throw accept valid types") {
13 | SecretTypeConfig.lookupAndValidateSecretTypeValue(mockSecretReturn("string")) should be(SecretType.STRING)
14 | SecretTypeConfig.lookupAndValidateSecretTypeValue(mockSecretReturn("STRING")) should be(SecretType.STRING)
15 | SecretTypeConfig.lookupAndValidateSecretTypeValue(mockSecretReturn("json")) should be(SecretType.JSON)
16 | SecretTypeConfig.lookupAndValidateSecretTypeValue(mockSecretReturn("JSON")) should be(SecretType.JSON)
17 | }
18 |
19 | test("lookupAndValidateSecretTypeValue should throw error on invalid type") {
20 | val secRetFn = mockSecretReturn("nonExistentSecretType")
21 | intercept[ConnectException] {
22 | SecretTypeConfig.lookupAndValidateSecretTypeValue(secRetFn)
23 | }.getMessage should startWith("nonExistentSecretType is not a valid secret type")
24 | }
25 |
26 | test("lookupAndValidateSecretTypeValue should return json by default") {
27 | SecretTypeConfig.lookupAndValidateSecretTypeValue(mockSecretReturn(null)) should be(SecretType.JSON)
28 | SecretTypeConfig.lookupAndValidateSecretTypeValue(mockSecretReturn("")) should be(SecretType.JSON)
29 | }
30 |
31 | private def mockSecretReturn(out: String): String => String = {
32 | val mockFn = mock[String => String]
33 | when(mockFn.apply(SecretTypeConfig.SECRET_TYPE)).thenReturn(out)
34 | mockFn
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/secret-provider/src/it/resources/logback.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |