to $TRAVIS_TAG"
40 | mvn --settings "${TRAVIS_BUILD_DIR}/.travis/mvn-settings.xml" org.codehaus.mojo:versions-maven-plugin:2.1:set -DnewVersion=$TRAVIS_TAG 1>/dev/null 2>/dev/null
41 | else
42 | echo "not on a tag -> keep snapshot version in pom.xml"
43 | fi
44 |
45 | # Run the maven deploy steps
46 | mvn deploy -P publish -DskipTests=true --settings "${TRAVIS_BUILD_DIR}/.travis/mvn-settings.xml"
47 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/SampleKeyNameExtractor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import io.quicksign.kafka.crypto.keyrepository.KeyNameExtractor;
23 |
24 |
25 | /**
26 | * Simple key name extractor that use the recordKey as the key name.
27 | * More complex implementation could leverage the key hidden structure
28 | * or take into account the current time.
29 | */
30 | public class SampleKeyNameExtractor implements KeyNameExtractor {
31 |
32 | @Override
33 | public String extractKeyName(String topic, Object recordKey) {
34 | return recordKey.toString();
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SampleKeyNameExtractor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.stream.keyrepo;
21 |
22 | import io.quicksign.kafka.crypto.keyrepository.KeyNameExtractor;
23 |
24 |
25 | /**
26 | * Simple key name extractor that use the recordKey as the key name.
27 | * More complex implementation could leverage the key hidden structure
28 | * or take into account the current time.
29 | */
30 | public class SampleKeyNameExtractor implements KeyNameExtractor {
31 |
32 | @Override
33 | public String extractKeyName(String topic, Object recordKey) {
34 | return "cpt" + recordKey;
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/generatedkey/src/main/java/io/quicksign/kafka/crypto/generatedkey/MasterKeyEncryption.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.generatedkey;
21 |
22 | /**
23 | * Encryption to encrypt/decrypt generated keys
24 | */
25 | public interface MasterKeyEncryption {
26 |
27 | /**
28 | * encrypt the provided key with a master key
29 | *
30 | * @param key the key to be encrypted
31 | * @return
32 | */
33 | byte[] encryptKey(byte[] key);
34 |
35 | /**
36 | * decrypt the key with a master key
37 | *
38 | * @param encryptedKey
39 | * @return the decrypted key or {@code null} if the key can not be decrypted
40 | */
41 | byte[] decryptKey(byte[] encryptedKey);
42 | }
43 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/Decryptor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto;
21 |
22 | /**
23 | * Interface for decryption. It encapsulates the retriaval of the decryption key using the key reference
24 | * and the decryption of the data
25 | */
26 | public interface Decryptor {
27 |
28 | /**
29 | * Decrypt the data.
30 | * It will return null if the key can not be retrieve or if decryption fails.
31 | *
32 | * @param data value to be decrypted
33 | * @param keyRef reference to the decryption key
34 | * @return the decrypted value or {@code null} if the key cannot be retrieved or if decryption fails
35 | */
36 | byte[] decrypt(byte[] data, byte[] keyRef);
37 | }
38 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/Encryptor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto;
21 |
22 | /**
23 | * Interface for encryption. It encapsulates the retriaval of the encryption key using the key reference
24 | * and the encryption of the data
25 | */
26 | public interface Encryptor {
27 |
28 | /**
29 | * Encrypt the data.
30 | * It will return null if encryption fails or if the key associated to the key reference
31 | * can not be found.
32 | *
33 | * @param value value to be encrypted
34 | * @param keyRef reference of the key
35 | * @return the encrypted value or {@code null} if the encryption fails
36 | */
37 | byte[] encrypt(byte[] value, byte[] keyRef);
38 | }
39 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/serializer/SerializerPairFactory.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.serializer;
21 |
22 | import org.apache.kafka.common.serialization.Serializer;
23 |
24 | /**
25 | * A factory to pair 2 {@link Serializer}
26 | *
27 | * Main usage is with traditional Kafka Producer
28 | */
29 | public interface SerializerPairFactory {
30 |
31 | /**
32 | * Pair the keySerializer with the value serializer
33 | *
34 | * @param keySerializer
35 | * @param valueSerializer
36 | * @param
37 | * @param
38 | * @return
39 | */
40 | SerializerPair build(Serializer keySerializer, Serializer valueSerializer);
41 | }
42 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/serdes/SerdeFactory.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.serdes;
21 |
22 | import org.apache.kafka.common.serialization.Serde;
23 |
24 | /**
25 | * A factory to pair 2 {@link org.apache.kafka.common.serialization.Serde Serde}
26 | *
27 | * Main usage is for Kafka Streams
28 | */
29 | public interface SerdeFactory {
30 |
31 | /**
32 | * Pair the keySerde with the valueSerde
33 | *
34 | * @param keySerde
35 | * @param valueSerde
36 | * @param
37 | * @param
38 | * @return
39 | */
40 | SerdesPair buildSerdesPair(Serde keySerde, Serde valueSerde);
41 |
42 | Serde buildSelfCryptoAwareSerde(Serde valueSerde);
43 | }
44 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/generateKeys.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Copyright 2018 Quicksign
4 | #
5 | # Licensed under the Apache License, Version 2.0 (the "License");
6 | # you may not use this file except in compliance with the License.
7 | # You may obtain a copy of the License at
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License. See accompanying LICENSE file.
14 |
15 | rm /tmp/samplestream.pkcs12
16 | rm /tmp/samplestream1.pkcs12
17 | rm /tmp/samplestream2.pkcs12
18 | for i in `seq 0 9`; do
19 | keytool -genseckey -keystore /tmp/samplestream.pkcs12 -storetype pkcs12 -storepass sample -keyalg AES -keysize 256 -alias cpt$i -keypass sample
20 | done
21 |
22 | for i in `seq 0 4`; do
23 | keytool -importkeystore -srckeystore /tmp/samplestream.pkcs12 -srcstorepass sample \
24 | -destkeystore /tmp/samplestream1.pkcs12 -deststoretype pkcs12 -deststorepass sample \
25 | -srcalias cpt$i
26 | done
27 |
28 | for i in `seq 5 9`; do
29 | keytool -importkeystore -srckeystore /tmp/samplestream.pkcs12 -srcstorepass sample \
30 | -destkeystore /tmp/samplestream2.pkcs12 -deststoretype pkcs12 -deststorepass sample \
31 | -srcalias cpt$i
32 | done
33 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/keyextractor/KeyReferenceExtractor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.keyextractor;
21 |
22 | /**
23 | * Used at encryption time to compute the keyref associated to the message key.
24 | *
25 | * The choice of a {@link KeyReferenceExtractor} has to be done accordingly
26 | * to the chosen {@link io.quicksign.kafka.crypto.encryption.KeyProvider KeyProvider}
27 | *
28 | * @see io.quicksign.kafka.crypto.encryption.KeyProvider
29 | */
30 | public interface KeyReferenceExtractor {
31 |
32 | /**
33 | * Compute the key reference for a message
34 | *
35 | * @param topic the topic name on which the message is about to be published
36 | * @param key the message key (in nominal case)
37 | * @return the keyref
38 | */
39 | byte[] extractKeyReference(String topic, Object key);
40 | }
41 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/serializer/SerializerPair.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.serializer;
21 |
22 | import org.apache.kafka.common.serialization.Serializer;
23 |
24 | /**
25 | * Represent paired {@link Serializer}
26 | *
27 | * @param
28 | * @param
29 | */
30 | public class SerializerPair {
31 |
32 | private final Serializer keySerializer;
33 | private final Serializer valueSerializer;
34 |
35 | public SerializerPair(Serializer keySerializer, Serializer valueSerializer) {
36 |
37 | this.keySerializer = keySerializer;
38 | this.valueSerializer = valueSerializer;
39 | }
40 |
41 | public Serializer getKeySerializer() {
42 | return keySerializer;
43 | }
44 |
45 | public Serializer getValueSerializer() {
46 | return valueSerializer;
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/encryption/CryptoAlgorithm.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.encryption;
21 |
22 | /**
23 | * Interface to handle cryptographic algorithm
24 | *
25 | * @see DefaultDecryptor
26 | * @see DefaultEncryptor
27 | */
28 | public interface CryptoAlgorithm {
29 |
30 | /**
31 | * Encrypt the data using the provided key
32 | *
33 | * @param data message to be encrypted
34 | * @param key encryption key
35 | * @return encrypted message
36 | * @throws Exception
37 | */
38 | byte[] encrypt(byte[] data, byte[] key) throws Exception;
39 |
40 | /**
41 | * Decrypt the data using the provided key
42 | *
43 | * @param encryptedData message to be decrypted
44 | * @param key encryption key
45 | * @return decrypted message
46 | * @throws Exception
47 | */
48 | byte[] decrypt(byte[] encryptedData, byte[] key) throws Exception;
49 | }
50 |
--------------------------------------------------------------------------------
/generatedkey/src/main/java/io/quicksign/kafka/crypto/generatedkey/PerRecordKeyProvider.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.generatedkey;
21 |
22 | import java.util.Optional;
23 |
24 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
25 |
26 | /**
27 | * This key provider consider key reference as the result of the encryption by a master.
28 | *
29 | * It will try to decrypt the key using the master key. The result will be {@link Optional#EMPTY} if the key can not be decrypted
30 | */
31 | public class PerRecordKeyProvider implements KeyProvider {
32 |
33 | private final MasterKeyEncryption masterKeyEncryption;
34 |
35 | public PerRecordKeyProvider(MasterKeyEncryption masterKeyEncryption) {
36 |
37 | this.masterKeyEncryption = masterKeyEncryption;
38 | }
39 |
40 |
41 | @Override
42 | public Optional getKey(byte[] keyRef) {
43 | return Optional.ofNullable(masterKeyEncryption.decryptKey(keyRef));
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/index.adoc:
--------------------------------------------------------------------------------
1 | # Quicksign Kafka Encryption
2 |
3 | v{project-version}
4 |
5 | ## Motivation
6 |
7 | Personal data protection is becoming increasingly important for everyone these days. We chose no compromise, we encrypt
8 | everything end to end.
9 |
10 | Quicksign uses Kafka as the backbone of its microservices architecture, we therefore needed a solution
11 | to achieve end to end encryption in Kafka. Since it didn't appear to be available, we had to implement our own. We
12 | believe opening it to the community will help broaden best practices in terms of personal data protection and improve
13 | its security through a larger audience scrutiny.
14 |
15 | CAUTION: This library is the actual one used on our platform but some particular encryption and operational details
16 | are not revealed here and as such this library on its own doesn't reflect the overall mechanism used at Quicksign
17 | to protect our users data.
18 |
19 | ## Usage
20 |
21 | ### Import maven dependency
22 |
23 | [source,xml,subs="attributes+"]
24 | ....
25 |
26 | {project-groupId}
27 | kafka-encryption-core
28 | {project-version}
29 |
30 | ....
31 |
32 | ## Design goals
33 |
34 | * Support multiple encryption key management strategies (KMS, embedded, per message, rolling windows, per tenant, custom)
35 | * Support for Kafka Streams intermediate topics
36 | * Detect when a payload is not encrypted to skip decryption
37 | * Support for Camel
38 | * Support for Spring Boot
39 |
40 | ## Use cases
41 | include::./samples/keyrepo-sample/index.adoc[]
42 | include::./samples/generatedkey-sample/index.adoc[]
43 | include::./samples/kafkastream-with-keyrepo-sample/index.adoc[]
44 |
--------------------------------------------------------------------------------
/generatedkey/src/main/java/io/quicksign/kafka/crypto/generatedkey/AES256CryptoKeyGenerator.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.generatedkey;
21 |
22 | import java.security.NoSuchAlgorithmException;
23 |
24 | import javax.crypto.KeyGenerator;
25 | import javax.crypto.SecretKey;
26 |
27 | /**
28 | * Generator for AES-256 encryption key
29 | */
30 | public class AES256CryptoKeyGenerator implements CryptoKeyGenerator {
31 |
32 | /**
33 | * Generate a new AES-256 encryption key
34 | *
35 | * @return a new AES-256 encryption key
36 | */
37 | @Override
38 | public byte[] generateKey() {
39 | try {
40 | KeyGenerator keyGenerator = KeyGenerator.getInstance("AES");
41 | keyGenerator.init(256);
42 | SecretKey secretKey = keyGenerator.generateKey();
43 | return secretKey.getEncoded();
44 | }
45 | catch (NoSuchAlgorithmException e) {
46 | throw new RuntimeException("unable to handle AES encryption", e);
47 | }
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/index.adoc:
--------------------------------------------------------------------------------
1 | ### Key repository sample (Streams API)
2 |
3 | This is a dummy sample of a bank account. Each account has an associated
4 | key to encrypt all operations associated to the account.
5 |
6 | For this account, we manage the bank view and the agency view.
7 |
8 | * The bank has access to the full key repository (acount 0 to 9).
9 | * Agency 1 has only access to a subset of key repository (account 0 to 4)
10 | * Agency 2 has only access to a subset of key repostory (account 5 to 9)
11 |
12 | All operations are written encrypted into a common topic `operations`
13 |
14 | #### Running the sample
15 |
16 | [source,indent=0]
17 | ....
18 | cd samples/kafkastream-with-keyrepo-sample
19 | docker-compose up # on windows and OSX, you need to adjust
20 | sh generateKeys.sh
21 | sh runSamples.sh
22 | ....
23 |
24 | include::../../doc/running-docker-kafka.adoc[]
25 |
26 | #### Usage
27 |
28 |
29 | ##### Sample Main
30 |
31 | [source,java,indent=0]
32 | ....
33 | include::./src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SamplesMain.java[tags=main]
34 | ....
35 |
36 | ##### Producer side
37 |
38 | .SampleProducer.java
39 | [source,java,indent=0]
40 | ....
41 | include::./src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SampleProducer.java[tags=produce]
42 | ....
43 |
44 | ##### Stream side
45 |
46 | .SampleStream.java
47 | [source,java,indent=0]
48 | ....
49 | include::./src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SampleStream.java[tags=stream]
50 | ....
51 |
52 |
53 | and display
54 |
55 | .SampleStream.java
56 | [source,java,indent=0]
57 | ....
58 | include::./src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SampleStream.java[tags=display]
59 | ....
60 |
--------------------------------------------------------------------------------
/generatedkey/src/main/java/io/quicksign/kafka/crypto/generatedkey/KeyPerRecordKeyReferenceExtractor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.generatedkey;
21 |
22 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
23 |
24 | /**
25 | * KeyReference extractor that will generate an new key for each record.
26 | * The key reference will be the key encrypted by the master key.
27 | */
28 | public class KeyPerRecordKeyReferenceExtractor implements KeyReferenceExtractor {
29 |
30 | private final CryptoKeyGenerator cryptoKeyGenerator;
31 | private final MasterKeyEncryption masterKeyEncryption;
32 |
33 | public KeyPerRecordKeyReferenceExtractor(CryptoKeyGenerator cryptoKeyGenerator,
34 | MasterKeyEncryption masterKeyEncryption) {
35 |
36 | this.cryptoKeyGenerator = cryptoKeyGenerator;
37 | this.masterKeyEncryption = masterKeyEncryption;
38 | }
39 |
40 |
41 | @Override
42 | public byte[] extractKeyReference(String topic, Object key) {
43 | return masterKeyEncryption.encryptKey(cryptoKeyGenerator.generateKey());
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/samples/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 |
21 | io.quicksign
22 | kafka-encryption-parent
23 | 0.0.0-SNAPSHOT
24 |
25 |
26 | kafka-encryption-samples
27 | pom
28 |
29 | kafka-encryption-samples
30 | Kafka Record Encryption Samples
31 |
32 |
33 | generatedkey-sample
34 | keyrepo-sample
35 | kafkastream-with-keyrepo-sample
36 |
37 |
38 |
39 |
40 |
41 | org.apache.maven.plugins
42 | maven-deploy-plugin
43 | 2.7
44 |
45 | true
46 |
47 |
48 |
49 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/SampleKeyRepository.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import java.util.Map;
23 | import java.util.Optional;
24 | import java.util.concurrent.ConcurrentHashMap;
25 |
26 | import io.quicksign.kafka.crypto.generatedkey.AES256CryptoKeyGenerator;
27 | import io.quicksign.kafka.crypto.generatedkey.CryptoKeyGenerator;
28 | import io.quicksign.kafka.crypto.keyrepository.KeyRepository;
29 |
30 | /**
31 | * In a real life scenario your encryption keys could be stored encrypted in a persistent storage accessible by
32 | * the producer and the consumer, you would of course cache them locally in memory to avoid introducing too much
33 | * latency. In our example, the repository is a simple map.
34 | */
35 | public class SampleKeyRepository implements KeyRepository {
36 |
37 | // Use an AES256 key generator
38 | private CryptoKeyGenerator cryptoKeyGenerator = new AES256CryptoKeyGenerator();
39 | private Map keyStore = new ConcurrentHashMap<>();
40 |
41 | @Override
42 | public Optional getKey(String keyName) {
43 | return Optional.of(keyStore.computeIfAbsent(keyName, k -> cryptoKeyGenerator.generateKey()));
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/SampleKeyNameObfuscator.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import java.nio.charset.Charset;
23 | import java.util.Arrays;
24 |
25 | import io.quicksign.kafka.crypto.keyrepository.KeyNameObfuscator;
26 |
27 |
28 | /**
29 | * In the payload we store the encryption key reference which is simply an obfuscated version fo the keyName.
30 | * This simple implementation creates the encryption key reference by swapping 2 bytes from the encryption key name.
31 | */
32 | public class SampleKeyNameObfuscator implements KeyNameObfuscator {
33 | @Override
34 | public byte[] obfuscate(String keyName) {
35 | return swapFirstAndLast(keyName.getBytes(Charset.forName("UTF-8")));
36 | }
37 |
38 | @Override
39 | public String unObfuscate(byte[] keyref) {
40 | byte[] copy = Arrays.copyOf(keyref, keyref.length);
41 | return new String(swapFirstAndLast(copy), Charset.forName("UTF-8"));
42 | }
43 |
44 | private byte[] swapFirstAndLast(byte[] keyRef) {
45 | byte[] copy = Arrays.copyOf(keyRef, keyRef.length);
46 | byte first = copy[0];
47 | copy[0] = copy[copy.length - 1];
48 | copy[copy.length - 1] = first;
49 | return copy;
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SampleKeyNameObfuscator.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.stream.keyrepo;
21 |
22 | import java.nio.charset.Charset;
23 | import java.util.Arrays;
24 |
25 | import io.quicksign.kafka.crypto.keyrepository.KeyNameObfuscator;
26 |
27 |
28 | /**
29 | * In the payload we store the encryption key reference which is simply an obfuscated version fo the keyName.
30 | * This simple implementation creates the encryption key reference by swapping 2 bytes from the encryption key name.
31 | */
32 | public class SampleKeyNameObfuscator implements KeyNameObfuscator {
33 | @Override
34 | public byte[] obfuscate(String keyName) {
35 | return swapFirstAndLast(keyName.getBytes(Charset.forName("UTF-8")));
36 | }
37 |
38 | @Override
39 | public String unObfuscate(byte[] keyref) {
40 | byte[] copy = Arrays.copyOf(keyref, keyref.length);
41 | return new String(swapFirstAndLast(copy), Charset.forName("UTF-8"));
42 | }
43 |
44 | private byte[] swapFirstAndLast(byte[] keyRef) {
45 | byte[] copy = Arrays.copyOf(keyRef, keyRef.length);
46 | byte first = copy[0];
47 | copy[0] = copy[copy.length - 1];
48 | copy[copy.length - 1] = first;
49 | return copy;
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/keyrepository/src/main/java/io/quicksign/kafka/crypto/keyrepository/RepositoryBasedKeyReferenceExtractor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.keyrepository;
21 |
22 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
23 |
24 |
25 | /**
26 | * KeyReference used when the key to use is stored on a repository, and the key to use can be computed from the record key
27 | *
28 | * @see KeyNameExtractor
29 | * @see KeyNameObfuscator
30 | */
31 | public class RepositoryBasedKeyReferenceExtractor implements KeyReferenceExtractor {
32 |
33 | private final KeyNameExtractor keyNameExtractor;
34 | private final KeyNameObfuscator keyNameObfuscator;
35 |
36 | /**
37 | * @param keyNameExtractor used to compute the keyName
38 | * @param keyNameObfuscator used to obfuscate the keyName
39 | */
40 | public RepositoryBasedKeyReferenceExtractor(KeyNameExtractor keyNameExtractor, KeyNameObfuscator keyNameObfuscator) {
41 |
42 | this.keyNameExtractor = keyNameExtractor;
43 | this.keyNameObfuscator = keyNameObfuscator;
44 | }
45 |
46 |
47 | @Override
48 | public byte[] extractKeyReference(String topic, Object key) {
49 | String keyName = keyNameExtractor.extractKeyName(topic, key);
50 | return keyName == null ? null : keyNameObfuscator.obfuscate(keyName);
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/keyrepository/src/main/java/io/quicksign/kafka/crypto/keyrepository/RepositoryBasedKeyProvider.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.keyrepository;
21 |
22 | import java.util.Optional;
23 |
24 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
25 |
26 | /**
27 | * KeyProvider based on a key repository
28 | *
29 | * @see KeyRepository
30 | * @see KeyNameObfuscator
31 | */
32 | public class RepositoryBasedKeyProvider implements KeyProvider {
33 |
34 | private final KeyRepository keyRepository;
35 | private final KeyNameObfuscator keyNameObfuscator;
36 |
37 | /**
38 | * @param keyRepository the repository for retrieving keys
39 | * @param keyNameObfuscator KeyNameObfuscator used to unobfuscate the keyName
40 | */
41 | public RepositoryBasedKeyProvider(KeyRepository keyRepository, KeyNameObfuscator keyNameObfuscator) {
42 | this.keyRepository = keyRepository;
43 | this.keyNameObfuscator = keyNameObfuscator;
44 | }
45 |
46 | /**
47 | * Retrieve the key on the key repository. It will first unobfucate the keyRef to obtain the keyName.
48 | *
49 | * @param keyRef the reference of the key to retrieve
50 | * @return
51 | */
52 | @Override
53 | public Optional getKey(byte[] keyRef) {
54 | return keyRepository.getKey(keyNameObfuscator.unObfuscate(keyRef));
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/core/src/test/java/io/quicksign/kafka/crypto/encryption/DefaultDecryptorTest.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.encryption;
21 |
22 | import static org.assertj.core.api.Assertions.assertThat;
23 | import static org.mockito.BDDMockito.given;
24 |
25 | import java.nio.charset.StandardCharsets;
26 | import java.util.Optional;
27 |
28 | import org.junit.Test;
29 | import org.junit.runner.RunWith;
30 | import org.mockito.InjectMocks;
31 | import org.mockito.Mock;
32 | import org.mockito.runners.MockitoJUnitRunner;
33 |
34 | @RunWith(MockitoJUnitRunner.class)
35 | public class DefaultDecryptorTest {
36 |
37 | @Mock
38 | KeyProvider keyProvider;
39 |
40 | @Mock
41 | CryptoAlgorithm encryptionAlgorithm;
42 |
43 | @InjectMocks
44 | DefaultDecryptor defaultDecryptor;
45 |
46 | @Test
47 | public void testDecrypt() throws Exception {
48 | byte[] keyRef = "keyref".getBytes(StandardCharsets.UTF_8);
49 | byte[] key = "key".getBytes(StandardCharsets.UTF_8);
50 | byte[] clearData = "clear data".getBytes(StandardCharsets.UTF_8);
51 | byte[] encodedData = "encoded data".getBytes(StandardCharsets.UTF_8);
52 |
53 | given(keyProvider.getKey(keyRef)).willReturn(Optional.of(key));
54 | given(encryptionAlgorithm.decrypt(encodedData, key)).willReturn(clearData);
55 |
56 | byte[] res = defaultDecryptor.decrypt(encodedData, keyRef);
57 | assertThat(res).isEqualTo(clearData);
58 |
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/core/src/test/java/io/quicksign/kafka/crypto/encryption/DefaultEncryptorTest.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.encryption;
21 |
22 | import static org.assertj.core.api.Assertions.assertThat;
23 | import static org.mockito.BDDMockito.given;
24 |
25 | import java.nio.charset.StandardCharsets;
26 | import java.util.Optional;
27 |
28 | import org.junit.Test;
29 | import org.junit.runner.RunWith;
30 | import org.mockito.InjectMocks;
31 | import org.mockito.Mock;
32 | import org.mockito.runners.MockitoJUnitRunner;
33 |
34 | @RunWith(MockitoJUnitRunner.class)
35 | public class DefaultEncryptorTest {
36 |
37 | @Mock
38 | KeyProvider keyProvider;
39 |
40 | @Mock
41 | CryptoAlgorithm encryptionAlgorithm;
42 |
43 | @InjectMocks
44 | DefaultEncryptor defaultEncryptor;
45 |
46 | @Test
47 | public void testEncrypt() throws Exception {
48 | byte[] keyRef = "keyref".getBytes(StandardCharsets.UTF_8);
49 | byte[] key = "key".getBytes(StandardCharsets.UTF_8);
50 | byte[] clearData = "clear data".getBytes(StandardCharsets.UTF_8);
51 | byte[] encodedData = "encoded data".getBytes(StandardCharsets.UTF_8);
52 |
53 | given(keyProvider.getKey(keyRef)).willReturn(Optional.of(key));
54 | given(encryptionAlgorithm.encrypt(clearData, key)).willReturn(encodedData);
55 |
56 | byte[] res = defaultEncryptor.encrypt(clearData, keyRef);
57 |
58 | assertThat(res).isEqualTo(encodedData);
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/encryption/DefaultEncryptor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.encryption;
21 |
22 | import java.util.Optional;
23 |
24 | import org.slf4j.Logger;
25 | import org.slf4j.LoggerFactory;
26 |
27 | import io.quicksign.kafka.crypto.Encryptor;
28 |
29 | /**
30 | * Default implementation of Encryptor.
31 | * It uses a {@link KeyProvider} to retrieve the key associated to keyr references.
32 | * It use a {@link CryptoAlgorithm} to encrypt the data
33 | */
34 | public class DefaultEncryptor implements Encryptor {
35 |
36 | private static final Logger log = LoggerFactory.getLogger(DefaultEncryptor.class);
37 |
38 | private final KeyProvider keyProvider;
39 | private final CryptoAlgorithm cryptoAlgorithm;
40 |
41 | public DefaultEncryptor(KeyProvider keyProvider, CryptoAlgorithm cryptoAlgorithm) {
42 |
43 | this.keyProvider = keyProvider;
44 | this.cryptoAlgorithm = cryptoAlgorithm;
45 | }
46 |
47 |
48 | /**
49 | * {@inheritDoc}
50 | */
51 | @Override
52 | public byte[] encrypt(byte[] value, byte[] keyRef) {
53 | //error on key retrieving must stop the world
54 | Optional maybeKey = keyProvider.getKey(keyRef);
55 | return maybeKey.map(key -> {
56 | try {
57 | return cryptoAlgorithm.encrypt(value, key);
58 | }
59 | catch (Exception e) {
60 | log.error("error while encrypting data", e);
61 | return null;
62 | }
63 | }).orElse(null);
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/package-info.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | /**
21 | * A common use case is to choose the encryption key accordingly to the record key.
22 | * For example, in an event log, you may want to encrypt all records relative to the
23 | * same transaction with the same encryption key and the transaction id can be deducted
24 | * from the record key.
25 | *
26 | * To acheive this, the key serializer and the value serializer will be paired.
27 | * The key serializer will be wrapped to call the a
28 | * {@link io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor KeyReferenceExtractor}
29 | * and put in the context the cryptographic key reference, that will be used to encrypt the record value.
30 | *
31 | * This based on the fact that {@link org.apache.kafka.clients.producer.KafkaProducer#doSend(org.apache.kafka.clients.producer.ProducerRecord, org.apache.kafka.clients.producer.Callback) KafkaProducer}
32 | * and {@link org.apache.kafka.streams.processor.internals.RecordCollectorImpl#send(java.lang.String, java.lang.Object, java.lang.Object, org.apache.kafka.common.header.Headers, java.lang.Integer, java.lang.Long, org.apache.kafka.common.serialization.Serializer, org.apache.kafka.common.serialization.Serializer) RecordCollector}
33 | * (for streams)
34 | * call key serialization before value serialization.
35 | *
36 | *
37 | * @see io.quicksign.kafka.crypto.pairing.serializer.CryptoSerializerPairFactory
38 | * @see io.quicksign.kafka.crypto.pairing.serdes.CryptoSerdeFactory
39 | */
40 | package io.quicksign.kafka.crypto.pairing;
41 |
42 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/encryption/DefaultDecryptor.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.encryption;
21 |
22 | import java.util.Optional;
23 |
24 | import org.slf4j.Logger;
25 | import org.slf4j.LoggerFactory;
26 |
27 | import io.quicksign.kafka.crypto.Decryptor;
28 |
29 | /**
30 | * Default implementation of Decryptor.
31 | * It uses a {@link KeyProvider} to retrieve the key associated to keyr references.
32 | * It use a {@link CryptoAlgorithm} to decrypt the data
33 | */
34 | public class DefaultDecryptor implements Decryptor {
35 |
36 | private static final Logger log = LoggerFactory.getLogger(DefaultDecryptor.class);
37 |
38 | private final KeyProvider keyProvider;
39 | private final CryptoAlgorithm cryptoAlgorithm;
40 |
41 | public DefaultDecryptor(KeyProvider keyProvider, CryptoAlgorithm cryptoAlgorithm) {
42 |
43 | this.keyProvider = keyProvider;
44 | this.cryptoAlgorithm = cryptoAlgorithm;
45 | }
46 |
47 |
48 | /**
49 | * {@inheritDoc}
50 | */
51 | @Override
52 | public byte[] decrypt(byte[] value, byte[] keyRef) {
53 | //error on key retrieving must stop the world
54 | Optional maybeKey = keyProvider.getKey(keyRef);
55 | return maybeKey.map(key -> {
56 | try {
57 | return cryptoAlgorithm.decrypt(value, key);
58 | }
59 | catch (Exception e) {
60 | log.error("error while decrypting data", e);
61 | return null;
62 | }
63 | }
64 | ).orElse(null);
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/serializer/CryptoSerializerPairFactory.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.serializer;
21 |
22 | import org.apache.kafka.common.serialization.Serializer;
23 |
24 | import io.quicksign.kafka.crypto.CryptoSerializer;
25 | import io.quicksign.kafka.crypto.Encryptor;
26 | import io.quicksign.kafka.crypto.pairing.internal.CryptoAwareSerializerWrapper;
27 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
28 |
29 | /**
30 | * A factory to pair 2 serializers
31 | *
32 | * - the keySerializer is wrapped to call the {@link KeyReferenceExtractor}
33 | * - the valueSerializer is wrapped into a {@link CryptoSerializer}
34 | *
35 | * The keyref extracted by the wrapped key serializer will be shared with the wrapped value serializer using Kafka headers
36 | */
37 | public class CryptoSerializerPairFactory implements SerializerPairFactory {
38 |
39 | private final Encryptor encryptor;
40 | private final KeyReferenceExtractor keyReferenceExtractor;
41 |
42 | public CryptoSerializerPairFactory(Encryptor encryptor, KeyReferenceExtractor keyReferenceExtractor) {
43 | this.encryptor = encryptor;
44 | this.keyReferenceExtractor = keyReferenceExtractor;
45 | }
46 |
47 | /**
48 | * {@inheritDoc}
49 | */
50 | @Override
51 | public SerializerPair build(Serializer keySerializer, Serializer valueSerializer) {
52 | Serializer newKeySerializer = new CryptoAwareSerializerWrapper(keySerializer, keyReferenceExtractor, null);
53 | Serializer newvalueSerializer = new CryptoSerializer<>(valueSerializer, encryptor, null);
54 | return new SerializerPair<>(newKeySerializer, newvalueSerializer);
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/generatedkey/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 |
21 | io.quicksign
22 | kafka-encryption-parent
23 | 0.0.0-SNAPSHOT
24 |
25 |
26 | kafka-encryption-generatedkey
27 | jar
28 |
29 | kafka-encryption-generatedkey
30 | Kafka Record Encryption - generated keys
31 |
32 |
33 |
34 |
35 | io.quicksign
36 | kafka-encryption-core
37 |
38 |
39 |
40 | org.slf4j
41 | slf4j-api
42 |
43 |
44 |
45 | org.mockito
46 | mockito-core
47 | 1.10.19
48 | test
49 |
50 |
51 |
52 | junit
53 | junit
54 | 4.13.1
55 | test
56 |
57 |
58 |
59 | org.assertj
60 | assertj-core
61 | 3.6.2
62 | test
63 |
64 |
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/keyrepository/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 |
21 | io.quicksign
22 | kafka-encryption-parent
23 | 0.0.0-SNAPSHOT
24 |
25 |
26 | kafka-encryption-keyrepository
27 | jar
28 |
29 | kafka-encryption-keyrepository
30 | Kafka Record Encryption - key repository
31 |
32 |
33 |
34 |
35 | io.quicksign
36 | kafka-encryption-core
37 |
38 |
39 |
40 | org.slf4j
41 | slf4j-api
42 |
43 |
44 |
45 | org.mockito
46 | mockito-core
47 | 1.10.19
48 | test
49 |
50 |
51 |
52 | junit
53 | junit
54 | 4.13.1
55 | test
56 |
57 |
58 |
59 | org.assertj
60 | assertj-core
61 | 3.6.2
62 | test
63 |
64 |
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/core/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 |
21 | io.quicksign
22 | kafka-encryption-parent
23 | 0.0.0-SNAPSHOT
24 |
25 |
26 | kafka-encryption-core
27 | jar
28 |
29 | kafka-encryption-core
30 | Kafka Record Encryption
31 |
32 |
33 |
34 |
35 | org.slf4j
36 | slf4j-api
37 |
38 |
39 |
40 | org.apache.kafka
41 | kafka-clients
42 |
43 |
44 |
45 | org.apache.kafka
46 | kafka-streams
47 |
48 |
49 |
50 | org.mockito
51 | mockito-core
52 | 1.10.19
53 | test
54 |
55 |
56 |
57 | junit
58 | junit
59 | 4.13.1
60 | test
61 |
62 |
63 |
64 | org.assertj
65 | assertj-core
66 | 3.6.2
67 | test
68 |
69 |
70 |
71 |
72 |
73 |
--------------------------------------------------------------------------------
/samples/generatedkey-sample/src/main/java/io/quicksign/kafka/crypto/samples/generatedkey/KeyStoreBasedMasterKey.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.generatedkey;
21 |
22 | import java.io.File;
23 | import java.io.FileInputStream;
24 | import java.io.InputStream;
25 | import java.security.Key;
26 | import java.security.KeyStore;
27 |
28 | import io.quicksign.kafka.crypto.generatedkey.MasterKeyEncryption;
29 |
30 | public class KeyStoreBasedMasterKey implements MasterKeyEncryption {
31 |
32 | private final Key masterKey;
33 | private final AesGcmNoPaddingCryptoAlgorithm aesGcmNoPaddingCryptoAlgorithm;
34 |
35 |
36 | public KeyStoreBasedMasterKey(File masterKeyFile, String masterKeyPass, String alias, AesGcmNoPaddingCryptoAlgorithm aesGcmNoPaddingCryptoAlgorithm) {
37 | this.aesGcmNoPaddingCryptoAlgorithm = aesGcmNoPaddingCryptoAlgorithm;
38 | try (InputStream keystoreStream = new FileInputStream(masterKeyFile)) {
39 | KeyStore keystore = KeyStore.getInstance("PKCS12");
40 | keystore.load(keystoreStream, masterKeyPass.toCharArray());
41 | if (!keystore.containsAlias(alias)) {
42 | throw new RuntimeException("Alias for key not found");
43 | }
44 | masterKey = keystore.getKey(alias, masterKeyPass.toCharArray());
45 |
46 | }
47 | catch (Exception e) {
48 | throw new RuntimeException(e);
49 | }
50 | }
51 |
52 |
53 | @Override
54 | public byte[] encryptKey(byte[] key) {
55 | try {
56 | return aesGcmNoPaddingCryptoAlgorithm.encrypt(key, masterKey.getEncoded());
57 | }
58 | catch (Exception e) {
59 | throw new RuntimeException(e);
60 | }
61 | }
62 |
63 | @Override
64 | public byte[] decryptKey(byte[] encryptedKey) {
65 | try {
66 | return aesGcmNoPaddingCryptoAlgorithm.decrypt(encryptedKey, masterKey.getEncoded());
67 | }
68 | catch (Exception e) {
69 | return null;
70 | }
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/KeyStoreBasedKeyRepository.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.stream.keyrepo;
21 |
22 | import java.io.File;
23 | import java.io.FileInputStream;
24 | import java.io.InputStream;
25 | import java.security.Key;
26 | import java.security.KeyStore;
27 | import java.util.Optional;
28 | import java.util.concurrent.ConcurrentHashMap;
29 | import java.util.concurrent.ConcurrentMap;
30 |
31 | import io.quicksign.kafka.crypto.keyrepository.KeyRepository;
32 |
33 | public class KeyStoreBasedKeyRepository implements KeyRepository {
34 |
35 | private final KeyStore keyStore;
36 | private final ConcurrentMap> keyCache = new ConcurrentHashMap<>();
37 | private final String keyPass;
38 |
39 | public KeyStoreBasedKeyRepository(File masterKeyFile, String keyPass) {
40 | this.keyPass = keyPass;
41 | try (InputStream keystoreStream = new FileInputStream(masterKeyFile)) {
42 | KeyStore keystore = KeyStore.getInstance("PKCS12");
43 | keystore.load(keystoreStream, keyPass.toCharArray());
44 |
45 | this.keyStore = keystore;
46 | }
47 | catch (Exception e) {
48 | throw new RuntimeException(e);
49 | }
50 | }
51 |
52 |
53 | @Override
54 | public Optional getKey(String keyName) {
55 | Optional maybeKey = keyCache.computeIfAbsent(keyName, this::loadKey);
56 |
57 | return maybeKey.map(Key::getEncoded);
58 | }
59 |
60 | private synchronized Optional loadKey(String keyName) {
61 | try {
62 | if (keyStore.containsAlias(keyName)) {
63 | return Optional.ofNullable(keyStore.getKey(keyName, keyPass.toCharArray()));
64 | }
65 | else {
66 | return Optional.empty();
67 | }
68 | }
69 | catch (Exception e) {
70 | throw new RuntimeException("error while loading key", e);
71 | }
72 | }
73 | }
74 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/SampleRawConsumer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import java.util.Collections;
23 | import java.util.Properties;
24 |
25 | import org.apache.kafka.clients.consumer.ConsumerConfig;
26 | import org.apache.kafka.clients.consumer.ConsumerRecords;
27 | import org.apache.kafka.clients.consumer.KafkaConsumer;
28 | import org.apache.kafka.common.serialization.LongDeserializer;
29 | import org.apache.kafka.common.serialization.StringDeserializer;
30 |
31 | // This consumer does not decrypt the messages... it is here to show you that the data
32 | // is encrypted...
33 | public class SampleRawConsumer implements Runnable {
34 |
35 | public SampleRawConsumer() {
36 | }
37 |
38 | @Override
39 | public void run() {
40 |
41 | Properties consumerProperties = new Properties();
42 | consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
43 | consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "sampleraw");
44 | consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
45 |
46 | try (KafkaConsumer consumer = new KafkaConsumer(
47 | consumerProperties,
48 | new LongDeserializer(),
49 | new StringDeserializer())) {
50 |
51 | consumer.subscribe(Collections.singleton("sampletopic"));
52 | for (; true; ) {
53 | ConsumerRecords records = consumer.poll(1000L);
54 | records.forEach(
55 | record -> System.out.println(
56 | "-------------------------------------------------------------\n" +
57 | "raw record: key=" + record.key() + ", offset=" + record.offset() + ", value=" + record.value() +
58 | "\n-------------------------------------------------------------\n\n")
59 | );
60 | }
61 | }
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/SamplesMain.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import java.util.concurrent.ExecutorService;
23 | import java.util.concurrent.Executors;
24 |
25 | import io.quicksign.kafka.crypto.encryption.CryptoAlgorithm;
26 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
27 | import io.quicksign.kafka.crypto.keyrepository.RepositoryBasedKeyProvider;
28 | import io.quicksign.kafka.crypto.keyrepository.RepositoryBasedKeyReferenceExtractor;
29 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
30 |
31 | public class SamplesMain {
32 |
33 |
34 | public static void main(String... args) {
35 |
36 | // tag::main[]
37 |
38 | // 1- SETUP
39 |
40 | // key provider backed by a simple in memory key repository
41 | KeyProvider keyProvider = new RepositoryBasedKeyProvider(new SampleKeyRepository(), new SampleKeyNameObfuscator());
42 |
43 | // We create an encryption keyref for each record's key.
44 | // Two records with the same record key have the same encryption key ref.
45 | KeyReferenceExtractor keyReferenceExtractor = new RepositoryBasedKeyReferenceExtractor(new SampleKeyNameExtractor(), new SampleKeyNameObfuscator());
46 |
47 | // The payload is encrypted using AES
48 | CryptoAlgorithm cryptoAlgorithm = new AesGcmNoPaddingCryptoAlgorithm();
49 |
50 |
51 | // 2- RUN
52 |
53 | ExecutorService executorService = Executors.newFixedThreadPool(3);
54 |
55 | // the producer encrypts the message
56 | executorService.submit(new SampleProducer(keyProvider, keyReferenceExtractor, cryptoAlgorithm));
57 |
58 | // this consumer reads them but don't decrypt them... so you can see that it can't be read by default
59 | executorService.submit(new SampleRawConsumer());
60 |
61 | // this consumer reads and decrypts... and dump in clear the payload...
62 | executorService.submit(new SampleDecryptingConsumer(keyProvider, cryptoAlgorithm));
63 |
64 | // end::main[]
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/samples/generatedkey-sample/src/main/java/io/quicksign/kafka/crypto/samples/generatedkey/SampleRawConsumer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.generatedkey;
21 |
22 | import java.util.Collections;
23 | import java.util.Properties;
24 |
25 | import org.apache.kafka.clients.consumer.ConsumerConfig;
26 | import org.apache.kafka.clients.consumer.ConsumerRecords;
27 | import org.apache.kafka.clients.consumer.KafkaConsumer;
28 | import org.apache.kafka.common.serialization.LongDeserializer;
29 | import org.apache.kafka.common.serialization.StringDeserializer;
30 |
31 | // This consumer does not decrypt the messages... it is here to show you that the data
32 | // is encrypted...
33 | public class SampleRawConsumer implements Runnable {
34 |
35 | public SampleRawConsumer() {
36 | }
37 |
38 |
39 | @Override
40 | public void run() {
41 |
42 | Properties consumerProperties = new Properties();
43 | consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
44 | consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "sampleraw");
45 | consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
46 |
47 | try (KafkaConsumer consumer = new KafkaConsumer(
48 | consumerProperties,
49 | new LongDeserializer(),
50 | new StringDeserializer())) {
51 |
52 | consumer.subscribe(Collections.singleton("sampletopic"));
53 | for (; true; ) {
54 | ConsumerRecords records = consumer.poll(1000L);
55 | records.forEach(
56 | record -> System.out.println(
57 | "-------------------------------------------------------------\n" +
58 | "raw record: key=" + record.key() + ", offset=" + record.offset() + ", value=" + record.value() +
59 | "\n-------------------------------------------------------------\n\n"
60 | )
61 | );
62 | }
63 | }
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/samples/generatedkey-sample/src/main/java/io/quicksign/kafka/crypto/samples/generatedkey/SamplesMain.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.generatedkey;
21 |
22 | import java.io.File;
23 | import java.util.Arrays;
24 | import java.util.Optional;
25 | import java.util.Set;
26 | import java.util.concurrent.ExecutorService;
27 | import java.util.concurrent.Executors;
28 | import java.util.function.Function;
29 | import java.util.stream.Collectors;
30 |
31 | import io.quicksign.kafka.crypto.generatedkey.MasterKeyEncryption;
32 |
33 | public class SamplesMain {
34 |
35 | public static void main(String... args) {
36 | Set modules = Arrays.stream(args)
37 | .map(Modules::getByOption)
38 | .filter(Optional::isPresent)
39 | .map(Optional::get)
40 | .collect(Collectors.toSet());
41 |
42 | // tag::masterkey[]
43 | MasterKeyEncryption masterKeyEncryption = new KeyStoreBasedMasterKey(
44 | new File("/tmp/sample.pkcs12"),
45 | "sample", "sample",
46 | new AesGcmNoPaddingCryptoAlgorithm()
47 | );
48 | // end::masterkey[]
49 |
50 |
51 | ExecutorService executorService = Executors.newFixedThreadPool(3);
52 |
53 | for (Modules activeModule : modules) {
54 | Runnable task = activeModule.getBuilder().apply(masterKeyEncryption);
55 | executorService.submit(task);
56 | }
57 | }
58 |
59 | private enum Modules {
60 | PRODUCER("--producer", mk -> new SampleProducer(mk)),
61 | CONSUMER("--consumer", mk -> new SampleDecryptingConsumer(mk)),
62 | RAW_CONSUMER("--rawConsumer", mk -> new SampleRawConsumer());
63 |
64 | private final String option;
65 | private final Function builder;
66 |
67 | Modules(String option, Function builder) {
68 | this.option = option;
69 | this.builder = builder;
70 | }
71 |
72 | public static Optional getByOption(String option) {
73 | return Arrays.stream(values()).filter(m -> m.option.equals(option)).findFirst();
74 | }
75 |
76 | public Function getBuilder() {
77 | return builder;
78 | }
79 | }
80 | }
81 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: java
2 | install: "/bin/true"
3 | script:
4 | - "./test"
5 | env:
6 | global:
7 | - secure: ug5U2l40AI2F3gpzo6YN+pWFrzxCayb6PSjH2n5fn4kr6s6BDdNHYjUHp+mtsTSvXGzBNJcxIcmuulWFsTV1iSPJaYX5Ke7TjgI8A4pSTytNnuxVOLTENc9T8+uKgq0n0BPPH0ru2o90D8pqIz+6xlg956AUvTqkZVP94ydhT1Z89Pq+LGJVTRLnOXMPvYl9XqtusGB/oVUXIjuzqMS1FHcTru352Zn53dgd4h+5g8ikh5a/qiygRbpiyBPvjXz//ft0iN8dr7iOaxAFoS93nNqxnHpJ058T49bMhD6V82BraYnZG8wlDxe+kRaFbBWC24rsP9Aov1jcA8x/q50MC2BTQ8B8I2QEd+bxkhcPgqmNKfDY5fP75tPeYb9ZBC1zD9XM1U/lmJd5isYTw2uxkV+uhBxYOhXDY2g3BbOwD5a/7YiRJd2/UGdMJHmYC+m09Cr4SPzNjkoJZBdRaGUPMeAwUvU5mI4E+yW7N21INuPFXX4GIAKBxC4QKOt5bAbW0D3shYKBcjJ+HUUz0b6XyLHmRD1v7MPRUWQITJWEOuH5/lhut4MKkUxjv0b1O6HMXrBpYoEZ3seIKqXaRlONMQoYhwWHTABPG9GpZrVzKRyTXWORgbppAVveJd88M8asiR7vZbx8u3dQEJ/m6vC7Qu9U1lM0ZdI/ZwYY432r1J4=
8 | - secure: Y+d6V70Kv3QuamZ6AVKKs4tcvFCnBwCzcWISsTCwoa9aiVV3lItubi4xhX6w+7x2XInSYuWSEDiMDIPgtuQthSJ66749gMuwzNB+JOMdX6ou01Fgz3Ng4JvKcAkCMaUlm40pg8SlbyLpigyjrF4yyq+KsJrX8n6OxW7MdH3+6LFztUdmIEVnFchb4xYcJ/Kuj8Z9Qv6++QeDdGeuhCy0SKskAq3KnViNcjKez1+El0OMLwYCrAcjca+9/AfuMcG4hBDpsvxvX5hHsllyMq7HZnrGidEqGNjNMAOKRfNslPtg0NFYv3xUZxP9NEylHCe25lc5tmrpXWEZYn9a79uuvw3SAl1uCuZwMHvijNf04y3eJQH2PKuaa98EesjzEmHY/m8Q4ciXS6h0Wi4HzWLr/l9onJvA0c2ibKgfZ0LOmZ1cQdHYt9VsFEognvLiLFDzPsJ/gSzNKOWRqa7VwdwI6eskEf+TknXfGXhwjz5M03lZVyZBHFlrY2SxGBnv1qf+xotU7jF6owj3/clVxPy0bJFVQZDUVOPi6xe+OFp954q6DtWast+HnEdztUcghw4gdJ1v57WTbp8ShdNc53VyUUSa805tYyKcmhUuKwPdcUNqWZgRTxa6+OQF4PUf6ED0G5ClY9bJUDnJRJgq1MgW0Vb3PG0iDeEVitKb8wP7gww=
9 | - secure: NKp7vfUdgVMzjmOUva+8uuSt/p5PRxHm72OfhcTPvgBoo1KqZcvF8vU5HlxCau3lwQs9H33K3SC+NG1jl2QJtW4gRprRBig5piEHsbEuIYtscNWpszaIJH7hlNW/dUiXil6Ml1Nn2ZAYjxuBKIa/YMIjmFYK5O2dXfx1qFRgmUFlTXWtInLoLwTtBu6FIlN7sdo0V5tooZTYHQj5krTLf14X2ztNyUjM+xX/yCFstt8xVXv/Ubf9RYKupANtJUUyAvKl6KI6T+hcPbdd96BVdLbLLV8aBSF6+GQctfs9xIRycXvUYXhiu992A5Ix/H5npyD5Fc9xParmnTCU//D6/gYTLHl0tpbl7imjTcKWlfcqZTTLrUId+m19NUJQ/05WkmPnALO4uxq8ZiFyLFtEFzd1QttKM8OR/qhJcqnJm1wGsNogRxDIADID+fLfV7B+6YOU7HgBvKtoDq/LVCccYUDG8vlXfLPs29egzpWGcrxaEGfRqUyfgrTW8wxjROHIEPk+blTl4OqshRWlVlfH+ctShTgAPh682fEooXSqM1xm8VRY5dVBLX0OHMJsjgjCqrc88PcY2Ndt9lqQlLW8JvzglHdzHEcYDJZzH41SHbsT+ah2NJASBEuBVRJo4kC/ol2xPmbC7cM9bUOmCF/cB5v3DmUhnVbVFGxsjOpIypo=
10 | - secure: CdSu+A26H4ritU8FqgKpqHkMR+rYTBH2eMBAmGtbuN4T0X10bJBGLGAHsKXdLne+YoiLEe75AhzCmh36Bw6hQ0J2c6MQL7T25rmb7iGaNeDM3U0BxCM/iZKFMNevqxX8fqWNkc3vx7UZkqP/eXxr5kQYZdNRGxuVTKrHrbARZdW6pLfvAbCJMO88f8irOsYi1T8qG+LJjW3lkNCq4vkf0AX9EN4Az5HK7yIZMgbpYa+3wNgmCyfFEn7rT+LMl0ZFhugKx0VVnOuzEyz9w05jlu91XvnkQiohoMbqQATr73yan99OeJLy84cd2tQFz0iqaE1DHW4ZzBLh0l0PUEmrCn3KwKnsOBkjKMDg2nwkU1kMjo7HwXoMzdwCS11aCyhaASv8Cef+vY7c41UvMSvI/Lpq6KmFmPzm1Wd7tZ01BJJf/Q6Ot6wIYk0R2NHAydMYtPFvbz3ydNDvMHePUKXROP1rvbpfjoCAzVNn7aiXNOI+mT6Iy/qiIlkJgo/BG+80QIPx4XHoitvVnxwQ46G+gfmRVZMqpgybnJPn/cKAOeCsqBI/RxeTSs3FbXAEEXKhnTygF2NGk7q14oQJ+xooT3mm9RAiKiLkRmctzSYSyPTg9/wVlp845e7ciEgGC+tyLcOb52geKTre8xjInTHDcrLUoFjUzj1Mtu3z3HJ/mcs=
11 | before_deploy:
12 | - openssl aes-256-cbc -K $encrypted_4f0d00631887_key -iv $encrypted_4f0d00631887_iv
13 | -in .travis/gpg.asc.enc -out .travis/gpg.asc -d
14 | deploy:
15 | - skip_cleanup: true
16 | provider: script
17 | script: "./deploy"
18 | on:
19 | branch: master
20 | - skip_cleanup: true
21 | provider: script
22 | script: "./deploy"
23 | on:
24 | tags: true
25 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/serdes/SerdesPair.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.serdes;
21 |
22 | import org.apache.kafka.common.serialization.Serde;
23 | import org.apache.kafka.streams.kstream.Consumed;
24 | import org.apache.kafka.streams.kstream.Grouped;
25 | import org.apache.kafka.streams.kstream.Materialized;
26 | import org.apache.kafka.streams.kstream.Produced;
27 | import org.apache.kafka.streams.kstream.Serialized;
28 | import org.apache.kafka.streams.processor.StateStore;
29 |
30 | /**
31 | * Represent paired keySerde and valueSerde
32 | *
33 | * @param
34 | * @param
35 | */
36 | public class SerdesPair {
37 |
38 | private final Serde keySerde;
39 | private final Serde valueSerde;
40 |
41 | public SerdesPair(Serde keySerde, Serde valueSerde) {
42 |
43 | this.keySerde = keySerde;
44 | this.valueSerde = valueSerde;
45 | }
46 |
47 | public Serde getKeySerde() {
48 | return keySerde;
49 | }
50 |
51 | public Serde getValueSerde() {
52 | return valueSerde;
53 | }
54 |
55 | /**
56 | * Build a {@link Serialized} using the keySerde and valueSerde of the pair
57 | *
58 | * @return
59 | */
60 | public Serialized toSerialized() {
61 | return Serialized.with(keySerde, valueSerde);
62 | }
63 |
64 | /**
65 | * Build a {@link Grouped} using the keySerde and valueSerde of the pair
66 | */
67 | public Grouped toGrouped() {
68 | return Grouped.with(keySerde, valueSerde);
69 | }
70 |
71 | /**
72 | * Build a {@link Produced} using the keySerde and valueSerde of the pair
73 | *
74 | * @return
75 | */
76 | public Produced toProduced() {
77 | return Produced.with(keySerde, valueSerde);
78 | }
79 |
80 | /**
81 | * Build a {@link Consumed} using the keySerde and valueSerde of the pair
82 | *
83 | * @return
84 | */
85 | public Consumed toConsumed() {
86 | return Consumed.with(keySerde, valueSerde);
87 | }
88 |
89 |
90 | /**
91 | * Apply the keySerde and valueSerde of the pair to a {@link Materialized}
92 | *
93 | * @param materialized
94 | * @param
95 | * @return
96 | */
97 | public Materialized applyTo(Materialized materialized) {
98 | return materialized.withKeySerde(keySerde).withValueSerde(valueSerde);
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/SampleProducer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import java.util.Properties;
23 |
24 | import org.apache.kafka.clients.producer.KafkaProducer;
25 | import org.apache.kafka.clients.producer.ProducerConfig;
26 | import org.apache.kafka.clients.producer.ProducerRecord;
27 | import org.apache.kafka.common.serialization.LongSerializer;
28 | import org.apache.kafka.common.serialization.StringSerializer;
29 |
30 | import io.quicksign.kafka.crypto.Encryptor;
31 | import io.quicksign.kafka.crypto.encryption.CryptoAlgorithm;
32 | import io.quicksign.kafka.crypto.encryption.DefaultEncryptor;
33 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
34 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
35 | import io.quicksign.kafka.crypto.pairing.serializer.CryptoSerializerPairFactory;
36 | import io.quicksign.kafka.crypto.pairing.serializer.SerializerPair;
37 |
38 | public class SampleProducer implements Runnable {
39 |
40 | private final KeyProvider keyProvider;
41 | private final KeyReferenceExtractor keyReferenceExtractor;
42 | private final CryptoAlgorithm cryptoAlgorithm;
43 |
44 | public SampleProducer(KeyProvider keyProvider, KeyReferenceExtractor keyReferenceExtractor, CryptoAlgorithm cryptoAlgorithm) {
45 | this.keyProvider = keyProvider;
46 | this.keyReferenceExtractor = keyReferenceExtractor;
47 | this.cryptoAlgorithm = cryptoAlgorithm;
48 | }
49 |
50 | @Override
51 | public void run() {
52 |
53 | // tag::produce[]
54 |
55 | Encryptor encryptor = new DefaultEncryptor(keyProvider, cryptoAlgorithm);
56 |
57 | // Wrap base LongSerializer and StringSerializer with encrypted wrappers
58 | CryptoSerializerPairFactory cryptoSerializerPairFactory = new CryptoSerializerPairFactory(encryptor, keyReferenceExtractor);
59 | SerializerPair serializerPair = cryptoSerializerPairFactory.build(new LongSerializer(), new StringSerializer());
60 |
61 | Properties producerProperties = new Properties();
62 | producerProperties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
63 |
64 | try (KafkaProducer producer =
65 | new KafkaProducer<>(producerProperties, serializerPair.getKeySerializer(), serializerPair.getValueSerializer())) {
66 |
67 | for (long i = 0L; i < Long.MAX_VALUE; i++) {
68 | producer.send(new ProducerRecord<>("sampletopic", i, "test number " + i));
69 | try {
70 | Thread.sleep(1000L);
71 | }
72 | catch (InterruptedException e) {
73 | return;
74 | }
75 | }
76 | }
77 | // end::produce[]
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/SampleDecryptingConsumer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import java.util.Collections;
23 | import java.util.Properties;
24 |
25 | import org.apache.kafka.clients.consumer.ConsumerConfig;
26 | import org.apache.kafka.clients.consumer.ConsumerRecords;
27 | import org.apache.kafka.clients.consumer.KafkaConsumer;
28 | import org.apache.kafka.common.serialization.LongDeserializer;
29 | import org.apache.kafka.common.serialization.StringDeserializer;
30 |
31 | import io.quicksign.kafka.crypto.CryptoDeserializerFactory;
32 | import io.quicksign.kafka.crypto.Decryptor;
33 | import io.quicksign.kafka.crypto.encryption.CryptoAlgorithm;
34 | import io.quicksign.kafka.crypto.encryption.DefaultDecryptor;
35 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
36 |
37 | public class SampleDecryptingConsumer implements Runnable {
38 |
39 | private final KeyProvider keyProvider;
40 | private final CryptoAlgorithm cryptoAlgorithm;
41 |
42 | public SampleDecryptingConsumer(KeyProvider keyProvider, CryptoAlgorithm cryptoAlgorithm) {
43 | this.keyProvider = keyProvider;
44 | this.cryptoAlgorithm = cryptoAlgorithm;
45 |
46 | }
47 |
48 | @Override
49 | public void run() {
50 |
51 | // tag::consume[]
52 | // The key is embedded in each message
53 |
54 | Decryptor decryptor = new DefaultDecryptor(keyProvider, cryptoAlgorithm);
55 |
56 | // Construct decrypting deserializer
57 | CryptoDeserializerFactory cryptoDeserializerFactory = new CryptoDeserializerFactory(decryptor);
58 |
59 | Properties consumerProperties = new Properties();
60 | consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
61 | consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "samplecrypted");
62 | consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
63 |
64 | try (KafkaConsumer consumer = new KafkaConsumer(
65 | consumerProperties,
66 | new LongDeserializer(),
67 | cryptoDeserializerFactory.buildFrom(new StringDeserializer()))) {
68 |
69 | consumer.subscribe(Collections.singleton("sampletopic"));
70 | for (; true; ) {
71 | ConsumerRecords records = consumer.poll(1000L);
72 | records.forEach(
73 | record -> System.out.println(
74 | "-------------------------------------------------------------\n" +
75 | "decrypted record: key=" + record.key() + ", offset=" + record.offset() + ", value=" + record.value() +
76 | "\n-------------------------------------------------------------\n\n")
77 | );
78 | }
79 | }
80 | // end::consume[]
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SamplesMain.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.stream.keyrepo;
21 |
22 | import java.io.File;
23 | import java.util.concurrent.ExecutorService;
24 | import java.util.concurrent.Executors;
25 |
26 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
27 | import io.quicksign.kafka.crypto.keyrepository.KeyRepository;
28 | import io.quicksign.kafka.crypto.keyrepository.RepositoryBasedKeyProvider;
29 | import io.quicksign.kafka.crypto.keyrepository.RepositoryBasedKeyReferenceExtractor;
30 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
31 |
32 | public class SamplesMain {
33 |
34 | public static void main(String... args) {
35 |
36 | // tag::main[]
37 | KeyRepository fullKeyRepository = new KeyStoreBasedKeyRepository(
38 | new File("/tmp/samplestream.pkcs12"),
39 | "sample"
40 | );
41 |
42 | KeyRepository agency1KeyRepository = new KeyStoreBasedKeyRepository(
43 | new File("/tmp/samplestream1.pkcs12"),
44 | "sample"
45 | );
46 |
47 | KeyRepository agency2KeyRepository = new KeyStoreBasedKeyRepository(
48 | new File("/tmp/samplestream2.pkcs12"),
49 | "sample"
50 | );
51 |
52 | KeyProvider fullKeyProvider = new RepositoryBasedKeyProvider(fullKeyRepository, new SampleKeyNameObfuscator());
53 | KeyProvider agency1KeyProvider = new RepositoryBasedKeyProvider(agency1KeyRepository, new SampleKeyNameObfuscator());
54 | KeyProvider agency2KeyProvider = new RepositoryBasedKeyProvider(agency2KeyRepository, new SampleKeyNameObfuscator());
55 |
56 | // We create an encryption keyref for each record's key.
57 | // Two records with the same record key have the same encryption key ref.
58 | KeyReferenceExtractor keyReferenceExtractor = new RepositoryBasedKeyReferenceExtractor(new SampleKeyNameExtractor(), new SampleKeyNameObfuscator());
59 |
60 |
61 | ExecutorService executorService = Executors.newFixedThreadPool(4);
62 |
63 |
64 | SampleProducer sampleProducer = new SampleProducer(fullKeyProvider, keyReferenceExtractor);
65 |
66 | executorService.submit(sampleProducer);
67 |
68 | try {
69 | Thread.sleep(10000l);
70 | }
71 | catch (InterruptedException e) {
72 | System.exit(0);
73 | }
74 |
75 | SampleStream fullView = new SampleStream("full", fullKeyProvider, keyReferenceExtractor);
76 | SampleStream agency1View = new SampleStream("agency1", agency1KeyProvider, keyReferenceExtractor);
77 | SampleStream agency2View = new SampleStream("agency2", agency2KeyProvider, keyReferenceExtractor);
78 |
79 | executorService.submit(fullView);
80 | executorService.submit(agency1View);
81 | executorService.submit(agency2View);
82 |
83 | // end::main[]
84 |
85 | }
86 |
87 |
88 | }
89 |
--------------------------------------------------------------------------------
/samples/generatedkey-sample/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 | io.quicksign
21 | kafka-encryption-generatedkey-sample
22 | 0.0.0-SNAPSHOT
23 | jar
24 | kafka-encryption-generatedkey-sample
25 | Kafka Record Encryption - generated keys sample
26 |
27 |
28 | 1.8
29 | 1.8
30 | 1.8
31 | UTF-8
32 |
33 | 0.2.0
34 |
35 |
36 |
37 |
38 |
39 | io.quicksign
40 | kafka-encryption-core
41 | ${kafka-encryption.version}
42 |
43 |
44 |
45 | io.quicksign
46 | kafka-encryption-generatedkey
47 | ${kafka-encryption.version}
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 | org.apache.maven.plugins
56 | maven-shade-plugin
57 | 3.2.0
58 |
59 |
60 | package
61 |
62 | shade
63 |
64 |
65 | true
66 |
67 |
68 | io.quicksign.kafka.crypto.samples.generatedkey.SamplesMain
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 | org.apache.maven.plugins
77 | maven-deploy-plugin
78 | 2.7
79 |
80 | true
81 |
82 |
83 |
84 |
85 |
86 |
87 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/keyrepo/AesGcmNoPaddingCryptoAlgorithm.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.keyrepo;
21 |
22 | import java.io.ByteArrayOutputStream;
23 | import java.nio.ByteBuffer;
24 | import java.nio.charset.StandardCharsets;
25 | import java.security.NoSuchAlgorithmException;
26 | import java.security.SecureRandom;
27 |
28 | import javax.crypto.Cipher;
29 | import javax.crypto.CipherOutputStream;
30 | import javax.crypto.spec.GCMParameterSpec;
31 | import javax.crypto.spec.SecretKeySpec;
32 |
33 | import io.quicksign.kafka.crypto.encryption.CryptoAlgorithm;
34 |
35 | public class AesGcmNoPaddingCryptoAlgorithm implements CryptoAlgorithm {
36 |
37 | private static final String KEY_SPEC = "AES";
38 | private static final String ALGO_TRANSFORMATION_STRING = "AES/GCM/NoPadding";
39 | private static int IV_SIZE = 96;
40 | private static int TAG_BIT_LENGTH = 128;
41 | private static String TAG = "sample";
42 |
43 |
44 | private final SecureRandom secureRandom;
45 |
46 | public AesGcmNoPaddingCryptoAlgorithm() {
47 | try {
48 | this.secureRandom = SecureRandom.getInstance("SHA1PRNG");
49 | }
50 | catch (NoSuchAlgorithmException e) {
51 | throw new RuntimeException("unable to init secureRandom", e);
52 | }
53 | }
54 |
55 | @Override
56 | public byte[] encrypt(byte[] data, byte[] key) throws Exception {
57 | SecretKeySpec secretKeySpec = new SecretKeySpec(key, KEY_SPEC);
58 | byte[] iv = new byte[IV_SIZE];
59 | secureRandom.nextBytes(iv);
60 | GCMParameterSpec gcmParamSpec = new GCMParameterSpec(TAG_BIT_LENGTH, iv);
61 | Cipher cipher = Cipher.getInstance(ALGO_TRANSFORMATION_STRING);
62 | cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec, gcmParamSpec, secureRandom);
63 | cipher.updateAAD(TAG.getBytes(StandardCharsets.UTF_8));
64 |
65 | ByteArrayOutputStream baos = new ByteArrayOutputStream();
66 | baos.write(iv);
67 |
68 | try (CipherOutputStream cipherOutputStream = new CipherOutputStream(baos, cipher)) {
69 | cipherOutputStream.write(data);
70 | }
71 |
72 | return baos.toByteArray();
73 | }
74 |
75 | @Override
76 | public byte[] decrypt(byte[] encryptedData, byte[] key) throws Exception {
77 | SecretKeySpec secretKeySpec = new SecretKeySpec(key, KEY_SPEC);
78 | ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
79 | byte[] iv = new byte[IV_SIZE];
80 | byteBuffer.get(iv);
81 | GCMParameterSpec gcmParamSpec = new GCMParameterSpec(TAG_BIT_LENGTH, iv);
82 | Cipher cipher = Cipher.getInstance(ALGO_TRANSFORMATION_STRING);
83 | cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, gcmParamSpec, secureRandom);
84 | cipher.updateAAD(TAG.getBytes(StandardCharsets.UTF_8));
85 |
86 | byte[] encryptedPayload = new byte[byteBuffer.remaining()];
87 | byteBuffer.get(encryptedPayload);
88 |
89 | return cipher.doFinal(encryptedPayload);
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/internal/CryptoAwareSerializerWrapper.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.internal;
21 |
22 | import java.util.Map;
23 |
24 | import org.apache.kafka.common.header.Headers;
25 | import org.apache.kafka.common.serialization.Serializer;
26 |
27 | import io.quicksign.kafka.crypto.KafkaCryptoConstants;
28 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
29 |
30 | /**
31 | * A wrapper for {@link Serializer}.
32 | * It will call the {@link KeyReferenceExtractor} and then serialize the data using the underlying serializer
33 | *
34 | * @param
35 | */
36 | public class CryptoAwareSerializerWrapper implements Serializer {
37 |
38 | private final Serializer rawSerializer;
39 | private final KeyReferenceExtractor keyReferenceExtractor;
40 | private final ThreadLocal keyRefHolder;
41 |
42 | /**
43 | * @param rawSerializer the Serializer to use
44 | * @param keyReferenceExtractor the KeyReferenceExtractor to use
45 | * @param keyRefHolder the ThreadLocal to share the keyref (only used in the context of a Kafka Stream)
46 | */
47 | public CryptoAwareSerializerWrapper(Serializer rawSerializer, KeyReferenceExtractor keyReferenceExtractor, ThreadLocal keyRefHolder) {
48 | this.rawSerializer = rawSerializer;
49 | this.keyReferenceExtractor = keyReferenceExtractor;
50 | this.keyRefHolder = keyRefHolder;
51 | }
52 |
53 | @Override
54 | public void configure(Map configs, boolean isKey) {
55 | this.rawSerializer.configure(configs, isKey);
56 | }
57 |
58 | /**
59 | * Call the KeyReferenceExtractor with the topic and the data and set the computed value into the ThreadLocal reference holder.
60 | *
61 | * This method is called in the context of a kafka stream and not in the Kafka Producer
62 | *
63 | * @param topic
64 | * @param data
65 | * @return the result of the underlying serializer
66 | */
67 | @Override
68 | public byte[] serialize(String topic, T data) {
69 | this.keyRefHolder.set(keyReferenceExtractor.extractKeyReference(topic, data));
70 |
71 | return this.rawSerializer.serialize(topic, data);
72 | }
73 |
74 | @Override
75 | public void close() {
76 | this.rawSerializer.close();
77 | }
78 |
79 | /**
80 | * Call the KeyReferenceExtractor with the topic and the data and set the result in the kafka header {@link KafkaCryptoConstants#KEY_REF_HEADER}
81 | *
82 | * This method is called by the Kafka Producer
83 | *
84 | * @param topic
85 | * @param headers
86 | * @param data
87 | * @return the result of the underlying serializer
88 | */
89 | @Override
90 | public byte[] serialize(String topic, Headers headers, T data) {
91 | headers.add(KafkaCryptoConstants.KEY_REF_HEADER, keyReferenceExtractor.extractKeyReference(topic, data));
92 | return this.rawSerializer.serialize(topic, headers, data);
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/samples/generatedkey-sample/src/main/java/io/quicksign/kafka/crypto/samples/generatedkey/AesGcmNoPaddingCryptoAlgorithm.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.generatedkey;
21 |
22 | import java.io.ByteArrayOutputStream;
23 | import java.nio.ByteBuffer;
24 | import java.nio.charset.StandardCharsets;
25 | import java.security.NoSuchAlgorithmException;
26 | import java.security.SecureRandom;
27 |
28 | import javax.crypto.Cipher;
29 | import javax.crypto.CipherOutputStream;
30 | import javax.crypto.spec.GCMParameterSpec;
31 | import javax.crypto.spec.SecretKeySpec;
32 |
33 | import io.quicksign.kafka.crypto.encryption.CryptoAlgorithm;
34 |
35 | public class AesGcmNoPaddingCryptoAlgorithm implements CryptoAlgorithm {
36 |
37 | private static final String KEY_SPEC = "AES";
38 | private static final String ALGO_TRANSFORMATION_STRING = "AES/GCM/NoPadding";
39 | private static int IV_SIZE = 96;
40 | private static int TAG_BIT_LENGTH = 128;
41 | private static String TAG = "sample";
42 |
43 |
44 | private final SecureRandom secureRandom;
45 |
46 | public AesGcmNoPaddingCryptoAlgorithm() {
47 | try {
48 | this.secureRandom = SecureRandom.getInstance("SHA1PRNG");
49 | }
50 | catch (NoSuchAlgorithmException e) {
51 | throw new RuntimeException("unable to init secureRandom", e);
52 | }
53 | }
54 |
55 | @Override
56 | public byte[] encrypt(byte[] data, byte[] key) throws Exception {
57 | SecretKeySpec secretKeySpec = new SecretKeySpec(key, KEY_SPEC);
58 | byte[] iv = new byte[IV_SIZE];
59 | secureRandom.nextBytes(iv);
60 | GCMParameterSpec gcmParamSpec = new GCMParameterSpec(TAG_BIT_LENGTH, iv);
61 | Cipher cipher = Cipher.getInstance(ALGO_TRANSFORMATION_STRING);
62 | cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec, gcmParamSpec, secureRandom);
63 | cipher.updateAAD(TAG.getBytes(StandardCharsets.UTF_8));
64 |
65 | ByteArrayOutputStream baos = new ByteArrayOutputStream();
66 | baos.write(iv);
67 |
68 | try (CipherOutputStream cipherOutputStream = new CipherOutputStream(baos, cipher)) {
69 | cipherOutputStream.write(data);
70 | }
71 |
72 | return baos.toByteArray();
73 | }
74 |
75 | @Override
76 | public byte[] decrypt(byte[] encryptedData, byte[] key) throws Exception {
77 | SecretKeySpec secretKeySpec = new SecretKeySpec(key, KEY_SPEC);
78 | ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
79 | byte[] iv = new byte[IV_SIZE];
80 | byteBuffer.get(iv);
81 | GCMParameterSpec gcmParamSpec = new GCMParameterSpec(TAG_BIT_LENGTH, iv);
82 | Cipher cipher = Cipher.getInstance(ALGO_TRANSFORMATION_STRING);
83 | cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, gcmParamSpec, secureRandom);
84 | cipher.updateAAD(TAG.getBytes(StandardCharsets.UTF_8));
85 |
86 | byte[] encryptedPayload = new byte[byteBuffer.remaining()];
87 | byteBuffer.get(encryptedPayload);
88 |
89 | return cipher.doFinal(encryptedPayload);
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 | io.quicksign
21 | kafka-encryption-kafkastream-keyrepo-sample
22 | 0.0.0-SNAPSHOT
23 | jar
24 | kafka-encryption-kafkastream-keyrepo-sample
25 | Kafka Record Encryption - stream with key repository sample
26 |
27 |
28 | 1.8
29 | 1.8
30 | 1.8
31 | UTF-8
32 |
33 | 0.2.0
34 |
35 |
36 |
37 |
38 |
39 | io.quicksign
40 | kafka-encryption-core
41 | ${kafka-encryption.version}
42 |
43 |
44 |
45 | io.quicksign
46 | kafka-encryption-keyrepository
47 | ${kafka-encryption.version}
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 | org.apache.maven.plugins
56 | maven-shade-plugin
57 | 3.2.0
58 |
59 |
60 | package
61 |
62 | shade
63 |
64 |
65 | true
66 |
67 |
68 | io.quicksign.kafka.crypto.samples.stream.keyrepo.SamplesMain
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 | org.apache.maven.plugins
77 | maven-deploy-plugin
78 | 2.7
79 |
80 | true
81 |
82 |
83 |
84 |
85 |
86 |
87 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/AesGcmNoPaddingCryptoAlgorithm.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.stream.keyrepo;
21 |
22 | import java.io.ByteArrayOutputStream;
23 | import java.nio.ByteBuffer;
24 | import java.nio.charset.StandardCharsets;
25 | import java.security.NoSuchAlgorithmException;
26 | import java.security.SecureRandom;
27 |
28 | import javax.crypto.Cipher;
29 | import javax.crypto.CipherOutputStream;
30 | import javax.crypto.spec.GCMParameterSpec;
31 | import javax.crypto.spec.SecretKeySpec;
32 |
33 | import io.quicksign.kafka.crypto.encryption.CryptoAlgorithm;
34 |
35 | public class AesGcmNoPaddingCryptoAlgorithm implements CryptoAlgorithm {
36 |
37 | private static final String KEY_SPEC = "AES";
38 | private static final String ALGO_TRANSFORMATION_STRING = "AES/GCM/NoPadding";
39 | private static int IV_SIZE = 96;
40 | private static int TAG_BIT_LENGTH = 128;
41 | private static String TAG = "sample";
42 |
43 |
44 | private final SecureRandom secureRandom;
45 |
46 | public AesGcmNoPaddingCryptoAlgorithm() {
47 | try {
48 | this.secureRandom = SecureRandom.getInstance("SHA1PRNG");
49 | }
50 | catch (NoSuchAlgorithmException e) {
51 | throw new RuntimeException("unable to init secureRandom", e);
52 | }
53 | }
54 |
55 | @Override
56 | public byte[] encrypt(byte[] data, byte[] key) throws Exception {
57 | SecretKeySpec secretKeySpec = new SecretKeySpec(key, KEY_SPEC);
58 | byte[] iv = new byte[IV_SIZE];
59 | secureRandom.nextBytes(iv);
60 | GCMParameterSpec gcmParamSpec = new GCMParameterSpec(TAG_BIT_LENGTH, iv);
61 | Cipher cipher = Cipher.getInstance(ALGO_TRANSFORMATION_STRING);
62 | cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec, gcmParamSpec, secureRandom);
63 | cipher.updateAAD(TAG.getBytes(StandardCharsets.UTF_8));
64 |
65 | ByteArrayOutputStream baos = new ByteArrayOutputStream();
66 | baos.write(iv);
67 |
68 | try (CipherOutputStream cipherOutputStream = new CipherOutputStream(baos, cipher)) {
69 | cipherOutputStream.write(data);
70 | }
71 |
72 | return baos.toByteArray();
73 | }
74 |
75 | @Override
76 | public byte[] decrypt(byte[] encryptedData, byte[] key) throws Exception {
77 | SecretKeySpec secretKeySpec = new SecretKeySpec(key, KEY_SPEC);
78 | ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
79 | byte[] iv = new byte[IV_SIZE];
80 | byteBuffer.get(iv);
81 | GCMParameterSpec gcmParamSpec = new GCMParameterSpec(TAG_BIT_LENGTH, iv);
82 | Cipher cipher = Cipher.getInstance(ALGO_TRANSFORMATION_STRING);
83 | cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, gcmParamSpec, secureRandom);
84 | cipher.updateAAD(TAG.getBytes(StandardCharsets.UTF_8));
85 |
86 | byte[] encryptedPayload = new byte[byteBuffer.remaining()];
87 | byteBuffer.get(encryptedPayload);
88 |
89 | return cipher.doFinal(encryptedPayload);
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SampleProducer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.stream.keyrepo;
21 |
22 | import java.util.Properties;
23 | import java.util.Random;
24 |
25 | import org.apache.kafka.clients.producer.KafkaProducer;
26 | import org.apache.kafka.clients.producer.ProducerConfig;
27 | import org.apache.kafka.clients.producer.ProducerRecord;
28 | import org.apache.kafka.common.serialization.IntegerSerializer;
29 | import org.apache.kafka.common.serialization.StringSerializer;
30 |
31 | import io.quicksign.kafka.crypto.Encryptor;
32 | import io.quicksign.kafka.crypto.encryption.DefaultEncryptor;
33 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
34 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
35 | import io.quicksign.kafka.crypto.pairing.serializer.CryptoSerializerPairFactory;
36 | import io.quicksign.kafka.crypto.pairing.serializer.SerializerPair;
37 |
38 | public class SampleProducer implements Runnable {
39 |
40 |
41 | private final KeyProvider keyProvider;
42 | private final KeyReferenceExtractor keyReferenceExtractor;
43 |
44 | public SampleProducer(KeyProvider keyProvider, KeyReferenceExtractor keyReferenceExtractor) {
45 | this.keyProvider = keyProvider;
46 | this.keyReferenceExtractor = keyReferenceExtractor;
47 | }
48 |
49 | @Override
50 | public void run() {
51 |
52 | // tag::produce[]
53 |
54 | // The payload is encrypted using AES
55 | AesGcmNoPaddingCryptoAlgorithm cryptoAlgorithm = new AesGcmNoPaddingCryptoAlgorithm();
56 | Encryptor encryptor = new DefaultEncryptor(keyProvider, cryptoAlgorithm);
57 |
58 | // Wrap base LongSerializer and StringSerializer with encrypted wrappers
59 | CryptoSerializerPairFactory cryptoSerializerPairFactory = new CryptoSerializerPairFactory(encryptor,
60 | keyReferenceExtractor);
61 | SerializerPair serializerPair = cryptoSerializerPairFactory.build(new IntegerSerializer(), new StringSerializer());
62 |
63 | Properties producerProperties = new Properties();
64 | producerProperties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
65 |
66 | Random random = new Random();
67 |
68 | try (KafkaProducer producer =
69 | new KafkaProducer<>(producerProperties, serializerPair.getKeySerializer(), serializerPair.getValueSerializer())) {
70 |
71 | for (long i = 0L; i < Long.MAX_VALUE; i++) {
72 | long accountId = i % 10l;
73 | producer.send(new ProducerRecord<>("operations", (int) accountId, "" + (random.nextInt(1000) - 500)));
74 |
75 | if (i % 100 == 99) {
76 | try {
77 | Thread.sleep(2000L);
78 | }
79 | catch (InterruptedException e) {
80 | return;
81 | }
82 | }
83 |
84 | }
85 | }
86 | // end::produce[]
87 |
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/samples/generatedkey-sample/src/main/java/io/quicksign/kafka/crypto/samples/generatedkey/SampleDecryptingConsumer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.generatedkey;
21 |
22 | import java.util.Collections;
23 | import java.util.Properties;
24 |
25 | import org.apache.kafka.clients.consumer.ConsumerConfig;
26 | import org.apache.kafka.clients.consumer.ConsumerRecords;
27 | import org.apache.kafka.clients.consumer.KafkaConsumer;
28 | import org.apache.kafka.common.serialization.LongDeserializer;
29 | import org.apache.kafka.common.serialization.StringDeserializer;
30 |
31 | import io.quicksign.kafka.crypto.CryptoDeserializerFactory;
32 | import io.quicksign.kafka.crypto.Decryptor;
33 | import io.quicksign.kafka.crypto.encryption.DefaultDecryptor;
34 | import io.quicksign.kafka.crypto.generatedkey.MasterKeyEncryption;
35 | import io.quicksign.kafka.crypto.generatedkey.PerRecordKeyProvider;
36 |
37 | public class SampleDecryptingConsumer implements Runnable {
38 |
39 | private final MasterKeyEncryption masterKeyEncryption;
40 |
41 | public SampleDecryptingConsumer(MasterKeyEncryption masterKeyEncryption) {
42 |
43 | this.masterKeyEncryption = masterKeyEncryption;
44 | }
45 |
46 |
47 | @Override
48 | public void run() {
49 |
50 | // tag::consume[]
51 | // The key is embedded in each message
52 | PerRecordKeyProvider keyProvider = new PerRecordKeyProvider(masterKeyEncryption);
53 |
54 | // The payload is encrypted using AES
55 | AesGcmNoPaddingCryptoAlgorithm cryptoAlgorithm = new AesGcmNoPaddingCryptoAlgorithm();
56 | Decryptor decryptor = new DefaultDecryptor(keyProvider, cryptoAlgorithm);
57 |
58 | // Construct decrypting deserializer
59 | CryptoDeserializerFactory cryptoDeserializerFactory = new CryptoDeserializerFactory(decryptor);
60 |
61 | Properties consumerProperties = new Properties();
62 | consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
63 | consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "samplecrypted");
64 | consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
65 |
66 | try (KafkaConsumer consumer = new KafkaConsumer(
67 | consumerProperties,
68 | new LongDeserializer(),
69 | cryptoDeserializerFactory.buildFrom(new StringDeserializer()))) {
70 |
71 | consumer.subscribe(Collections.singleton("sampletopic"));
72 | for (; true; ) {
73 | ConsumerRecords records = consumer.poll(1000L);
74 | records.forEach(
75 | record -> System.out.println(
76 | "-------------------------------------------------------------\n" +
77 | "decrypted record: key=" + record.key() + ", offset=" + record.offset() + ", value=" + record.value() +
78 | "\n-------------------------------------------------------------\n\n"
79 | )
80 | );
81 | }
82 | }
83 | // end::consume[]
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/samples/keyrepo-sample/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 | io.quicksign
21 | kafka-encryption-keyrepo-sample
22 | 0.0.0-SNAPSHOT
23 | jar
24 | kafka-encryption-keyrepo-sample
25 | Kafka Record Encryption - key repository sample
26 |
27 |
28 | 1.8
29 | 1.8
30 | 1.8
31 | UTF-8
32 |
33 | 0.2.0
34 |
35 |
36 |
37 |
38 |
39 | io.quicksign
40 | kafka-encryption-core
41 | ${kafka-encryption.version}
42 |
43 |
44 |
45 | io.quicksign
46 | kafka-encryption-generatedkey
47 | ${kafka-encryption.version}
48 |
49 |
50 |
51 | io.quicksign
52 | kafka-encryption-keyrepository
53 | ${kafka-encryption.version}
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 | org.apache.maven.plugins
62 | maven-shade-plugin
63 | 3.2.0
64 |
65 |
66 | package
67 |
68 | shade
69 |
70 |
71 | true
72 |
73 |
74 | io.quicksign.kafka.crypto.samples.keyrepo.SamplesMain
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 | org.apache.maven.plugins
83 | maven-deploy-plugin
84 | 2.7
85 |
86 | true
87 |
88 |
89 |
90 |
91 |
92 |
93 |
--------------------------------------------------------------------------------
/samples/generatedkey-sample/src/main/java/io/quicksign/kafka/crypto/samples/generatedkey/SampleProducer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.generatedkey;
21 |
22 | import java.util.Properties;
23 |
24 | import org.apache.kafka.clients.producer.KafkaProducer;
25 | import org.apache.kafka.clients.producer.ProducerConfig;
26 | import org.apache.kafka.clients.producer.ProducerRecord;
27 | import org.apache.kafka.common.serialization.LongSerializer;
28 | import org.apache.kafka.common.serialization.StringSerializer;
29 |
30 | import io.quicksign.kafka.crypto.Encryptor;
31 | import io.quicksign.kafka.crypto.encryption.DefaultEncryptor;
32 | import io.quicksign.kafka.crypto.generatedkey.AES256CryptoKeyGenerator;
33 | import io.quicksign.kafka.crypto.generatedkey.KeyPerRecordKeyReferenceExtractor;
34 | import io.quicksign.kafka.crypto.generatedkey.MasterKeyEncryption;
35 | import io.quicksign.kafka.crypto.generatedkey.PerRecordKeyProvider;
36 | import io.quicksign.kafka.crypto.pairing.serializer.CryptoSerializerPairFactory;
37 | import io.quicksign.kafka.crypto.pairing.serializer.SerializerPair;
38 |
39 | public class SampleProducer implements Runnable {
40 |
41 | private final MasterKeyEncryption masterKeyEncryption;
42 |
43 | public SampleProducer(MasterKeyEncryption masterKeyEncryption) {
44 | this.masterKeyEncryption = masterKeyEncryption;
45 | }
46 |
47 | @Override
48 | public void run() {
49 |
50 | // tag::produce[]
51 | // Use an AES256 key generator
52 | AES256CryptoKeyGenerator cryptoKeyGenerator = new AES256CryptoKeyGenerator();
53 |
54 | // Generate a different key for each message and encrypt it using the master key
55 | KeyPerRecordKeyReferenceExtractor keyReferenceExtractor = new KeyPerRecordKeyReferenceExtractor(
56 | cryptoKeyGenerator, masterKeyEncryption);
57 |
58 | // The key is embedded in each message
59 | PerRecordKeyProvider keyProvider = new PerRecordKeyProvider(masterKeyEncryption);
60 |
61 | // The payload is encrypted using AES
62 | AesGcmNoPaddingCryptoAlgorithm cryptoAlgorithm = new AesGcmNoPaddingCryptoAlgorithm();
63 | Encryptor encryptor = new DefaultEncryptor(keyProvider, cryptoAlgorithm);
64 |
65 | // Wrap base LongSerializer and StringSerializer with encrypted wrappers
66 | CryptoSerializerPairFactory cryptoSerializerPairFactory = new CryptoSerializerPairFactory(encryptor,
67 | keyReferenceExtractor);
68 | SerializerPair serializerPair = cryptoSerializerPairFactory.build(new LongSerializer(), new StringSerializer());
69 |
70 | Properties producerProperties = new Properties();
71 | producerProperties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
72 |
73 | try (KafkaProducer producer =
74 | new KafkaProducer<>(producerProperties, serializerPair.getKeySerializer(), serializerPair.getValueSerializer())) {
75 |
76 | for (long i = 0L; i < Long.MAX_VALUE; i++) {
77 | producer.send(new ProducerRecord<>("sampletopic", i, "test number " + i));
78 | if (i % 10 == 9) {
79 |
80 | try {
81 | Thread.sleep(1000L);
82 | }
83 | catch (InterruptedException e) {
84 | return;
85 | }
86 | }
87 | }
88 | }
89 | // end::produce[]
90 |
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/pairing/serdes/CryptoSerdeFactory.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.pairing.serdes;
21 |
22 | import org.apache.kafka.common.serialization.Serde;
23 | import org.apache.kafka.common.serialization.Serdes;
24 |
25 | import io.quicksign.kafka.crypto.CryptoDeserializer;
26 | import io.quicksign.kafka.crypto.CryptoSerializer;
27 | import io.quicksign.kafka.crypto.Decryptor;
28 | import io.quicksign.kafka.crypto.Encryptor;
29 | import io.quicksign.kafka.crypto.pairing.internal.CryptoAwareSerializerWrapper;
30 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
31 |
32 | /**
33 | * Factory for pairing 2 serde using encryption.
34 | *
35 | * - The serializer of the keySerde will be wrapped to call the {@link KeyReferenceExtractor}
36 | * - The serializer of the valueSerde will be wrapped into a {@link CryptoSerializer}
37 | * - The deserializer of the valueSerde will be wrapped into a {@link CryptoDeserializer}
38 | *
39 | * The keyref extracted by the wrapped key serializer will be shared with the wrapped value serializer using a {@link ThreadLocal}
40 | *
41 | * @See io.quicksign.kafka.crypto.pairing.internal.CryptoAwareSerializerWrapper
42 | */
43 | public class CryptoSerdeFactory implements SerdeFactory {
44 |
45 | private final Encryptor encryptor;
46 | private final Decryptor decryptor;
47 | private final KeyReferenceExtractor keyReferenceExtractor;
48 |
49 | /**
50 | * @param encryptor used for value encryption
51 | * @param decryptor used for value decryption
52 | * @param keyReferenceExtractor used to
53 | */
54 | public CryptoSerdeFactory(Encryptor encryptor, Decryptor decryptor, KeyReferenceExtractor keyReferenceExtractor) {
55 |
56 | this.encryptor = encryptor;
57 | this.decryptor = decryptor;
58 | this.keyReferenceExtractor = keyReferenceExtractor;
59 | }
60 |
61 | public Serde buildFrom(Serde rawSerde) {
62 | return buildFrom(rawSerde, null);
63 | }
64 |
65 | private Serde buildFrom(Serde rawSerde, ThreadLocal keyRefHolder) {
66 | return Serdes.serdeFrom(new CryptoSerializer<>(rawSerde.serializer(), encryptor, keyRefHolder),
67 | new CryptoDeserializer<>(rawSerde.deserializer(), decryptor));
68 | }
69 |
70 | /**
71 | * {@inheritDoc}
72 | */
73 | @Override
74 | public SerdesPair buildSerdesPair(Serde keySerde, Serde valueSerde) {
75 | ThreadLocal keyRefHolder = new ThreadLocal<>();
76 | Serde newKeySerde = Serdes.serdeFrom(
77 | new CryptoAwareSerializerWrapper(keySerde.serializer(), keyReferenceExtractor, keyRefHolder),
78 | keySerde.deserializer());
79 | Serde newValueSerde = buildFrom(valueSerde, keyRefHolder);
80 |
81 | return new SerdesPair<>(newKeySerde, newValueSerde);
82 | }
83 |
84 | /**
85 | * used when the keyref can be deducted directly from the value
86 | *
87 | * @param valueSerde
88 | * @param
89 | * @return
90 | */
91 | @Override
92 | public Serde buildSelfCryptoAwareSerde(Serde valueSerde) {
93 | ThreadLocal keyRefHolder = new ThreadLocal<>();
94 | Serde cryptoSerde = buildFrom(valueSerde, keyRefHolder);
95 | Serde selfAwareSerde = Serdes.serdeFrom(
96 | new CryptoAwareSerializerWrapper(cryptoSerde.serializer(), keyReferenceExtractor, keyRefHolder),
97 | cryptoSerde.deserializer());
98 | return selfAwareSerde;
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/doc/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 |
19 | io.quicksign
20 | kafka-encryption-parent
21 | 0.0.0-SNAPSHOT
22 |
23 |
24 | 4.0.0
25 |
26 | kafka-encryption-doc
27 |
28 |
29 | 1.5.6
30 | 1.5.6
31 | 1.7.26
32 |
33 |
34 |
35 | ${basedir}/../target
36 |
37 |
38 |
39 | org.asciidoctor
40 | asciidoctor-maven-plugin
41 |
42 | ${basedir}/..
43 | true
44 |
45 | ${project.groupId}
46 | ${project.artifactId}
47 | ${project.version}
48 |
49 |
50 |
51 |
52 | asciidoc-to-html
53 | generate-resources
54 |
55 | process-asciidoc
56 |
57 |
58 | html5
59 | coderay
60 |
61 | .
62 | left
63 | 4
64 | font
65 | true
66 |
67 |
68 | -
69 | true
70 |
71 |
72 |
73 | ${basedir}/..
74 | ${project.build.directory}/generated-docs/
75 |
76 | **/*.jpg
77 | **/*.gif
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 | org.asciidoctor
90 | asciidoctor-maven-plugin
91 | ${asciidoctor.maven.plugin.version}
92 |
93 |
94 |
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/core/src/test/java/io/quicksign/kafka/crypto/CryptoDeserializerTest.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto;
21 |
22 | import static org.assertj.core.api.Assertions.assertThat;
23 | import static org.mockito.BDDMockito.given;
24 | import static org.mockito.Mockito.verifyZeroInteractions;
25 |
26 | import java.nio.ByteBuffer;
27 | import java.nio.charset.StandardCharsets;
28 |
29 | import org.apache.kafka.common.header.Headers;
30 | import org.apache.kafka.common.header.internals.RecordHeaders;
31 | import org.apache.kafka.common.serialization.Deserializer;
32 | import org.junit.Test;
33 | import org.junit.runner.RunWith;
34 | import org.mockito.InjectMocks;
35 | import org.mockito.Mock;
36 | import org.mockito.runners.MockitoJUnitRunner;
37 |
38 | @RunWith(MockitoJUnitRunner.class)
39 | public class CryptoDeserializerTest {
40 |
41 | @Mock
42 | Decryptor decryptor;
43 |
44 | @Mock
45 | Deserializer rawDeserializer;
46 |
47 | @InjectMocks
48 | CryptoDeserializer cryptoDeserializer;
49 |
50 | @Test
51 | public void testDeserializeWhenKeyRefIsSet() {
52 | byte[] encoded = "encoded".getBytes(StandardCharsets.UTF_8);
53 | byte[] keyRef = "keyref1".getBytes(StandardCharsets.UTF_8);
54 |
55 | ByteBuffer encodedValue = ByteBuffer.allocate(KafkaCryptoConstants.ENCRYPTED_PREFIX.length + Integer.BYTES + keyRef.length + encoded.length);
56 | encodedValue.put(KafkaCryptoConstants.ENCRYPTED_PREFIX);
57 | encodedValue.putInt(keyRef.length);
58 | encodedValue.put(keyRef);
59 | encodedValue.put(encoded);
60 |
61 |
62 | given(decryptor.decrypt(encoded, keyRef)).willReturn("decoded".getBytes(StandardCharsets.UTF_8));
63 | given(rawDeserializer.deserialize("topic1", new RecordHeaders(), "decoded".getBytes(StandardCharsets.UTF_8))).willReturn("deserialized value");
64 |
65 | RecordHeaders recordHeaders = new RecordHeaders();
66 | String value = cryptoDeserializer.deserialize("topic1", recordHeaders, encodedValue.array());
67 |
68 | assertThat(value).isEqualTo("deserialized value");
69 |
70 | assertThat(recordHeaders.lastHeader(KafkaCryptoConstants.KEY_REF_HEADER).value()).isEqualTo(keyRef);
71 | }
72 |
73 | @Test
74 | public void testDeserializeWhenKeyRefIsNotSet() {
75 | Headers headers = new RecordHeaders();
76 |
77 | byte[] clearValue = "clearValue".getBytes(StandardCharsets.UTF_8);
78 |
79 | ByteBuffer rawValue = ByteBuffer.allocate(KafkaCryptoConstants.ENCRYPTED_PREFIX.length + Integer.BYTES + clearValue.length);
80 | rawValue.put(KafkaCryptoConstants.ENCRYPTED_PREFIX);
81 | rawValue.putInt(0);
82 | rawValue.put(clearValue);
83 |
84 | given(rawDeserializer.deserialize("topic1", headers, clearValue)).willReturn("deserialized value");
85 |
86 | String value = cryptoDeserializer.deserialize("topic1", headers, rawValue.array());
87 |
88 | assertThat(value).isEqualTo("deserialized value");
89 | assertThat(headers.lastHeader(KafkaCryptoConstants.KEY_REF_HEADER).value()).isNull();
90 | verifyZeroInteractions(decryptor);
91 | }
92 |
93 | @Test
94 | public void testDeserializeWhenNoEncryptionStructure() {
95 | Headers headers = new RecordHeaders();
96 |
97 | byte[] clearValue = "clearValue".getBytes(StandardCharsets.UTF_8);
98 |
99 | given(rawDeserializer.deserialize("topic1", headers, clearValue)).willReturn("deserialized value");
100 |
101 | String value = cryptoDeserializer.deserialize("topic1", headers, clearValue);
102 |
103 | assertThat(value).isEqualTo("deserialized value");
104 | assertThat(headers.lastHeader(KafkaCryptoConstants.KEY_REF_HEADER).value()).isNull();
105 | verifyZeroInteractions(decryptor);
106 | }
107 |
108 |
109 | }
110 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/CryptoSerializer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto;
21 |
22 | import static io.quicksign.kafka.crypto.KafkaCryptoConstants.ENCRYPTED_PREFIX;
23 | import static io.quicksign.kafka.crypto.KafkaCryptoConstants.KEY_REF_HEADER;
24 |
25 | import java.nio.ByteBuffer;
26 | import java.util.Map;
27 | import java.util.Properties;
28 |
29 | import org.apache.kafka.common.header.Header;
30 | import org.apache.kafka.common.header.Headers;
31 | import org.apache.kafka.common.serialization.Serializer;
32 | import org.slf4j.Logger;
33 | import org.slf4j.LoggerFactory;
34 |
35 | /**
36 | * Serializer for encrypted
37 | *
38 | * Data is first serialized with the underlying serializer
39 | *
40 | * If a key reference is found either in the Kafka Header {@link KafkaCryptoConstants#KEY_REF_HEADER},
41 | * or in the {@link ThreadLocal} keyRefHolder, then the serialized data is encrypted using the {@link Encryptor}.
42 | * If data has been encrypted, the result of the serialization is a byte array having the following structure:
43 | *
magic_bytes({@link KafkaCryptoConstants#ENCRYPTED_PREFIX})(6 bytes)|keyref.length(4 bytes)|keyref|encrypted_data
44 | *
45 | *
46 | * If the result of encryption is null, then the result of serialization will be null.
47 | *
48 | * If no key reference was found, the result of serialization will be directly the output of the underlying Serializer.
49 | *
50 | *
51 | * Please note that this Serializer has no default constructor, so it can not be configured via properties.
52 | * So if you want to use it into a Producer, you will have to use {@link org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer(Properties, Serializer, Serializer)}
53 | * or {@link org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer(Map, Serializer, Serializer)} with your
54 | * instance of CryptoDeserializer
55 | *
56 | *
57 | * @param
58 | */
59 | public class CryptoSerializer implements Serializer {
60 |
61 | private static final Logger log = LoggerFactory.getLogger(CryptoSerializer.class);
62 |
63 |
64 | private final Serializer rawSerializer;
65 | private final Encryptor encryptor;
66 | private final ThreadLocal keyRefHolder;
67 |
68 | /**
69 | * @param rawSerializer Serializer to serialize data before encryption
70 | * @param encryptor {@link Encryptor} to encrypt data
71 | * @param keyRefHolder {@link ThreadLocal} used to communicate the key reference when using Kafka Stream (unused for regular Kafka Producer)
72 | */
73 | public CryptoSerializer(Serializer rawSerializer, Encryptor encryptor, ThreadLocal keyRefHolder) {
74 | this.rawSerializer = rawSerializer;
75 | this.encryptor = encryptor;
76 | this.keyRefHolder = keyRefHolder;
77 | }
78 |
79 |
80 | /**
81 | * serialize data with encryption (if needed). Key reference will be looked for exclusively in the Kafka Header
82 | * {@link KafkaCryptoConstants#KEY_REF_HEADER}
83 | *
84 | * @param topic
85 | * @param headers
86 | * @param data
87 | * @return
88 | */
89 | @Override
90 | public byte[] serialize(String topic, Headers headers, T data) {
91 | byte[] serializedData = rawSerializer.serialize(topic, headers, data);
92 | if (serializedData == null) {
93 | return null;
94 | }
95 | Header keyReferenceHeader = headers.lastHeader(KEY_REF_HEADER);
96 | return encrypt(serializedData, keyReferenceHeader == null ? null : keyReferenceHeader.value());
97 | }
98 |
99 |
100 | /**
101 | * serialize data with encryption (if needed). Key reference will be looked for into the {@link #keyRefHolder}
102 | *
103 | * @param topic
104 | * @param data
105 | * @return
106 | */
107 | @Override
108 | public byte[] serialize(String topic, T data) {
109 | byte[] serializedData = rawSerializer.serialize(topic, data);
110 | if (serializedData == null) {
111 | return null;
112 | }
113 | return encrypt(serializedData, keyRefHolder == null ? null : keyRefHolder.get());
114 | }
115 |
116 | private byte[] encrypt(byte[] serializedData, byte[] keyref) {
117 | if (keyref == null) {
118 | log.debug("keyref header not defined or null, we will send data unencrypted");
119 | return serializedData;
120 | }
121 | byte[] encryptedData = encryptor.encrypt(serializedData, keyref);
122 | if (encryptedData == null) {
123 | return null;
124 | }
125 | return ByteBuffer.allocate(ENCRYPTED_PREFIX.length + Integer.BYTES + keyref.length + encryptedData.length)
126 | .put(ENCRYPTED_PREFIX)
127 | .putInt(keyref.length)
128 | .put(keyref)
129 | .put(encryptedData)
130 | .array();
131 | }
132 |
133 |
134 | @Override
135 | public void configure(Map configs, boolean isKey) {
136 |
137 | rawSerializer.configure(configs, isKey);
138 | }
139 |
140 | @Override
141 | public void close() {
142 | rawSerializer.close();
143 | }
144 | }
145 |
--------------------------------------------------------------------------------
/core/src/test/java/io/quicksign/kafka/crypto/CryptoSerializerTest.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto;
21 |
22 | import static io.quicksign.kafka.crypto.KafkaCryptoConstants.ENCRYPTED_PREFIX;
23 | import static org.assertj.core.api.Assertions.assertThat;
24 | import static org.mockito.BDDMockito.given;
25 | import static org.mockito.Mockito.verifyZeroInteractions;
26 |
27 | import java.nio.ByteBuffer;
28 | import java.nio.charset.StandardCharsets;
29 |
30 | import org.apache.kafka.common.header.Headers;
31 | import org.apache.kafka.common.header.internals.RecordHeaders;
32 | import org.apache.kafka.common.serialization.Serializer;
33 | import org.junit.Test;
34 | import org.junit.runner.RunWith;
35 | import org.mockito.InjectMocks;
36 | import org.mockito.Mock;
37 | import org.mockito.runners.MockitoJUnitRunner;
38 |
39 | @RunWith(MockitoJUnitRunner.class)
40 | public class CryptoSerializerTest {
41 |
42 | @Mock
43 | Serializer rawSerializer;
44 |
45 | @Mock
46 | Encryptor encryptor;
47 |
48 | @Mock
49 | ThreadLocal keyRefHolder;
50 |
51 | @InjectMocks
52 | CryptoSerializer cryptoSerializer;
53 |
54 | @Test
55 | public void testSerializeWhenKeyRefHeaderIsSet() {
56 | final String keyRef = "org1";
57 | final int keyRefSize = toByteArray(keyRef).length;
58 |
59 | final String encoded = "encodedValue";
60 | final int encodedSize = toByteArray(encoded).length;
61 |
62 | Headers headers = new RecordHeaders().add(KafkaCryptoConstants.KEY_REF_HEADER, toByteArray(keyRef));
63 | given(rawSerializer.serialize("topic1", headers, "final value"))
64 | .willReturn("clear serialized value".getBytes(StandardCharsets.UTF_8));
65 | given(encryptor.encrypt("clear serialized value".getBytes(StandardCharsets.UTF_8), toByteArray(keyRef)))
66 | .willReturn(toByteArray(encoded));
67 |
68 | byte[] result = cryptoSerializer.serialize("topic1", headers, "final value");
69 |
70 | assertThat(result).hasSize(ENCRYPTED_PREFIX.length + Integer.BYTES + keyRefSize + encodedSize);
71 | ByteBuffer byteBuffer = ByteBuffer.wrap(result);
72 | byte[] prefix = new byte[ENCRYPTED_PREFIX.length];
73 | byteBuffer.get(prefix);
74 | assertThat(prefix).isEqualTo(ENCRYPTED_PREFIX);
75 | assertThat(byteBuffer.getInt()).isEqualTo(keyRefSize);
76 | byte[] resultKeyRef = new byte[keyRefSize];
77 | byteBuffer.get(resultKeyRef);
78 | assertThat(resultKeyRef).isEqualTo(toByteArray(keyRef));
79 | byte[] resultPayload = new byte[encodedSize];
80 | byteBuffer.get(resultPayload);
81 | assertThat(resultPayload).isEqualTo(toByteArray(encoded));
82 |
83 | }
84 |
85 | @Test
86 | public void testSerializeWhenKeyRefHeaderIsNotSet() {
87 | final String clearPayload = "clear serialized value";
88 | final int clearPayloadSize = toByteArray(clearPayload).length;
89 |
90 | Headers headers = new RecordHeaders();
91 | given(rawSerializer.serialize("topic1", headers, "final value"))
92 | .willReturn(toByteArray(clearPayload));
93 |
94 | byte[] result = cryptoSerializer.serialize("topic1", headers, "final value");
95 |
96 | assertThat(result).isEqualTo("clear serialized value".getBytes(StandardCharsets.UTF_8));
97 |
98 | verifyZeroInteractions(encryptor);
99 | }
100 |
101 | @Test
102 | public void testSerializeWhenKeyRefHeaderIsSetToNull() {
103 | final String clearPayload = "clear serialized value";
104 | final int clearPayloadSize = toByteArray(clearPayload).length;
105 | Headers headers = new RecordHeaders().add(KafkaCryptoConstants.KEY_REF_HEADER, null);
106 | given(rawSerializer.serialize("topic1", headers, "final value"))
107 | .willReturn(toByteArray(clearPayload));
108 |
109 | byte[] result = cryptoSerializer.serialize("topic1", headers, "final value");
110 |
111 | assertThat(result).isEqualTo("clear serialized value".getBytes(StandardCharsets.UTF_8));
112 | verifyZeroInteractions(encryptor);
113 | }
114 |
115 | @Test
116 | public void testSerializeWithThreadLocal() {
117 | final String keyRef = "org1";
118 | final int keyRefSize = toByteArray(keyRef).length;
119 |
120 | final String encoded = "encodedValue";
121 | final int encodedSize = toByteArray(encoded).length;
122 |
123 | given(keyRefHolder.get()).willReturn(toByteArray(keyRef));
124 |
125 | given(rawSerializer.serialize("topic1", "final value"))
126 | .willReturn("clear serialized value".getBytes(StandardCharsets.UTF_8));
127 | given(encryptor.encrypt("clear serialized value".getBytes(StandardCharsets.UTF_8), toByteArray(keyRef)))
128 | .willReturn(toByteArray(encoded));
129 |
130 | byte[] result = cryptoSerializer.serialize("topic1", "final value");
131 |
132 | assertThat(result).hasSize(ENCRYPTED_PREFIX.length + Integer.BYTES + keyRefSize + encodedSize);
133 | ByteBuffer byteBuffer = ByteBuffer.wrap(result);
134 | byte[] prefix = new byte[ENCRYPTED_PREFIX.length];
135 | byteBuffer.get(prefix);
136 | assertThat(prefix).isEqualTo(ENCRYPTED_PREFIX);
137 | assertThat(byteBuffer.getInt()).isEqualTo(keyRefSize);
138 | byte[] resultKeyRef = new byte[keyRefSize];
139 | byteBuffer.get(resultKeyRef);
140 | assertThat(resultKeyRef).isEqualTo(toByteArray(keyRef));
141 | byte[] resultPayload = new byte[encodedSize];
142 | byteBuffer.get(resultPayload);
143 | assertThat(resultPayload).isEqualTo(toByteArray(encoded));
144 |
145 | }
146 |
147 | private byte[] toByteArray(String s) {
148 | return s.getBytes(StandardCharsets.UTF_8);
149 | }
150 | }
151 |
--------------------------------------------------------------------------------
/core/src/main/java/io/quicksign/kafka/crypto/CryptoDeserializer.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto;
21 |
22 | import static io.quicksign.kafka.crypto.KafkaCryptoConstants.ENCRYPTED_PREFIX;
23 | import static io.quicksign.kafka.crypto.KafkaCryptoConstants.KEY_REF_HEADER;
24 |
25 | import java.nio.ByteBuffer;
26 | import java.util.Map;
27 | import java.util.Properties;
28 |
29 | import org.apache.kafka.common.header.Headers;
30 | import org.apache.kafka.common.header.internals.RecordHeaders;
31 | import org.apache.kafka.common.serialization.Deserializer;
32 | import org.slf4j.Logger;
33 | import org.slf4j.LoggerFactory;
34 |
35 | import io.quicksign.kafka.crypto.utils.ArrayUtils;
36 |
37 | /**
38 | * Deserializer for encrypted data
39 | *
40 | * If the data to deserialize starts with the magic bytes {@link KafkaCryptoConstants#ENCRYPTED_PREFIX},
41 | * then the data is decrypted using the {@link Decryptor}.The result is then deserialized using the underlying Deserializer.
42 | * If the data where not successfully decrypted, the result of the deserialization will be {@code null}
43 | *
44 | *
45 | * If the data to deserialize does not starts with the magic bytes, it is directly deserialized using the underlying Deserializer
46 | *
47 | *
48 | * Please note that this Deserializer has no default constructor, so it can not be configured via properties.
49 | * So if you want to use it into a Consumer, you will have to use {@link org.apache.kafka.clients.consumer.KafkaConsumer#KafkaConsumer(Properties, Deserializer, Deserializer)}
50 | * or {@link org.apache.kafka.clients.consumer.KafkaConsumer#KafkaConsumer(Map, Deserializer, Deserializer)} with your
51 | * instance of CryptoDeserializer
52 | *
53 | *
54 | * @param
55 | * @see CryptoSerializer
56 | * @see Decryptor
57 | */
58 | public class CryptoDeserializer implements Deserializer {
59 |
60 | private static final Logger log = LoggerFactory.getLogger(CryptoDeserializer.class);
61 |
62 | private final Deserializer rawDeserializer;
63 | private final Decryptor decryptor;
64 |
65 | /**
66 | * @param rawDeserializer deserializer to deserialize clear data
67 | * @param decryptor Decryptor used to decrypt the data
68 | */
69 | public CryptoDeserializer(Deserializer rawDeserializer, Decryptor decryptor) {
70 |
71 | this.rawDeserializer = rawDeserializer;
72 | this.decryptor = decryptor;
73 | }
74 |
75 |
76 | /**
77 | * deserialize the data (with decryption if needed)
78 | * The keyref used to deserialize the data will be added to the header {@link KafkaCryptoConstants#KEY_REF_HEADER} (may be {@code null})
79 | *
80 | * @param topic
81 | * @param headers they will be enriched with header {@link KafkaCryptoConstants#KEY_REF_HEADER}
82 | * @param data
83 | * @return the deserialized data
84 | */
85 | @Override
86 | public T deserialize(String topic, Headers headers, byte[] data) {
87 | if (data == null) {
88 | return null;
89 | }
90 |
91 | DecryptedDataWithKeyRef decryptedDataWithKeyRef = decrypt(data);
92 |
93 |
94 | T deserializedValue = rawDeserializer.deserialize(topic, headers, decryptedDataWithKeyRef.decryptedData);
95 |
96 | headers.add(KEY_REF_HEADER, decryptedDataWithKeyRef.keyRef);
97 |
98 | return deserializedValue;
99 | }
100 |
101 | @Override
102 | public void configure(Map configs, boolean isKey) {
103 | this.rawDeserializer.configure(configs, isKey);
104 | }
105 |
106 | /**
107 | * deserialize the data (with decryption if needed)
108 | * It is equivalent to:
109 | * {@code
110 | * deserialize(topic, new RecordHeaders(), data);
111 | * }
112 | *
113 | * @param topic
114 | * @param data
115 | * @return
116 | * @see #deserialize(String, Headers, byte[])
117 | */
118 | @Override
119 | public T deserialize(String topic, byte[] data) {
120 | return deserialize(topic, new RecordHeaders(), data);
121 | }
122 |
123 |
124 | private DecryptedDataWithKeyRef decrypt(byte[] data) {
125 | if (ArrayUtils.startWith(data, ENCRYPTED_PREFIX)) {
126 | ByteBuffer byteBuffer = ByteBuffer.wrap(data,
127 | ENCRYPTED_PREFIX.length, data.length - ENCRYPTED_PREFIX.length);
128 | int keyRefLength = byteBuffer.getInt();
129 | byte[] decryptedData;
130 | byte[] keyRef = null;
131 | if (keyRefLength == 0) {
132 | log.debug("not key ref, data are not encrypted");
133 | decryptedData = new byte[byteBuffer.remaining()];
134 | byteBuffer.get(decryptedData);
135 | }
136 | else {
137 | keyRef = new byte[keyRefLength];
138 | byteBuffer.get(keyRef);
139 | byte[] encryptedData = new byte[byteBuffer.remaining()];
140 | byteBuffer.get(encryptedData);
141 | decryptedData = decryptor.decrypt(encryptedData, keyRef);
142 |
143 | }
144 | return new DecryptedDataWithKeyRef(keyRef, decryptedData);
145 | }
146 | else {
147 | return new DecryptedDataWithKeyRef(null, data);
148 | }
149 |
150 | }
151 |
152 |
153 | @Override
154 | public void close() {
155 | this.rawDeserializer.close();
156 | }
157 |
158 |
159 | private static class DecryptedDataWithKeyRef {
160 |
161 | private final byte[] keyRef;
162 | private final byte[] decryptedData;
163 |
164 | DecryptedDataWithKeyRef(byte[] keyRef, byte[] decryptedData) {
165 |
166 | this.keyRef = keyRef;
167 | this.decryptedData = decryptedData;
168 | }
169 |
170 | }
171 | }
172 |
--------------------------------------------------------------------------------
/samples/kafkastream-with-keyrepo-sample/src/main/java/io/quicksign/kafka/crypto/samples/stream/keyrepo/SampleStream.java:
--------------------------------------------------------------------------------
1 | /*-
2 | * #%L
3 | * Kafka Encryption
4 | * %%
5 | * Copyright (C) 2018 Quicksign
6 | * %%
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | * #L%
19 | */
20 | package io.quicksign.kafka.crypto.samples.stream.keyrepo;
21 |
22 | import java.util.Properties;
23 |
24 | import org.apache.kafka.common.serialization.Serdes;
25 | import org.apache.kafka.streams.KafkaStreams;
26 | import org.apache.kafka.streams.StreamsBuilder;
27 | import org.apache.kafka.streams.StreamsConfig;
28 | import org.apache.kafka.streams.kstream.Materialized;
29 | import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier;
30 | import org.apache.kafka.streams.state.KeyValueIterator;
31 | import org.apache.kafka.streams.state.QueryableStoreTypes;
32 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
33 | import org.apache.kafka.streams.state.Stores;
34 |
35 | import com.fasterxml.jackson.databind.ObjectMapper;
36 | import com.fasterxml.jackson.databind.node.ArrayNode;
37 | import com.fasterxml.jackson.databind.node.JsonNodeFactory;
38 | import com.fasterxml.jackson.databind.node.ObjectNode;
39 | import io.quicksign.kafka.crypto.Decryptor;
40 | import io.quicksign.kafka.crypto.Encryptor;
41 | import io.quicksign.kafka.crypto.encryption.DefaultDecryptor;
42 | import io.quicksign.kafka.crypto.encryption.DefaultEncryptor;
43 | import io.quicksign.kafka.crypto.encryption.KeyProvider;
44 | import io.quicksign.kafka.crypto.pairing.keyextractor.KeyReferenceExtractor;
45 | import io.quicksign.kafka.crypto.pairing.serdes.CryptoSerdeFactory;
46 | import io.quicksign.kafka.crypto.pairing.serdes.SerdesPair;
47 |
48 | public class SampleStream implements Runnable {
49 |
50 | private final String name;
51 | private final KeyProvider keyProvider;
52 | private final KeyReferenceExtractor keyReferenceExtractor;
53 |
54 | public SampleStream(String name, KeyProvider keyProvider, KeyReferenceExtractor keyReferenceExtractor) {
55 |
56 | this.name = name;
57 | this.keyProvider = keyProvider;
58 | this.keyReferenceExtractor = keyReferenceExtractor;
59 | }
60 |
61 |
62 | @Override
63 | public void run() {
64 | try {
65 | // tag::stream[]
66 | AesGcmNoPaddingCryptoAlgorithm cryptoAlgorithm = new AesGcmNoPaddingCryptoAlgorithm();
67 | Decryptor decryptor = new DefaultDecryptor(keyProvider, cryptoAlgorithm);
68 | Encryptor encryptor = new DefaultEncryptor(keyProvider, cryptoAlgorithm);
69 | CryptoSerdeFactory cryptoSerdeFactory = new CryptoSerdeFactory(encryptor, decryptor, keyReferenceExtractor);
70 |
71 | KeyValueBytesStoreSupplier storeSupplier = Stores.inMemoryKeyValueStore(name + "_balance");
72 |
73 |
74 | Properties props = new Properties();
75 | props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, name);
76 | props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
77 | props.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
78 | props.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
79 |
80 |
81 | StreamsBuilder streamsBuilder = new StreamsBuilder();
82 |
83 | SerdesPair serdesPair = cryptoSerdeFactory.buildSerdesPair(Serdes.Integer(), Serdes.String());
84 |
85 | streamsBuilder.stream("operations", serdesPair.toConsumed())
86 | .filter((i, s) -> s != null) // messages that were not decrypted (because key not in repository) are null
87 | .groupByKey()
88 | .reduce((s1, s2) -> "" + (Integer.valueOf(s1) + Integer.valueOf(s2)),
89 | serdesPair.applyTo(Materialized.as(storeSupplier)));
90 |
91 | KafkaStreams kafkaStreams = new KafkaStreams(streamsBuilder.build(), props);
92 | kafkaStreams.start();
93 |
94 | // end::stream[]
95 |
96 | ObjectMapper mapper = new ObjectMapper();
97 |
98 | for (; true; ) {
99 | try {
100 | Thread.sleep(3000L);
101 | }
102 | catch (InterruptedException e) {
103 | return;
104 | }
105 | try {
106 |
107 | // tag::display[]
108 | ReadOnlyKeyValueStore store = kafkaStreams.store(
109 | name + "_balance", QueryableStoreTypes.keyValueStore()
110 | );
111 |
112 | ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode();
113 |
114 | KeyValueIterator iterator = store.all();
115 | iterator.forEachRemaining(kv -> {
116 | ObjectNode node = JsonNodeFactory.instance.objectNode();
117 | node.put("account", kv.key);
118 | node.put("balance", kv.value);
119 | arrayNode.add(node);
120 | });
121 |
122 | ObjectNode node = JsonNodeFactory.instance.objectNode();
123 | node.set(name, arrayNode);
124 |
125 | System.out.println(
126 | "--[look up from KV store]------------------------------------\n" +
127 | mapper.writerWithDefaultPrettyPrinter().writeValueAsString(node) +
128 | "\n--------------------------------------\n\n"
129 | );
130 | // end::display[]
131 | }
132 | catch (Exception e) {
133 |
134 | }
135 | }
136 | }
137 | catch (Throwable e) {
138 | System.out.println("error on " + name);
139 | e.printStackTrace();
140 | }
141 |
142 | }
143 | }
144 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # kafka-encryption
2 |
3 | [](https://travis-ci.com/QuickSign/kafka-encryption)
4 |
5 | Kafka-encryption is a Java framework that eases the encryption/decryption of Kafka
6 | record's value at the serializer/deserializer level.
7 |
8 | ## Design goals
9 |
10 | * Support or allow multiple encryption key management strategies (KMS, embedded, per message, rolling windows, per tenant, custom)
11 | * Support for Kafka Streams intermediate topics
12 | * Detect when a payload is not encrypted to skip decryption
13 | * Support for Spring Boot
14 | * Support for Camel
15 |
16 | ## Customization
17 |
18 | This framework exposes some high level Interfaces to let you customize the crypto
19 | Serializer/Deserializer internals.
20 |
21 | This framework is used on our platform. For obvious reason we do not reveal here our custom
22 | implementations of these interfaces. They would probably be useless to you anyway.
23 |
24 | However, and this is the good news, we provide in our examples some working implementations that
25 | you can definitely leverage.
26 |
27 | ## Terminology & basic explanation.
28 |
29 | As you explore the code or the examples, you may get confused by the terminology used.
30 |
31 | Do not confuse the Kafka `record's key` and the `encryption key` that is used to encrypt the record's value.
32 |
33 | You may also get confused by what we call a `key name` and a `key reference`.
34 |
35 | A `key name` is in general used to lookup an encryption key in a repository, but it could also be the `encryption key` itself.
36 |
37 | A `key reference` or `key ref` is derived from the `key name`. It can be for example an obfuscated or
38 | encrypted version of the `key name`. The `key ref` is stored in the record's value as a prefix of the encrypted value. .
39 |
40 | ## Examples
41 |
42 | We provide 3 examples that work out of the box. Do not use their code as is in production (we don't).
43 | Hopefully you can replace some of the implementations provided in the examples with your own.
44 |
45 | TIP: When studying the samples' code, to ease your pain start by studying
46 | the SamplesMain and SampleProducer.
47 |
48 | ### Example 1 - samples/generatedkey-sample : one encryption key per record
49 |
50 | This example uses the classic consumer API. It neither relies on the record's key nor on an
51 | encryption key repository. Instead the `encryption key` is encrypted and transmitted in the record's value.
52 |
53 | As a developer using the framework, in this example we provide 2 custom implementations to support our need.
54 | These implementations are used to construct the CryptoSerializerPairFactory.
55 |
56 | Here is roughly what this example demonstrates:
57 |
58 | __Serializer__
59 |
60 | * Generates a new `encryption key` for each record
61 | * Encrypts the record's value using the `encryption key` (see AesGcmNoPaddingCryptoAlgorithm).
62 | * Uses the master encryption key (see KeyStoreBasedMasterKey) to encrypt the `encryption key`. The encrypted encryption key is the `key ref`. Note that the master encryption key is stored in a Java KeyStore which is itself protected by a password.
63 |
64 | __Deserializer__
65 |
66 | * It extracts the `key ref` from the record's value.
67 | * Uses the master encryption key (see KeyStoreBasedMasterKey) to decrypt the `encryption key` out of the `key ref`.
68 | * Decrypts the record's value using the `encryption key` (see AesGcmNoPaddingCryptoAlgorithm).
69 |
70 | ### Example 2 - samples/kafkastreams-with-keyrepo-sample : one encryption key per record
71 |
72 | This example uses the Kafka Streams API. It creates a KTable, its content is also encrypted.
73 | We use one `encryption key` per record's key.
74 | The `encryption key` is stored in Java KeyStore, it is not transmitted in the record's value.
75 |
76 | As a developer using the framework, in this example we provide 4 custom implementations to support our need.
77 | These implementations are used to construct the CryptoSerializerPairFactory.
78 |
79 | Here is roughly what this example demonstrates:
80 |
81 | __Serializer__
82 |
83 | * Uses the record's key as the `key name` (see SampleKeyNameExtractor)
84 | * Using the `key name`, looks up the `encryption key` from the KeyStoreBasedKeyRepository
85 | * Uses the SampleKeyNameObfuscator provided to create the `key ref` by simply swapping some bytes from the `key name`.
86 | * It encrypts the record's value using `encryption key` (see AesGcmNoPaddingCryptoAlgorithm)
87 |
88 | __Deserializer__
89 |
90 | * Extracts the `key ref` from the record's value
91 | * Uses the SampleKeyNameObfuscator to obtain the `key name` out of the `key ref`
92 | * Looks up the `encryption key` from the KeyStoreBasedKeyRepository using the `key name`
93 | * Decrypts the record's value using the `encryption key` (see AesGcmNoPaddingCryptoAlgorithm)
94 |
95 |
96 | ### Example 3 - samples/keyrepo-sample : one encryption key per record's key.
97 |
98 | This example uses the classic consumer API. There is one `encryption key` per record's key.
99 | The `encryption key` is stored in an in memory encryption key repository, it is not transmitted in
100 | the record's value.
101 |
102 | As a developer using the framework, in this example we provide 4 custom implementations to support our need.
103 | These implementations are used to construct the CryptoSerializerPairFactory.
104 |
105 | Here is roughly what this example demonstrates:
106 |
107 | __Serializer__
108 |
109 | * Uses the record's key as the `key name` (see SampleKeyNameExtractor)
110 | * Using the `key name`, looks up the `encryption key` from the SampleKeyRepository, a basic in memory encryption key repository.
111 | * Uses the SampleKeyNameObfuscator provided to create the `key ref` by simply swapping some bytes from the `key name`.
112 | * It encrypts the record's value using `encryption key` (see AesGcmNoPaddingCryptoAlgorithm)
113 |
114 | __Deserializer__
115 |
116 | * Extracts the `key ref` from the record's value
117 | * Uses the SampleKeyNameObfuscator to obtain the `key name` out of the `key ref`
118 | * Looks up the `encryption key` from the SampleKeyRepository using the `key name`
119 | * Decrypts the record's value using the `encryption key` (see AesGcmNoPaddingCryptoAlgorithm)
120 |
121 | ### Troubleshooting
122 |
123 | In case the docker compose provided in the examples to run Kafka does not work for you, you may use this command:
124 |
125 | *On OSX and Windows*
126 |
127 | docker run --rm -p 2181:2181 -p 3030:3030 -p 8081-8083:8081-8083 -p 9581-9585:9581-9585 -p 9092:9092 -e ADV_HOST=192.168.99.100 landoop/fast-data-dev:2.0.1
128 |
129 | *On linux*
130 |
131 | docker run --rm --net=host -e ADV_HOST=localhost landoop/fast-data-dev:2.0.1
132 |
133 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
15 |
18 | 4.0.0
19 |
20 | io.quicksign
21 | kafka-encryption-parent
22 | 0.0.0-SNAPSHOT
23 | pom
24 |
25 | kafka-encryption-parent
26 | End to end encryption for Kafka
27 |
28 | https://github.com/Quicksign/kafka-encryption
29 |
30 |
31 | Julien Vaudour
32 | Quicksign
33 | julien.vaudour@quicksign.com
34 |
35 |
36 |
37 |
38 |
39 | The Apache Software License, Version 2.0
40 | http://www.apache.org/licenses/LICENSE-2.0.txt
41 |
42 |
43 |
44 |
45 | git@github.com:quicksign/kafka-encryption.git
46 | git@github.com:quicksign/kafka-encryption.git
47 | https://github.com/quicksign/kafka-encryption
48 |
49 |
50 |
51 | 1.8
52 | 1.8
53 | 1.8
54 | UTF-8
55 |
56 | 2.1.0
57 |
58 |
59 |
60 |
61 | ossrh
62 | https://oss.sonatype.org/content/repositories/snapshots
63 |
64 |
65 | ossrh
66 | https://oss.sonatype.org/service/local/staging/deploy/maven2/
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 | io.quicksign
75 | kafka-encryption-core
76 | ${project.version}
77 |
78 |
79 |
80 | io.quicksign
81 | kafka-encryption-generatedkey
82 | ${project.version}
83 |
84 |
85 |
86 | io.quicksign
87 | kafka-encryption-keyrepository
88 | ${project.version}
89 |
90 |
91 |
92 | org.slf4j
93 | slf4j-api
94 | 1.7.25
95 |
96 |
97 |
98 | org.apache.kafka
99 | kafka-clients
100 | ${kafka.version}
101 |
102 |
103 |
104 | org.apache.kafka
105 | kafka-streams
106 | ${kafka.version}
107 |
108 |
109 |
110 |
111 |
112 | core
113 | generatedkey
114 | keyrepository
115 |
116 |
117 |
118 |
119 |
120 | org.codehaus.mojo
121 | license-maven-plugin
122 | 1.16
123 |
124 | false
125 | true
126 |
127 |
128 |
129 | first
130 |
131 | update-file-header
132 |
133 | process-sources
134 |
135 | 2018
136 | Quicksign
137 | Kafka Encryption
138 | apache_v2
139 | false
140 | false
141 |
142 |
143 |
144 |
145 |
146 | org.apache.maven.plugins
147 | maven-javadoc-plugin
148 | 3.0.1
149 |
150 | protected
151 | true
152 | -Xdoclint:none
153 |
154 |
155 |
156 | attach-javadocs
157 |
158 | jar
159 |
160 |
161 |
162 |
163 |
164 | org.apache.maven.plugins
165 | maven-source-plugin
166 | 3.0.1
167 |
168 |
169 | attach-sources
170 |
171 | jar-no-fork
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 | publish
182 |
183 |
184 |
185 | org.apache.maven.plugins
186 | maven-gpg-plugin
187 | 1.6
188 |
189 |
190 | sign-artifacts
191 | verify
192 |
193 | sign
194 |
195 |
196 |
197 |
198 |
199 |
200 | org.sonatype.plugins
201 | nexus-staging-maven-plugin
202 | 1.6.8
203 | true
204 |
205 | ossrh
206 | https://oss.sonatype.org/
207 | true
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
--------------------------------------------------------------------------------