├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── pom.xml └── src ├── main └── java │ └── de │ └── saly │ └── kafka │ └── crypto │ ├── DecryptingDeserializer.java │ ├── EncryptingSerializer.java │ ├── RsaKeyGen.java │ └── SerdeCryptoBase.java ├── site ├── markdown │ └── index.md └── site.xml └── test └── java └── de └── saly └── kafka └── crypto └── EnDecryptionTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .classpath 3 | .settings/ 4 | .project 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | cache: 2 | directories: 3 | - $HOME/.m2 4 | language: java 5 | jdk: 6 | - openjdk6 7 | - openjdk7 8 | - oraclejdk7 9 | - oraclejdk8 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka End2End encryption 2 | 3 | A small library with no external dependencies which provide transparent AES end-to-end encryption for Apache Kafka. 4 | 5 | See [documentation](https://salyh.github.io/kafka-end-2-end-encryption/index.html) and this [blog post](https://blog.codecentric.de/en/2016/10/transparent-end-end-security-apache-kafka-part-1/) 6 | and [this slide deck](https://speakerdeck.com/salyh/transparent-end-to-end-security-for-apache-kafka-dh) 7 | 8 | * Original implementation: https://github.com/salyh/kafka-end-2-end-encryption/tree/master (https://blog.codecentric.de/en/2016/10/transparent-end-end-security-apache-kafka-part-1/) 9 | * ECDH implementation: https://github.com/salyh/kafka-end-2-end-encryption/tree/ecdh-gcm-native (https://speakerdeck.com/salyh/transparent-end-to-end-security-for-apache-kafka-dh) 10 | * ECIES implementation: https://github.com/salyh/kafka-end-2-end-encryption/tree/ecies 11 | 12 | ## License 13 | Apache 2, see LICENSE file 14 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | 3.1.0 7 | 8 | 9 | 10 | org.sonatype.oss 11 | oss-parent 12 | 7 13 | 14 | 15 | de.saly 16 | kafka-end-2-end-encryption 17 | 1.0.2-SNAPSHOT 18 | jar 19 | kafka-end-2-end-encryption 20 | kafka-end-2-end-encryption 21 | https://github.com/salyh/kafka-end-2-end-encryption 22 | 2016 23 | 24 | 25 | 26 | ossrh 27 | https://oss.sonatype.org/content/repositories/snapshots 28 | 29 | 30 | ossrh 31 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 32 | 33 | 34 | 35 | 36 | 37 | egit 38 | Eclipse egit 39 | https://repo.eclipse.org/content/repositories/egit-releases/ 40 | 41 | 42 | 43 | 44 | 0.8.2.0 45 | 4.12 46 | false 47 | github 48 | 49 | 50 | 51 | https://github.com/salyh/kafka-end-2-end-encryption 52 | scm:git:git@github.com:salyh/kafka-end-2-end-encryption.git 53 | scm:git:git@github.com:salyh/kafka-end-2-end-encryption.git 54 | HEAD 55 | 56 | 57 | 58 | GitHub 59 | https://github.com/salyh/kafka-end-2-end-encryption/issues 60 | 61 | 62 | 63 | 64 | org.apache.kafka 65 | kafka-clients 66 | ${kafka.version} 67 | provided 68 | 69 | 70 | junit 71 | junit 72 | ${junit.version} 73 | test 74 | 75 | 76 | 77 | 78 | 79 | 80 | org.apache.maven.plugins 81 | maven-enforcer-plugin 82 | 1.4.1 83 | 84 | 85 | enforce-java 86 | 87 | enforce 88 | 89 | 90 | 91 | 92 | 1.6 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-compiler-plugin 102 | 3.5.1 103 | 104 | 1.6 105 | 1.6 106 | UTF-8 107 | true 108 | true 109 | 110 | 111 | 112 | org.apache.maven.plugins 113 | maven-surefire-plugin 114 | 2.19.1 115 | 117 | 118 | 119 | org.apache.maven.plugins 120 | maven-source-plugin 121 | 3.0.1 122 | 123 | 124 | attach-sources 125 | 126 | jar 127 | 128 | 129 | 130 | 131 | 132 | org.apache.maven.plugins 133 | maven-release-plugin 134 | 2.5.2 135 | 136 | v@{project.version} 137 | false 138 | true 139 | true 140 | false 141 | release 142 | 143 | 144 | 145 | org.apache.maven.plugins 146 | maven-site-plugin 147 | 3.5.1 148 | 149 | 150 | 151 | org.apache.maven.doxia 152 | doxia-core 153 | 1.7 154 | 155 | 156 | org.apache.maven.doxia 157 | doxia-module-markdown 158 | 1.7 159 | 160 | 161 | 162 | 163 | com.github.github 164 | site-maven-plugin 165 | 0.12 166 | 167 | Creating site for ${project.version} 168 | 169 | 170 | 171 | 172 | site 173 | 174 | site 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | org.codehaus.mojo 185 | findbugs-maven-plugin 186 | 3.0.4 187 | 188 | 189 | org.apache.maven.plugins 190 | maven-pmd-plugin 191 | 3.6 192 | 193 | 194 | org.apache.maven.plugins 195 | maven-project-info-reports-plugin 196 | 2.9 197 | 198 | 199 | org.apache.maven.plugins 200 | maven-javadoc-plugin 201 | 2.10.4 202 | 203 | ${project.build.sourceEncoding} 204 | en 205 | true 206 | true 207 | ${project.name} 208 | -Xdoclint:none 209 | 210 | 211 | 212 | org.apache.maven.plugins 213 | maven-surefire-report-plugin 214 | 2.19.1 215 | 216 | 217 | org.apache.maven.plugins 218 | maven-jxr-plugin 219 | 2.5 220 | 221 | 222 | 223 | -------------------------------------------------------------------------------- /src/main/java/de/saly/kafka/crypto/DecryptingDeserializer.java: -------------------------------------------------------------------------------- 1 | package de.saly.kafka.crypto; 2 | 3 | import java.util.Map; 4 | 5 | import javax.crypto.Cipher; 6 | 7 | import org.apache.kafka.common.serialization.Deserializer; 8 | 9 | /** 10 | * This is a deserialization (for the Consumer) wrapper which adds transparent end-to-end message encryption. 11 | * Its intended to be used together with {@link EncryptingSerializer} 12 | *

13 | * Configuration

14 | *

19 | *

20 | * See {@link EncryptingSerializer} on how encryption works. 21 | *

22 | * This class will auto detect if an incoming message is encrypted. If not then no decryption attempt is made and message gets handled normally. 23 | *

24 | * Note: As Consumers are not multithreading-safe this deserializer is also not thread-safe 25 | *

26 | * @param The type to be deserialized from (applied to the wrapped deserializer) 27 | */ 28 | public class DecryptingDeserializer extends SerdeCryptoBase implements Deserializer { 29 | 30 | public static final String CRYPTO_VALUE_DESERIALIZER = "crypto.wrapped_deserializer"; 31 | private Deserializer inner; 32 | 33 | @SuppressWarnings("unchecked") 34 | @Override 35 | public void configure(Map configs, boolean isKey) { 36 | inner = newInstance(configs, CRYPTO_VALUE_DESERIALIZER, Deserializer.class); 37 | inner.configure(configs, isKey); 38 | init(Cipher.DECRYPT_MODE, configs, isKey); 39 | } 40 | 41 | @Override 42 | public T deserialize(String topic, byte[] data) { 43 | return inner.deserialize(topic, crypt(data)); 44 | } 45 | 46 | @Override 47 | public void close() { 48 | if (inner != null) { 49 | inner.close(); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/de/saly/kafka/crypto/EncryptingSerializer.java: -------------------------------------------------------------------------------- 1 | package de.saly.kafka.crypto; 2 | 3 | import java.util.Map; 4 | import java.util.concurrent.atomic.AtomicInteger; 5 | 6 | import javax.crypto.Cipher; 7 | 8 | import org.apache.kafka.common.serialization.Serializer; 9 | 10 | /** 11 | * 12 | * This is a serialization wrapper which adds message encryption. Its intended to be used together with {@link DecryptingDeserializer} 13 | *

14 | * Configuration

15 | *

21 | *

22 | * Each message is encrypted with "AES/CBC/PKCS5Padding" before its sent to Kafka. The AES key as well as the initialization vector are random. 23 | * The AES key is attached to the message in a RSA encrypted manner. The IV is also attached but not RSA encrypted. There is also a hash value 24 | * of the AES key to allow consumers caching of decrypted AES keys. Finally we have a few magic and header bytes. 25 | * The resulting byte array looks therefore like this: 26 | *

27 | * 

MMLLLHH..HHEEEE..EEEEIIII..IIIOOOOO....OOOOOO
28 | *

29 | *

37 | *

38 | * MMLLL is called the encryption header and consists of 5 bytes. 39 | *

40 | *

47 | *

48 | * RSA public/private keypair can be generated with
49 | * java -cp kafka-end-2-end-encryption-1.0.0.jar de.saly.kafka.crypto.RsaKeyGen 2048 50 | *

51 | * Note: As Producers are multithreading-safe this serializer is also thread-safe 52 | *

53 | * 54 | * @param The type to be serialized from (applied to the wrapped serializer) 55 | */ 56 | public class EncryptingSerializer extends SerdeCryptoBase implements Serializer { 57 | 58 | public static final String CRYPTO_VALUE_SERIALIZER = "crypto.wrapped_serializer"; 59 | public static final String CRYPTO_NEW_KEY_MSG_INTERVAL = "crypto.new_key_msg_interval"; 60 | public int msgInterval = -1; 61 | private Serializer inner; 62 | private final AtomicInteger msg = new AtomicInteger(); 63 | 64 | @SuppressWarnings("unchecked") 65 | @Override 66 | public void configure(Map configs, boolean isKey) { 67 | inner = newInstance(configs, CRYPTO_VALUE_SERIALIZER, Serializer.class); 68 | inner.configure(configs, isKey); 69 | init(Cipher.ENCRYPT_MODE, configs, isKey); 70 | String msgIntervalProperty = (String) configs.get(CRYPTO_NEW_KEY_MSG_INTERVAL); 71 | if (msgIntervalProperty != null && msgIntervalProperty.length() > 0) { 72 | msgInterval = Integer.parseInt(msgIntervalProperty); 73 | if (msgInterval < 1) { 74 | msgInterval = -1; 75 | } 76 | } 77 | } 78 | 79 | @Override 80 | public byte[] serialize(String topic, T data) { 81 | if (msgInterval > 0 && msg.compareAndSet(msgInterval, 0)) { 82 | newKey(); 83 | } else if (msgInterval > 0) { 84 | msg.incrementAndGet(); 85 | } 86 | return crypt(inner.serialize(topic, data)); 87 | } 88 | 89 | @Override 90 | public void close() { 91 | if (inner != null) { 92 | inner.close(); 93 | } 94 | } 95 | 96 | /** 97 | * Generate new AES key for the current thread 98 | */ 99 | public void newKey() { 100 | super.newKey(); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/de/saly/kafka/crypto/RsaKeyGen.java: -------------------------------------------------------------------------------- 1 | package de.saly.kafka.crypto; 2 | 3 | import java.io.File; 4 | import java.io.FileOutputStream; 5 | import java.security.KeyPair; 6 | import java.security.KeyPairGenerator; 7 | import java.util.UUID; 8 | 9 | public class RsaKeyGen { 10 | 11 | public static void main(String[] args) throws Exception { 12 | int keysize = (args != null && args.length > 0) ? Integer.parseInt(args[0]) : 2048; 13 | System.out.println("Keysize: "+keysize+" bits"); 14 | String uuid = UUID.randomUUID().toString(); 15 | File pubKey = new File("rsa_publickey_" + keysize + "_" + uuid); 16 | File privKey = new File("rsa_privatekey_" + keysize + "_" + uuid); 17 | 18 | KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); 19 | keyGen.initialize(keysize); 20 | KeyPair pair = keyGen.genKeyPair(); 21 | byte[] publicKey = pair.getPublic().getEncoded(); 22 | byte[] privateKey = pair.getPrivate().getEncoded(); 23 | 24 | FileOutputStream fout = new FileOutputStream(pubKey); 25 | fout.write(publicKey); 26 | fout.close(); 27 | 28 | fout = new FileOutputStream(privKey); 29 | fout.write(privateKey); 30 | fout.close(); 31 | 32 | System.out.println(pubKey.getAbsolutePath()); 33 | System.out.println(privKey.getAbsolutePath()); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/de/saly/kafka/crypto/SerdeCryptoBase.java: -------------------------------------------------------------------------------- 1 | package de.saly.kafka.crypto; 2 | 3 | import java.io.DataInputStream; 4 | import java.io.File; 5 | import java.io.FileInputStream; 6 | import java.io.IOException; 7 | import java.security.KeyFactory; 8 | import java.security.MessageDigest; 9 | import java.security.NoSuchAlgorithmException; 10 | import java.security.PrivateKey; 11 | import java.security.PublicKey; 12 | import java.security.SecureRandom; 13 | import java.security.spec.InvalidKeySpecException; 14 | import java.security.spec.PKCS8EncodedKeySpec; 15 | import java.security.spec.X509EncodedKeySpec; 16 | import java.util.Arrays; 17 | import java.util.HashMap; 18 | import java.util.Map; 19 | 20 | import javax.crypto.BadPaddingException; 21 | import javax.crypto.Cipher; 22 | import javax.crypto.IllegalBlockSizeException; 23 | import javax.crypto.SecretKey; 24 | import javax.crypto.spec.IvParameterSpec; 25 | import javax.crypto.spec.SecretKeySpec; 26 | import javax.xml.bind.DatatypeConverter; 27 | 28 | import org.apache.kafka.common.KafkaException; 29 | import org.apache.kafka.common.utils.Utils; 30 | 31 | public abstract class SerdeCryptoBase { 32 | 33 | public static final String CRYPTO_RSA_PRIVATEKEY_FILEPATH = "crypto.rsa.privatekey.filepath"; //consumer 34 | public static final String CRYPTO_RSA_PUBLICKEY_FILEPATH = "crypto.rsa.publickey.filepath"; //producer 35 | public static final String CRYPTO_HASH_METHOD = "crypto.hash_method"; 36 | public static final String CRYPTO_IGNORE_DECRYPT_FAILURES = "crypto.ignore_decrypt_failures"; 37 | public static final String CRYPTO_AES_KEY_LEN = "crypto.aes.key_len"; 38 | static final byte[] MAGIC_BYTES = new byte[] { (byte) 0xDF, (byte) 0xBB }; 39 | protected static final String DEFAULT_TRANSFORMATION = "AES/CBC/PKCS5Padding"; //TODO allow other like GCM 40 | private static final Map aesKeyCache = new HashMap(); 41 | private static final int MAGIC_BYTES_LENGTH = MAGIC_BYTES.length; 42 | private static final int HEADER_LENGTH = MAGIC_BYTES_LENGTH + 3; 43 | private static final String AES = "AES"; 44 | private static final String RSA = "RSA"; 45 | private static final String RSA_TRANFORMATION = "RSA/ECB/OAEPWithSHA-256AndMGF1Padding"; 46 | private static final int RSA_MULTIPLICATOR = 128; 47 | private int opMode; 48 | private String hashMethod = "SHA-256"; 49 | private int aesKeyLen = 128; 50 | private boolean ignoreDecryptFailures = false; 51 | private ProducerCryptoBundle producerCryptoBundle = null; 52 | private ConsumerCryptoBundle consumerCryptoBundle = null; 53 | 54 | protected SerdeCryptoBase() { 55 | 56 | } 57 | 58 | //not thread safe 59 | private class ConsumerCryptoBundle { 60 | 61 | private Cipher rsaDecrypt; 62 | final Cipher aesDecrypt = Cipher.getInstance(DEFAULT_TRANSFORMATION); 63 | 64 | private ConsumerCryptoBundle(PrivateKey privateKey) throws Exception { 65 | rsaDecrypt = Cipher.getInstance(RSA_TRANFORMATION); 66 | rsaDecrypt.init(Cipher.DECRYPT_MODE, privateKey); 67 | } 68 | 69 | private byte[] aesDecrypt(byte[] encrypted) throws KafkaException { 70 | try { 71 | if (encrypted[0] == MAGIC_BYTES[0] && encrypted[1] == MAGIC_BYTES[1]) { 72 | final byte hashLen = encrypted[2]; 73 | final byte rsaFactor = encrypted[3]; 74 | final byte ivLen = encrypted[4]; 75 | final int offset = HEADER_LENGTH + hashLen + (rsaFactor * RSA_MULTIPLICATOR) + ivLen; 76 | final String aesHash = DatatypeConverter.printHexBinary(Arrays.copyOfRange(encrypted, HEADER_LENGTH, HEADER_LENGTH + hashLen)); 77 | final byte[] iv = Arrays.copyOfRange(encrypted, HEADER_LENGTH + hashLen + (rsaFactor * RSA_MULTIPLICATOR), 78 | HEADER_LENGTH + hashLen + (rsaFactor * RSA_MULTIPLICATOR) + ivLen); 79 | 80 | byte[] aesKey; 81 | 82 | if ((aesKey = aesKeyCache.get(aesHash)) != null) { 83 | aesDecrypt.init(Cipher.DECRYPT_MODE, createAESSecretKey(aesKey), new IvParameterSpec(iv)); 84 | return crypt(aesDecrypt, encrypted, offset, encrypted.length - offset); 85 | } else { 86 | byte[] rsaEncryptedAesKey = Arrays.copyOfRange(encrypted, HEADER_LENGTH + hashLen, 87 | HEADER_LENGTH + hashLen + (rsaFactor * RSA_MULTIPLICATOR)); 88 | aesKey = crypt(rsaDecrypt, rsaEncryptedAesKey); 89 | aesDecrypt.init(Cipher.DECRYPT_MODE, createAESSecretKey(aesKey), new IvParameterSpec(iv)); 90 | aesKeyCache.put(aesHash, aesKey); 91 | return crypt(aesDecrypt, encrypted, offset, encrypted.length - offset); 92 | } 93 | } else { 94 | return encrypted; //not encrypted, just bypass decryption 95 | } 96 | } catch (Exception e) { 97 | if(ignoreDecryptFailures) { 98 | return encrypted; //Probably not encrypted, just bypass decryption 99 | } 100 | 101 | throw new KafkaException("Decrypt failed",e); 102 | } 103 | } 104 | } 105 | 106 | private class ThreadAwareKeyInfo { 107 | private final SecretKey aesKey; 108 | private final byte[] aesHash; 109 | private final byte[] rsaEncyptedAesKey; 110 | private final Cipher rsaCipher; 111 | private final Cipher aesCipher; 112 | private final SecureRandom random = new SecureRandom(); 113 | 114 | protected ThreadAwareKeyInfo(PublicKey publicKey) throws Exception { 115 | byte[] aesKeyBytes = new byte[aesKeyLen/8]; 116 | random.nextBytes(aesKeyBytes); 117 | aesCipher = Cipher.getInstance(DEFAULT_TRANSFORMATION); 118 | aesKey = createAESSecretKey(aesKeyBytes); 119 | aesHash = hash(aesKeyBytes); 120 | rsaCipher = Cipher.getInstance(RSA_TRANFORMATION); 121 | rsaCipher.init(Cipher.ENCRYPT_MODE, publicKey); 122 | rsaEncyptedAesKey = crypt(rsaCipher, aesKeyBytes); 123 | } 124 | } 125 | 126 | //threads safe 127 | private class ProducerCryptoBundle { 128 | 129 | private ThreadLocal keyInfo = new ThreadLocal() { 130 | @Override 131 | protected ThreadAwareKeyInfo initialValue() { 132 | try { 133 | return new ThreadAwareKeyInfo(publicKey); 134 | } catch (Exception e) { 135 | throw new KafkaException(e); 136 | } 137 | } 138 | }; 139 | private final PublicKey publicKey; 140 | 141 | private ProducerCryptoBundle(PublicKey publicKey) throws Exception { 142 | this.publicKey = publicKey; 143 | } 144 | 145 | private void newKey() throws Exception { 146 | keyInfo.remove(); 147 | } 148 | 149 | private byte[] aesEncrypt(byte[] plain) throws KafkaException { 150 | final ThreadAwareKeyInfo ki = keyInfo.get(); 151 | 152 | try { 153 | final byte[] aesIv = new byte[16]; 154 | ki.random.nextBytes(aesIv); 155 | ki.aesCipher.init(Cipher.ENCRYPT_MODE, ki.aesKey, new IvParameterSpec(aesIv)); 156 | return concatenate(MAGIC_BYTES, new byte[] { (byte) ki.aesHash.length, 157 | (byte) (ki.rsaEncyptedAesKey.length / RSA_MULTIPLICATOR), (byte) aesIv.length }, ki.aesHash, ki.rsaEncyptedAesKey, 158 | aesIv, crypt(ki.aesCipher, plain)); 159 | } catch (Exception e) { 160 | throw new KafkaException(e); 161 | } 162 | } 163 | } 164 | 165 | protected void init(int opMode, Map configs, boolean isKey) throws KafkaException { 166 | this.opMode = opMode; 167 | 168 | final String hashMethodProperty = (String) configs.get(CRYPTO_HASH_METHOD); 169 | 170 | if(hashMethodProperty != null && hashMethodProperty.length() != 0) { 171 | hashMethod = hashMethodProperty; 172 | } 173 | 174 | final String ignoreDecryptFailuresProperty = (String) configs.get(CRYPTO_IGNORE_DECRYPT_FAILURES); 175 | 176 | if(ignoreDecryptFailuresProperty != null && ignoreDecryptFailuresProperty.length() != 0) { 177 | ignoreDecryptFailures = Boolean.parseBoolean(ignoreDecryptFailuresProperty); 178 | } 179 | 180 | final String aesKeyLenProperty = (String) configs.get(CRYPTO_AES_KEY_LEN); 181 | 182 | if(aesKeyLenProperty != null && aesKeyLenProperty.length() != 0) { 183 | aesKeyLen = Integer.parseInt(aesKeyLenProperty); 184 | if(aesKeyLen < 128 || aesKeyLen % 8 != 0) { 185 | throw new KafkaException("Invalid aes key size, should be 128, 192 or 256"); 186 | } 187 | } 188 | 189 | try { 190 | if (opMode == Cipher.DECRYPT_MODE) { 191 | //Consumer 192 | String rsaPrivateKeyFile = (String) configs.get(CRYPTO_RSA_PRIVATEKEY_FILEPATH); 193 | consumerCryptoBundle = new ConsumerCryptoBundle(createRSAPrivateKey(readBytesFromFile(rsaPrivateKeyFile))); 194 | } else { 195 | //Producer 196 | String rsaPublicKeyFile = (String) configs.get(CRYPTO_RSA_PUBLICKEY_FILEPATH); 197 | producerCryptoBundle = new ProducerCryptoBundle(createRSAPublicKey(readBytesFromFile(rsaPublicKeyFile))); 198 | } 199 | } catch (Exception e) { 200 | throw new KafkaException(e); 201 | } 202 | } 203 | 204 | protected byte[] crypt(byte[] array) throws KafkaException { 205 | if (array == null || array.length == 0) { 206 | return array; 207 | } 208 | 209 | if (opMode == Cipher.DECRYPT_MODE) { 210 | //Consumer 211 | return consumerCryptoBundle.aesDecrypt(array); 212 | } else { 213 | //Producer 214 | return producerCryptoBundle.aesEncrypt(array); 215 | } 216 | } 217 | 218 | /** 219 | * Generate new AES key for the current thread 220 | */ 221 | protected void newKey() { 222 | try { 223 | producerCryptoBundle.newKey(); 224 | } catch (Exception e) { 225 | throw new KafkaException(e); 226 | } 227 | } 228 | 229 | //Hereafter there are only helper methods 230 | 231 | @SuppressWarnings("unchecked") 232 | protected T newInstance(Map map, String key, Class klass) throws KafkaException { 233 | Object val = map.get(key); 234 | if (val == null) { 235 | throw new KafkaException("No value for '" + key + "' found"); 236 | } else if (val instanceof String) { 237 | try { 238 | return (T) Utils.newInstance(Class.forName((String) val)); 239 | } catch (Exception e) { 240 | throw new KafkaException(e); 241 | } 242 | } else if (val instanceof Class) { 243 | return (T) Utils.newInstance((Class) val); 244 | } else { 245 | throw new KafkaException("Unexpected type '" + val.getClass() + "' for '" + key + "'"); 246 | } 247 | } 248 | 249 | private static PrivateKey createRSAPrivateKey(byte[] encodedKey) throws NoSuchAlgorithmException, InvalidKeySpecException { 250 | if (encodedKey == null || encodedKey.length == 0) { 251 | throw new IllegalArgumentException("Key bytes must not be null or empty"); 252 | } 253 | 254 | PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(encodedKey); 255 | KeyFactory kf = KeyFactory.getInstance(RSA); 256 | return kf.generatePrivate(spec); 257 | } 258 | 259 | private static SecretKey createAESSecretKey(byte[] encodedKey) { 260 | if (encodedKey == null || encodedKey.length == 0) { 261 | throw new IllegalArgumentException("Key bytes must not be null or empty"); 262 | } 263 | 264 | return new SecretKeySpec(encodedKey, AES); 265 | } 266 | 267 | private static PublicKey createRSAPublicKey(byte[] encodedKey) throws NoSuchAlgorithmException, InvalidKeySpecException { 268 | if (encodedKey == null || encodedKey.length == 0) { 269 | throw new IllegalArgumentException("Key bytes must not be null or empty"); 270 | } 271 | 272 | X509EncodedKeySpec spec = new X509EncodedKeySpec(encodedKey); 273 | KeyFactory kf = KeyFactory.getInstance(RSA); 274 | return kf.generatePublic(spec); 275 | } 276 | 277 | private static byte[] readBytesFromFile(String filename) throws IOException { 278 | if (filename == null) { 279 | throw new IllegalArgumentException("Filename must not be null"); 280 | } 281 | 282 | File f = new File(filename); 283 | DataInputStream dis = new DataInputStream(new FileInputStream(f)); 284 | byte[] bytes = new byte[(int) f.length()]; 285 | dis.readFully(bytes); 286 | dis.close(); 287 | return bytes; 288 | } 289 | 290 | private byte[] hash(byte[] toHash) { 291 | try { 292 | MessageDigest md = MessageDigest.getInstance(hashMethod); 293 | md.update(toHash); 294 | return md.digest(); 295 | } catch (Exception e) { 296 | throw new KafkaException(e); 297 | } 298 | } 299 | 300 | private static byte[] crypt(Cipher c, byte[] plain) throws IllegalBlockSizeException, BadPaddingException { 301 | return c.doFinal(plain); 302 | } 303 | 304 | private static byte[] crypt(Cipher c, byte[] plain, int offset, int len) throws IllegalBlockSizeException, BadPaddingException { 305 | return c.doFinal(plain, offset, len); 306 | } 307 | 308 | public static byte[] concatenate(byte[] a, byte[] b, byte[] c, byte[] d, byte[] e, byte[] f) { 309 | if (a != null && b != null && c != null && d != null && e != null && f != null) { 310 | byte[] rv = new byte[a.length + b.length + c.length + d.length + e.length + f.length]; 311 | System.arraycopy(a, 0, rv, 0, a.length); 312 | System.arraycopy(b, 0, rv, a.length, b.length); 313 | System.arraycopy(c, 0, rv, a.length + b.length, c.length); 314 | System.arraycopy(d, 0, rv, a.length + b.length + c.length, d.length); 315 | System.arraycopy(e, 0, rv, a.length + b.length + c.length + d.length, e.length); 316 | System.arraycopy(f, 0, rv, a.length + b.length + c.length + d.length + e.length, f.length); 317 | return rv; 318 | } else { 319 | throw new IllegalArgumentException("arrays must not be null"); 320 | } 321 | } 322 | } 323 | -------------------------------------------------------------------------------- /src/site/markdown/index.md: -------------------------------------------------------------------------------- 1 | # Kafka End2End encryption 2 | 3 | A small library with no external dependencies which provide transparent AES end-to-end encryption for Apache Kafka. 4 | 5 | ## Prerequisites 6 | 7 | * Kafka client 0.8.2.0 or higher 8 | * Java 6 or higher (Java 8 **stongly** recommended) 9 | * For AES 256 en-/decryption you have to install [JCE Unlimited Strength Jurisdiction Policy Files](http://www.oracle.com/technetwork/java/javase/overview/index.html) 10 | 11 | ## Features 12 | 13 | * Transparent encryption, will be configured as serializer/deserializer 14 | * End-to-end encryption of Kafka messages with random AES key(s) 15 | * AES key is attached in RSA encrypted form to every message 16 | * Consumer detects if a message is encrypted and can therefore handle also unencrypted messages 17 | * No external dependencies 18 | * On modern hardware with recent Java installed AES will be computed directly on the CPU (via AES-NI) 19 | 20 | ## Get it 21 | 22 | ### via Maven 23 | 24 | de.saly 25 | kafka-end-2-end-encryption 26 | 1.0.1 27 | 28 | 29 | ### or download .jar file 30 | 31 | [Here](http://search.maven.org/#search%7Cga%7C1%7Ckafka-end-2-end-encryption) 32 | 33 | ### or build it yourself 34 | 35 | mvn package 36 | 37 | Include the library in the classpath of the producer or consumer. 38 | 39 | ## Configuration 40 | 41 | This library provide a serializer and a deserializer which handles the encryption/decryption stuff and delegate the message then to an underlying serializer/deserializer. 42 | In other words: Your original serializer/deserializer will be wrapped with that one. 43 | 44 | ### Producer 45 | 46 | crypto.wrapped_serializer: #mandatory 47 | crypto.rsa.publickey.filepath: #mandatory 48 | crypto.aes.key_len: 128 #optional 49 | crypto.hash_method: SHA-256 #optional 50 | crypto.new_key_msg_interval: -1 #optional, Generate new AES every n messages (default is -1, that means never generate a new key) 51 | 52 | 53 | ### Consumer 54 | 55 | crypto.wrapped_deserializer: #mandatory 56 | crypto.rsa.privatekey.filepath: #mandatory 57 | #If set to true then the original message will be returned on decrypt failure 58 | #If set to false (the default) an exception will be thrown on decrypt failure 59 | crypto.ignore_decrypt_failures: false #optional 60 | 61 | ## Create a RSA key pair 62 | 63 | java -cp kafka-end-2-end-encryption-1.0.0.jar de.saly.kafka.crypto.RsaKeyGen 2048 64 | 65 | This creates a 2048-bit RSA key pair. The publickey is used on the consumer side to encrypt the AES key attached to every message. 66 | The privatekey is used on the consumer side to decrypt the AES key. 67 | 68 | ## Example 69 | 70 | ### Producer 71 | 72 | value.serializer: de.saly.kafka.crypto.EncryptingSerializer 73 | crypto.wrapped_serializer: org.apache.kafka.common.serialization.StringSerializer 74 | crypto.rsa.publickey.filepath: /opt/rsa_publickey_2048_db484e3c-c3f5-4197-bb40-2f60c498b157 75 | 76 | ### Consumer 77 | 78 | value.deserializer: de.saly.kafka.crypto.DecryptingDeserializer 79 | crypto.wrapped_deserializer: org.apache.kafka.common.serialization.StringDeserializer 80 | crypto.rsa.privatekey.filepath: /opt/rsa_privatekey_2048_db484e3c-c3f5-4197-bb40-2f60c498b157 81 | 82 | If you want also encrypt the key of the message use "key.serializer" and "key.deserializer" the same way. 83 | 84 | ## Limitations 85 | 86 | Currently value and key wrapped serializer cannot be different. 87 | -------------------------------------------------------------------------------- /src/site/site.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | Kafka End2End encryption 8 | Kafka End2End encryption 9 | /index.html 10 | 11 | 12 | 13 | 14 | true 15 | true 16 | true 17 | 18 | Kafka End2End encryption 19 | Kafka End2End encryption 20 | /index.html 21 | 22 | 23 | 24 | 25 | 26 | org.apache.maven.skins 27 | maven-fluido-skin 28 | 1.5 29 | 30 | 31 | 32 | 33 |

34 | 35 | 36 | 37 | 38 | 39 | 40 | 45 | 46 | -------------------------------------------------------------------------------- /src/test/java/de/saly/kafka/crypto/EnDecryptionTest.java: -------------------------------------------------------------------------------- 1 | package de.saly.kafka.crypto; 2 | 3 | import static org.junit.Assert.assertArrayEquals; 4 | import static org.junit.Assert.assertEquals; 5 | 6 | import java.io.File; 7 | import java.io.FileOutputStream; 8 | import java.security.KeyPair; 9 | import java.security.KeyPairGenerator; 10 | import java.util.ArrayList; 11 | import java.util.Arrays; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.Random; 16 | import java.util.concurrent.Callable; 17 | import java.util.concurrent.ExecutorService; 18 | import java.util.concurrent.Executors; 19 | import java.util.concurrent.Future; 20 | 21 | import javax.crypto.Cipher; 22 | 23 | import org.apache.kafka.common.KafkaException; 24 | import org.apache.kafka.common.serialization.ByteArrayDeserializer; 25 | import org.apache.kafka.common.serialization.ByteArraySerializer; 26 | import org.apache.kafka.common.serialization.Deserializer; 27 | import org.apache.kafka.common.serialization.StringDeserializer; 28 | import org.apache.kafka.common.serialization.StringSerializer; 29 | import org.junit.Assert; 30 | import org.junit.Assume; 31 | import org.junit.Test; 32 | 33 | public class EnDecryptionTest { 34 | 35 | private final static String TOPIC = "cryptedTestTopic"; 36 | private final File pubKey; 37 | private final File privKey; 38 | private final byte[] publicKey; 39 | private final byte[] privateKey; 40 | 41 | public EnDecryptionTest() throws Exception { 42 | pubKey = File.createTempFile("kafka", "crypto"); 43 | pubKey.deleteOnExit(); 44 | privKey = File.createTempFile("kafka", "crypto"); 45 | privKey.deleteOnExit(); 46 | 47 | KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); 48 | keyGen.initialize(2048); 49 | KeyPair pair = keyGen.genKeyPair(); 50 | publicKey = pair.getPublic().getEncoded(); 51 | privateKey = pair.getPrivate().getEncoded(); 52 | 53 | //System.out.println("private key format: "+pair.getPrivate().getFormat()); // PKCS#8 54 | //System.out.println("public key format: "+pair.getPublic().getFormat()); // X.509 55 | 56 | FileOutputStream fout = new FileOutputStream(pubKey); 57 | fout.write(publicKey); 58 | fout.close(); 59 | 60 | fout = new FileOutputStream(privKey); 61 | fout.write(privateKey); 62 | fout.close(); 63 | } 64 | 65 | @Test 66 | public void testBasicStandard() throws Exception { 67 | testBasic("", 128, -1); 68 | } 69 | 70 | @Test 71 | public void testAes256() throws Exception { 72 | Assume.assumeTrue(Cipher.getMaxAllowedKeyLength("AES") >= 256); 73 | testBasic("", 256, -1); 74 | } 75 | 76 | @Test 77 | public void testSHA1() throws Exception { 78 | testBasic("SHA1", 128, -1); 79 | } 80 | 81 | @Test 82 | public void testSHA1_192() throws Exception { 83 | Assume.assumeTrue(Cipher.getMaxAllowedKeyLength("AES") >= 192); 84 | testBasic("SHA1", 192, -1); 85 | } 86 | 87 | @Test 88 | public void testMD5_192() throws Exception { 89 | Assume.assumeTrue(Cipher.getMaxAllowedKeyLength("AES") >= 192); 90 | testBasic("MD5", 192, -1); 91 | } 92 | 93 | @Test 94 | public void testMSHA512_128() throws Exception { 95 | testBasic("SHA-512", 128, -1); 96 | } 97 | 98 | @Test(expected = KafkaException.class) 99 | public void testInvalidKeySize() throws Exception { 100 | testBasic("SHA1", 177, -1); 101 | } 102 | 103 | @Test(expected = KafkaException.class) 104 | public void testInvalidHashAlgo() throws Exception { 105 | testBasic("xxx", 128, -1); 106 | } 107 | 108 | @Test 109 | public void testBasicInterval1() throws Exception { 110 | testBasic("", 128, 1); 111 | } 112 | 113 | @Test 114 | public void testBasicInterval10() throws Exception { 115 | testBasic("", 128, 10); 116 | } 117 | 118 | @Test 119 | public void testMultithreadedStandard() throws Exception { 120 | testMultithreadedBasic(-1); 121 | } 122 | 123 | @Test 124 | public void testMultithreadedInterval1() throws Exception { 125 | testMultithreadedBasic(1); 126 | } 127 | 128 | @Test 129 | public void testMultithreadedInterval1000() throws Exception { 130 | testMultithreadedBasic(1000); 131 | } 132 | 133 | protected void testMultithreadedBasic(int msgInterval) throws Exception { 134 | final String str = "The quick brown fox jumps over the lazy dog"; 135 | 136 | final Map config = new HashMap(); 137 | config.put(SerdeCryptoBase.CRYPTO_RSA_PRIVATEKEY_FILEPATH, privKey.getAbsolutePath()); 138 | config.put(SerdeCryptoBase.CRYPTO_RSA_PUBLICKEY_FILEPATH, pubKey.getAbsolutePath()); 139 | config.put(EncryptingSerializer.CRYPTO_VALUE_SERIALIZER, StringSerializer.class.getName()); 140 | config.put(DecryptingDeserializer.CRYPTO_VALUE_DESERIALIZER, StringDeserializer.class); 141 | config.put(EncryptingSerializer.CRYPTO_NEW_KEY_MSG_INTERVAL, String.valueOf(msgInterval)); 142 | 143 | final EncryptingSerializer serializer = new EncryptingSerializer(); 144 | serializer.configure(config, false); 145 | 146 | final int threadCount = 200; 147 | 148 | final ExecutorService es = Executors.newFixedThreadPool(threadCount); 149 | final List> futures = new ArrayList>(); 150 | 151 | for (int i = 0; i < threadCount; i++) { 152 | Future f = es.submit(new Callable() { 153 | 154 | @Override 155 | public Exception call() throws Exception { 156 | try { 157 | final Deserializer deserializer = new DecryptingDeserializer(); 158 | deserializer.configure(config, false); 159 | for(int i=0; i<1000; i++) { 160 | final byte[] enc = serializer.serialize(TOPIC, str+i+Thread.currentThread().getName()); 161 | assertEquals(str+i+Thread.currentThread().getName(), deserializer.deserialize(TOPIC, enc)); 162 | } 163 | return null; 164 | } catch (Exception e) { 165 | return e; 166 | } 167 | 168 | } 169 | 170 | }); 171 | futures.add(f); 172 | } 173 | 174 | for (Future f : futures) { 175 | try { 176 | Exception e = f.get(); 177 | if (e != null) { 178 | throw e; 179 | } 180 | } catch (Exception e) { 181 | e.printStackTrace(); 182 | throw e; 183 | } 184 | } 185 | } 186 | 187 | protected void testBasic(String hashMethod, int keylen, int msgInterval) throws Exception { 188 | 189 | final Map config = new HashMap(); 190 | config.put(SerdeCryptoBase.CRYPTO_RSA_PRIVATEKEY_FILEPATH, privKey.getAbsolutePath()); 191 | config.put(SerdeCryptoBase.CRYPTO_RSA_PUBLICKEY_FILEPATH, pubKey.getAbsolutePath()); 192 | config.put(EncryptingSerializer.CRYPTO_VALUE_SERIALIZER, ByteArraySerializer.class.getName()); 193 | config.put(DecryptingDeserializer.CRYPTO_VALUE_DESERIALIZER, ByteArrayDeserializer.class); 194 | config.put(DecryptingDeserializer.CRYPTO_HASH_METHOD, hashMethod); 195 | config.put(DecryptingDeserializer.CRYPTO_AES_KEY_LEN, String.valueOf(keylen)); 196 | config.put(DecryptingDeserializer.CRYPTO_IGNORE_DECRYPT_FAILURES, "false"); 197 | config.put(EncryptingSerializer.CRYPTO_NEW_KEY_MSG_INTERVAL, String.valueOf(msgInterval)); 198 | 199 | final EncryptingSerializer serializer = new EncryptingSerializer(); 200 | serializer.configure(config, false); 201 | 202 | final Deserializer deserializer = new DecryptingDeserializer(); 203 | deserializer.configure(config, false); 204 | 205 | final Random rand = new Random(System.currentTimeMillis()); 206 | for (int i = 0; i < 1000; i++) { 207 | final byte[] b = new byte[i]; 208 | rand.nextBytes(b); 209 | Assert.assertArrayEquals(b, deserializer.deserialize(TOPIC, serializer.serialize(TOPIC, b))); 210 | } 211 | 212 | for (byte i = 0; i < Byte.MAX_VALUE; i++) { 213 | final byte[] b = new byte[i]; 214 | Arrays.fill(b, i); 215 | Assert.assertArrayEquals(b, deserializer.deserialize(TOPIC, serializer.serialize(TOPIC, b))); 216 | } 217 | 218 | serializer.newKey(); 219 | 220 | for (int i = 0; i < 100; i++) { 221 | final byte[] b = new byte[i]; 222 | rand.nextBytes(b); 223 | Assert.assertArrayEquals(b, deserializer.deserialize(TOPIC, serializer.serialize(TOPIC, b))); 224 | } 225 | byte[] plainText = "The quick brown fox jumps over the lazy dog".getBytes("UTF-8"); 226 | byte[] encryptedText = serializer.serialize(TOPIC, plainText); 227 | assertArrayEquals(SerdeCryptoBase.MAGIC_BYTES, Arrays.copyOfRange(encryptedText, 0, 2)); 228 | 229 | assertArrayEquals(plainText, deserializer.deserialize(TOPIC, plainText)); 230 | 231 | try { 232 | deserializer.deserialize(TOPIC, SerdeCryptoBase.MAGIC_BYTES); 233 | Assert.fail(); 234 | } catch (Exception e) { 235 | //expected 236 | } 237 | 238 | config.put(DecryptingDeserializer.CRYPTO_IGNORE_DECRYPT_FAILURES, "true"); 239 | deserializer.configure(config, false); 240 | 241 | try { 242 | assertArrayEquals(SerdeCryptoBase.MAGIC_BYTES, deserializer.deserialize(TOPIC, SerdeCryptoBase.MAGIC_BYTES)); 243 | } catch (Exception e) { 244 | Assert.fail(e.toString()); 245 | } 246 | } 247 | } 248 | --------------------------------------------------------------------------------