├── .github └── workflows │ ├── tests.yml │ └── trivy.yml ├── COPYRIGHT ├── LICENSE ├── README.md ├── bin ├── run-sink-connector.sh └── run-source-connector.sh ├── config ├── connect-avro-standalone-local.properties ├── connect-log4j.properties └── scylladb-sink-quickstart.properties ├── docker-compose.yml ├── documentation ├── CONFIG.md ├── EXAMPLES.md └── QUICKSTART.md ├── pom.xml └── src ├── main ├── java │ └── io │ │ └── connect │ │ └── scylladb │ │ ├── ClusterAddressTranslator.java │ │ ├── RecordConverter.java │ │ ├── RecordToBoundStatementConverter.java │ │ ├── ScyllaDbSchemaBuilder.java │ │ ├── ScyllaDbSession.java │ │ ├── ScyllaDbSessionFactory.java │ │ ├── ScyllaDbSessionImpl.java │ │ ├── ScyllaDbSinkConnector.java │ │ ├── ScyllaDbSinkConnectorConfig.java │ │ ├── ScyllaDbSinkTask.java │ │ ├── ScyllaDbSinkTaskHelper.java │ │ ├── TableMetadata.java │ │ ├── TableMetadataImpl.java │ │ ├── codec │ │ ├── ConvenienceCodecs.java │ │ ├── StringAbstractCodec.java │ │ ├── StringDurationCodec.java │ │ ├── StringInetCodec.java │ │ ├── StringTimeUuidCodec.java │ │ ├── StringUuidCodec.java │ │ └── StringVarintCodec.java │ │ ├── topictotable │ │ └── TopicConfigs.java │ │ └── utils │ │ ├── ListRecommender.java │ │ ├── NullOrReadableFile.java │ │ ├── ScyllaDbConstants.java │ │ ├── VersionUtil.java │ │ └── VisibleIfEqual.java └── resources │ └── io │ └── connect │ └── scylladb │ └── version.properties └── test ├── docker ├── configA │ ├── README.md │ ├── docker-compose.yml │ └── external.conf └── configB │ ├── README.md │ ├── docker-compose.yml │ └── external.conf ├── java └── io │ └── connect │ └── scylladb │ ├── ClusterAddressTranslatorTest.java │ ├── ScyllaDbSinkConnectorConfigTest.java │ ├── ScyllaDbSinkConnectorTest.java │ ├── ScyllaDbSinkTaskTest.java │ └── integration │ ├── RowValidator.java │ ├── ScyllaDbSinkConnectorIT.java │ ├── ScyllaNativeTypesIT.java │ ├── SinkRecordUtil.java │ ├── TestDataUtil.java │ └── codec │ ├── StringTimeUuidCodecTest.java │ └── StringUuidCodecTest.java └── resources └── log4j.properties /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | run-all-tests: 11 | name: Run all tests 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | 17 | - name: Set up JDK 8 18 | uses: actions/setup-java@v2 19 | with: 20 | java-version: '8' 21 | distribution: 'adopt' 22 | 23 | - name: Run all tests 24 | run: mvn -B verify 25 | -------------------------------------------------------------------------------- /.github/workflows/trivy.yml: -------------------------------------------------------------------------------- 1 | name: Vulnerability scan 2 | 3 | on: 4 | schedule: 5 | - cron: "44 16 * * *" 6 | push: 7 | pull_request: 8 | 9 | jobs: 10 | build: 11 | name: Trivy fs scan 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Run trivy in fs mode 17 | uses: aquasecurity/trivy-action@master 18 | with: 19 | scan-type: 'fs' 20 | scan-ref: '.' 21 | format: 'table' 22 | exit-code: '1' 23 | ignore-unfixed: false 24 | severity: 'CRITICAL,HIGH,MEDIUM' 25 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright ©ScyllaDB 2020 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ScyllaDB Sink Connector 2 | ======================== 3 | 4 | The ScyllaDB Sink Connector is a high-speed mechanism for reading records from Kafka and writing to ScyllaDB. 5 | 6 | Connector Installation 7 | ------------------------------- 8 | 9 | Clone the connector from Github repository and refer this [link](./documentation/QUICKSTART.md) for quickstart. 10 | 11 | ## Prerequisites 12 | The following are required to run the ScyllaDB Sink Connector: 13 | * Kafka Broker: Confluent Platform 3.3.0 or above. 14 | * Connect: Confluent Platform 4.1.0 or above. 15 | * Java 1.8 16 | * ScyllaDB: cqlsh 5.0.1 | Cassandra 3.0.8 | CQL spec 3.3.1 | Native protocol v4 17 | 18 | 19 | Usage Notes 20 | ----------- 21 | The ScyllaDB Sink Connector accepts two data formats from kafka. They are: 22 | * Avro Format 23 | * JSON with Schema 24 | * JSON without Schema 25 | 26 | **Note:** In case of JSON without schema, the table should already be present in the keyspace. 27 | 28 | This connector uses the topic name to determine the name of the table to write to. You can change this dynamically by using a 29 | transform like [Regex Router]() to change the topic name. 30 | 31 | To run this connector you can you a dockerized ScyllaDB instance. Follow this [link](https://hub.docker.com/r/scylladb/scylla/) for use. 32 | 33 | 34 | ----------------- 35 | Schema Management 36 | ----------------- 37 | 38 | You can configure this connector to manage the schema on the ScyllaDB cluster. When altering an existing table the key 39 | is ignored. This is to avoid the potential issues around changing a primary key on an existing table. The key schema is used to 40 | generate a primary key for the table when it is created. These fields must also be in the value schema. Data 41 | written to the table is always read from the value from Apache Kafka. This connector uses the topic to determine the name of 42 | the table to write to. This can be changed on the fly by using a transform to change the topic name. 43 | 44 | -------------------------- 45 | Time To Live (TTL) Support 46 | -------------------------- 47 | This connector provides support for TTL by which data can be automatically expired after a specific period. 48 | ``TTL`` value is the time to live value for the data. After that particular amount of time, data will be automatically deleted. For example, if the TTL value is set to 100 seconds then data would be automatically deleted after 100 seconds. 49 | To use this feature you have to set ``scylladb.ttl`` config with time(in seconds) for which you want to retain the data. If you don't specify this property then the record will be inserted with default TTL value null, meaning that written data will not expire. 50 | 51 | -------------------------------- 52 | Offset tracking Support in Kafka 53 | -------------------------------- 54 | This connector supports two types of offset tracking, but always stores them at least on Kafka. 55 | They will appear in internal ``__consumer_offsets`` topic and can be tracked by checking connector's consumer group 56 | using `kafka-consumer-groups` tool. 57 | 58 | **Offset stored in ScyllaDB Table** 59 | 60 | This is the default behaviour of the connector. The offsets will be additionally stored in table defined by `scylladb.offset.storage.table` property. 61 | Useful when all offsets need to be accessible in Scylla. 62 | 63 | **Offset stored in Kafka** 64 | 65 | For offsets to be managed only on Kafka, you must specify `scylladb.offset.storage.table.enable=false`. 66 | This will result in less total writes. Recommended option. 67 | 68 | 69 | ------------------- 70 | Delivery guarantees 71 | ------------------- 72 | This connector has at-least-once semantics. In case of a crash or restart, an `INSERT` operation of some rows 73 | might be performed multiple times (at least once). However, `INSERT` operations are idempotent in Scylla, meaning 74 | there won't be any duplicate rows in the destination table. 75 | 76 | The only time you could see the effect of duplicate `INSERT` operations is if your destination table has 77 | [Scylla CDC](https://docs.scylladb.com/using-scylla/cdc/) turned on. In the CDC log table you would see duplicate 78 | `INSERT` operations as separate CDC log rows. 79 | 80 | ----------------------- 81 | Reporting Kafka Metrics 82 | ----------------------- 83 | 84 | Refer the following [confluent documentation](https://docs.confluent.io/current/kafka/metrics-reporter.html) 85 | to access kafka related metrics. 86 | -------------------------------------------------------------------------------- /bin/run-sink-connector.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | # --------------------------------------- 5 | # Run from the project's parent directory 6 | # --------------------------------------- 7 | 8 | #[ 9 | : ${DEBUG:='n'} 10 | : ${SUSPEND:='n'} 11 | : ${BUILD:='n'} 12 | set -e 13 | 14 | if [ "$DEBUG" = "y" ]; then 15 | echo "Enabling debug on address 5005 with suspend=${SUSPEND}" 16 | export KAFKA_JMX_OPTS="-Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=${SUSPEND},address=5005" 17 | fi 18 | 19 | if [ "$BUILD" = "y" ]; then 20 | echo "Building the module" 21 | mvn clean package 22 | fi 23 | 24 | export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$(pwd)/config/connect-log4j.properties" 25 | 26 | echo "Starting standalone..." 27 | echo "Using log configuration ${KAFKA_LOG4J_OPTS}" 28 | 29 | connect-standalone config/connect-avro-standalone-local.properties config/my-sink-quickstart.properties 30 | -------------------------------------------------------------------------------- /bin/run-source-connector.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # --------------------------------------- 4 | # Run from the project's parent directory 5 | # --------------------------------------- 6 | 7 | #[ 8 | : ${DEBUG:='n'} 9 | : ${SUSPEND:='n'} 10 | : ${BUILD:='n'} 11 | set -e 12 | 13 | if [ "$DEBUG" = "y" ]; then 14 | echo "Enabling debug on address 5005 with suspend=${SUSPEND}" 15 | export KAFKA_JMX_OPTS="-Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=${SUSPEND},address=5005" 16 | fi 17 | 18 | if [ "$BUILD" = "y" ]; then 19 | echo "Building the module" 20 | mvn clean package 21 | fi 22 | 23 | export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$(pwd)/config/connect-log4j.properties" 24 | 25 | echo "Starting standalone..." 26 | echo "Using log configuration ${KAFKA_LOG4J_OPTS}" 27 | 28 | connect-standalone config/connect-avro-standalone-local.properties config/my-source-quickstart.properties 29 | -------------------------------------------------------------------------------- /config/connect-avro-standalone-local.properties: -------------------------------------------------------------------------------- 1 | # Sample configuration for a standalone Kafka Connect worker that uses Avro serialization and 2 | # integrates the the SchemaConfig Registry. 3 | # 4 | # This sample configuration assumes a local installation of Confluent Platform with all services 5 | # running on their default ports, and a local copy of the connector project. 6 | 7 | # Bootstrap Kafka servers. If multiple servers are specified, they should be comma-separated. 8 | bootstrap.servers=confluent:9092 9 | 10 | # The converters specify the format of data in Kafka and how to translate it into Connect data. 11 | # Every Connect user will need to configure these based on the format they want their data in 12 | # when loaded from or stored into Kafka 13 | key.converter=io.confluent.connect.avro.AvroConverter 14 | key.converter.schema.registry.url=http://confluent:8081 15 | value.converter=io.confluent.connect.avro.AvroConverter 16 | value.converter.schema.registry.url=http://confluent:8081 17 | 18 | # Local storage file for offset data 19 | offset.storage.file.filename=/tmp/connect.offsets 20 | 21 | # Confluent Control Center Integration -- uncomment these lines to enable Kafka client interceptors 22 | # that will report audit data that can be displayed and analyzed in Confluent Control Center 23 | # producer.interceptor.classes=io.confluent.connect.scylladb.monitoring.clients.interceptor.MonitoringProducerInterceptor 24 | # consumer.interceptor.classes=io.confluent.connect.scylladb.monitoring.clients.interceptor.MonitoringConsumerInterceptor 25 | 26 | # Load our plugin from the directory where a local Maven build creates the plugin archive. 27 | # Paths can be absolute or relative to the directory from where you run the Connect worker. 28 | plugin.path=target/components/packages/ -------------------------------------------------------------------------------- /config/connect-log4j.properties: -------------------------------------------------------------------------------- 1 | 2 | log4j.rootLogger=INFO, stdout 3 | 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n 7 | 8 | log4j.logger.org.apache.zookeeper=ERROR 9 | log4j.logger.org.I0Itec.zkclient=ERROR 10 | log4j.logger.org.reflections=ERROR 11 | log4j.logger.org.eclipse.jetty=ERROR 12 | #log4j.logger.org.apache.kafka.connect.runtime=DEBUG 13 | #log4j.logger.org.apache.kafka.clients.consumer.KafkaConsumer=DEBUG 14 | 15 | #log4j.logger.io.confluent.connect.scylladb=DEBUG 16 | -------------------------------------------------------------------------------- /config/scylladb-sink-quickstart.properties: -------------------------------------------------------------------------------- 1 | ### Mandatory configs: 2 | name=scylladb-sink-test 3 | topics= 4 | tasks.max=1 5 | connector.class=io.connect.scylladb.ScyllaDbSinkConnector 6 | 7 | scylladb.contact.points= 8 | #Eg. scylladb.contact.points=10.0.24.69,10.0.24.70,10.0.24.71 9 | # configure this to the public hostname of the Scylla nodes, the port will be taken from configuration scylladb.port 10 | 11 | #scylladb.contact.points={\"private_host1:port1\",\"public_host1:port1\", \"private_host2:port2\",\"public_host2:port2\", ...} 12 | #Eg. scylladb.contact.points={\"10.0.24.69:9042\": \"sl-eu-lon-2-portal.3.dblayer.com:15227\", \"10.0.24.71:9042\": \"sl-eu-lon-2-portal.2.dblayer.com:15229\", \"10.0.24.70:9042\": \"sl-eu-lon-2-portal.1.dblayer.com:15228\"} 13 | # configure this to a JSON string having key-values pairs of internal private network address(es) mapped to external network address(es). 14 | 15 | scylladb.keyspace= 16 | 17 | ### Connection based configs: 18 | #scylladb.port=9042 19 | #scylladb.loadbalancing.localdc=datacenter1 20 | #scylladb.compression=NONE 21 | #scylladb.security.enabled=false 22 | #scylladb.username=cassandra 23 | #scylladb.password=cassandra 24 | #scylladb.ssl.enabled=false 25 | 26 | ### Keyspace related configs: 27 | #scylladb.keyspace.create.enabled=true 28 | #scylladb.keyspace.replication.factor=3 29 | 30 | ### SSL based configs: 31 | #scylladb.ssl.provider=JDK 32 | #scylladb.ssl.truststore.path= 33 | #scylladb.ssl.truststore.password= 34 | #scylladb.ssl.keystore.path= 35 | #scylladb.ssl.keystore.password= 36 | #scylladb.ssl.cipherSuites= 37 | #scylladb.ssl.openssl.keyCertChain= 38 | #ssl.openssl.privateKey= 39 | 40 | ### ScyllaDB related configs: 41 | #behavior.on.error=FAIL 42 | 43 | ### Table related configs: 44 | #scylladb.table.manage.enabled=true 45 | #scylladb.table.create.compression.algorithm=NONE 46 | #scylladb.offset.storage.table=kafka_connect_offsets 47 | 48 | ### Topic to table related configs: 49 | #topic.my_topic.my_ks.my_table.mapping=column1=key.field1, column2=value.field1, __ttl=value.field2, __timestamp=value.field3, column3=header.field1 50 | #topic.my_topic.my_ks.my_table.consistencyLevel=LOCAL_ONE 51 | #topic.my_topic.my_ks.my_table.ttlSeconds=1 52 | #topic.my_topic.my_ks.my_table.deletesEnabled=true 53 | 54 | ### Writer configs 55 | #scylladb.consistency.level=LOCAL_QUORUM 56 | #scylladb.deletes.enabled=true 57 | #scylladb.execute.timeout.ms=30000 58 | #scylladb.ttl=null 59 | #scylladb.offset.storage.table.enable=true 60 | 61 | ### Converter configs(AVRO): 62 | #key.converter=io.confluent.connect.avro.AvroConverter 63 | #key.converter.schema.registry.url=http://localhost:8081 64 | #value.converter=io.confluent.connect.avro.AvroConverter 65 | #value.converter.schema.registry.url=http://localhost:8081 66 | #key.converter.schemas.enable=true 67 | #value.converter.schemas.enable=true 68 | 69 | ### Converter configs(JSON): 70 | #key.converter=org.apache.kafka.connect.json.JsonConverter 71 | #value.converter=org.apache.kafka.connect.json.JsonConverter 72 | #key.converter.schemas.enable=true 73 | #value.converter.schemas.enable=true 74 | #transforms=createKey 75 | #transforms.createKey.fields= 76 | #transforms.createKey.type=org.apache.kafka.connect.transforms.ValueToKey 77 | 78 | ### SMTs: 79 | #transforms=InsertSource,createKey 80 | #transforms.InsertSource.type=org.apache.kafka.connect.transforms.InsertField$Value 81 | #transforms.InsertSource.static.field=testing 82 | #transforms.InsertSource.static.value=success -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | version: "2" 3 | services: 4 | # for future e2e testing 5 | # zookeeper: 6 | # image: confluentinc/cp-zookeeper:3.2.2 7 | # environment: 8 | # ZOOKEEPER_CLIENT_PORT: "2181" 9 | # zk_id: "1" 10 | # ports: 11 | # - "2181:2181" 12 | # kafka: 13 | # hostname: kafka 14 | # image: confluentinc/cp-kafka:3.2.2 15 | # links: 16 | # - zookeeper 17 | # ports: 18 | # - "9092:9092" 19 | # environment: 20 | # KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 21 | # KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://:9092" 22 | # schema-registry: 23 | # image: confluentinc/cp-schema-registry:3.2.2 24 | # links: 25 | # - kafka 26 | # - zookeeper 27 | # ports: 28 | # - "8081:8081" 29 | # environment: 30 | # SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" 31 | # SCHEMA_REGISTRY_HOST_NAME: "schema-registry" 32 | scylladb: 33 | image: scylladb/scylla 34 | hostname: scylladb/scylla 35 | ports: 36 | - "9160:9160" 37 | - "9042:9042" 38 | -------------------------------------------------------------------------------- /documentation/CONFIG.md: -------------------------------------------------------------------------------- 1 | # ScyllaDB Sink Connector 2 | 3 | Configuration Properties 4 | ------------------------ 5 | 6 | To use this connector, specify the name of the connector class in the ``connector.class`` configuration property. 7 | 8 | connector.class=io.connect.scylladb.ScyllaDbSinkConnector 9 | 10 | Connector-specific configuration properties are described below. 11 | 12 | ### Connection 13 | 14 | ``scylladb.contact.points`` 15 | 16 | The ScyllaDB hosts to connect to. Scylla nodes use this list of hosts to find each other and learn the topology of the ring. 17 | You must change this if you are running multiple nodes. 18 | It's essential to put at least 2 hosts in case of bigger cluster, since if first host is down, 19 | it will contact second one and get the state of the cluster from it. 20 | Eg. When using the docker image, connect to the host it uses. 21 | To connect to private Scylla nodes, provide a JSON string having all internal private network address:port mapped to 22 | an external network address:port as key value pairs. Need to pass it as 23 | {\"private_host1:port1\",\"public_host1:port1\",\"private_host2:port2\",\"public_host2:port2\", ...} 24 | Eg. {\"10.0.24.69:9042\": \"sl-eu-lon-2-portal.3.dblayer.com:15227\", \"10.0.24.71:9042\": \"sl-eu-lon-2-portal.2.dblayer.com:15229\", \"10.0.24.70:9042\": \"sl-eu-lon-2-portal.1.dblayer.com:15228\"} 25 | 26 | * Type: List 27 | * Importance: High 28 | * Default Value: [localhost] 29 | 30 | ``scylladb.port`` 31 | 32 | The port the ScyllaDB hosts are listening on. 33 | Eg. When using a docker image, connect to the port it uses(use docker ps) 34 | 35 | * Type: Int 36 | * Importance: Medium 37 | * Default Value: 9042 38 | * Valid Values: ValidPort{start=1, end=65535} 39 | 40 | ``scylladb.loadbalancing.localdc`` 41 | 42 | The case-sensitive Data Center name local to the machine on which the connector is running. 43 | It is a recommended configuration if we have more than one DC. 44 | 45 | * Type: string 46 | * Default: "" 47 | * Importance: high 48 | 49 | ``scylladb.security.enabled`` 50 | 51 | To enable security while loading the sink connector and connecting to ScyllaDB. 52 | 53 | * Type: Boolean 54 | * Importance: High 55 | * Default Value: false 56 | 57 | ``scylladb.username`` 58 | 59 | The username to connect to ScyllaDB with. Set ``scylladb.security.enable = true`` to use this config. 60 | 61 | * Type: String 62 | * Importance: High 63 | * Default Value: cassandra 64 | 65 | ``scylladb.password`` 66 | 67 | The password to connect to ScyllaDB with. Set ``scylladb.security.enable = true`` to use this config. 68 | 69 | * Type: Password 70 | * Importance: High 71 | * Default Value: cassandra 72 | 73 | ``scylladb.compression`` 74 | 75 | Compression algorithm to use when connecting to ScyllaDB. 76 | 77 | * Type: string 78 | * Default: NONE 79 | * Valid Values: [NONE, SNAPPY, LZ4] 80 | * Importance: low 81 | 82 | ``scylladb.ssl.enabled`` 83 | 84 | Flag to determine if SSL is enabled when connecting to ScyllaDB. 85 | 86 | * Type: boolean 87 | * Default: false 88 | * Importance: high 89 | 90 | ### SSL 91 | 92 | ``scylladb.ssl.truststore.path`` 93 | 94 | Path to the Java Truststore. 95 | 96 | * Type: string 97 | * Default: "" 98 | * Importance: medium 99 | 100 | ``scylladb.ssl.truststore.password`` 101 | 102 | Password to open the Java Truststore with. 103 | 104 | * Type: password 105 | * Default: [hidden] 106 | * Importance: medium 107 | 108 | ``scylladb.ssl.provider`` 109 | 110 | The SSL Provider to use when connecting to ScyllaDB. 111 | 112 | * Type: string 113 | * Default: JDK 114 | * Valid Values: [JDK, OPENSSL, OPENSSL_REFCNT] 115 | * Importance: low 116 | 117 | ### Keyspace 118 | 119 | **Note**: Both keyspace and table names consist of only alphanumeric characters, 120 | cannot be empty and are limited in size to 48 characters (that limit exists 121 | mostly to avoid filenames, which may include the keyspace and table name, 122 | to go over the limits of certain file systems). By default, keyspace and table names 123 | are case insensitive (myTable is equivalent to mytable) but case sensitivity 124 | can be forced by using double-quotes ("myTable" is different from mytable). 125 | 126 | ``scylladb.keyspace`` 127 | 128 | The keyspace to write to. This keyspace is like a database in the ScyllaDB cluster. 129 | * Type: String 130 | * Importance: High 131 | 132 | ``scylladb.keyspace.create.enabled`` 133 | 134 | Flag to determine if the keyspace should be created if it does not exist. 135 | **Note**: Error if a new keyspace has to be created and the config is false. 136 | 137 | * Type: Boolean 138 | * Importance: High 139 | * Default Value: true 140 | 141 | ``scylladb.keyspace.replication.factor`` 142 | 143 | The replication factor to use if a keyspace is created by the connector. 144 | The Replication Factor (RF) is equivalent to the number of nodes where data (rows and partitions) 145 | are replicated. Data is replicated to multiple (RF=N) nodes 146 | 147 | * Type: int 148 | * Default: 3 149 | * Valid Values: [1,...] 150 | * Importance: high 151 | 152 | ### Table 153 | 154 | ``scylladb.table.manage.enabled`` 155 | 156 | Flag to determine if the connector should manage the table. 157 | 158 | * Type: Boolean 159 | * Importance: High 160 | * Default Value: true 161 | 162 | ``scylladb.table.create.compression.algorithm`` 163 | 164 | Compression algorithm to use when the table is created. 165 | 166 | * Type: string 167 | * Default: NONE 168 | * Valid Values: [NONE, SNAPPY, LZ4, DEFLATE] 169 | * Importance: medium 170 | 171 | ``scylladb.offset.storage.table`` 172 | 173 | The table within the ScyllaDB keyspace to store the offsets that have been read from Apache Kafka. 174 | 175 | * Type: String 176 | * Importance: Low 177 | * Default: kafka_connect_offsets 178 | 179 | ### Topic to Table 180 | 181 | These configurations can be specified for multiple Kafka topics from which records are being processed. 182 | Also, these topic level configurations will override the behavior of Connector level configurations such as 183 | ``scylladb.consistency.level``, ``scylladb.deletes.enabled`` and ``scylladb.ttl`` 184 | 185 | ``topic....mapping`` 186 | 187 | For mapping topic and fields from Kafka record's key, value and headers to ScyllaDB table and its columns. 188 | `my_topic` should refer to the topic name as seen in a SinkRecord passed to the Connector - which means after all Single Message Transforms. 189 | For example if you're using RegexRouter to change the topic name from `top1` to `top2` you would use `topic.top2.ks.table.mapping=...` property. 190 | 191 | **Note**: Ensure that the data type of the Kafka record's fields are compatible with the data type of the ScyllaDB column. 192 | In the Kafka topic mapping, you can optionally specify which column should be used as the ttl (time-to-live) and 193 | timestamp of the record being inserted into the database table using the special property __ttl and __timestamp. 194 | By default, the database internally tracks the write time(timestamp) of records inserted into Kafka. 195 | However, this __timestamp feature in the mapping supports the scenario where the Kafka records have an explicit 196 | timestamp field that you want to use as a write time for the database record produced by the connector. 197 | Eg. "topic.my_topic.my_ks.my_table.mapping": 198 | "column1=key.field1, column2=value.field1, __ttl=value.field2, __timestamp=value.field3, column3=header.field1" 199 | 200 | ``topic.my_topic.my_ks.my_table.consistencyLevel`` 201 | 202 | By using this property we can specify table wide consistencyLevel. 203 | 204 | ``topic.my_topic.my_ks.my_table.ttlSeconds`` 205 | 206 | By using this property we can specify table wide ttl(time-to-live). 207 | 208 | ``topic.my_topic.my_ks.my_table.deletesEnabled`` 209 | 210 | By using this property we can specify if tombstone records(records with Kafka value as null) 211 | should processed as delete request. 212 | 213 | 214 | ### Write 215 | 216 | ``scylladb.consistency.level`` 217 | 218 | The requested consistency level to use when writing to ScyllaDB. The Consistency Level (CL) determines how many replicas in a cluster that must acknowledge read or write operations before it is considered successful. 219 | 220 | * Type: String 221 | * Importance: High 222 | * Default Value: LOCAL_QUORUM 223 | * Valid Values: ``ANY``, ``ONE``, ``TWO``, ``THREE``, ``QUORUM``, ``ALL``, ``LOCAL_QUORUM``, ``EACH_QUORUM``, ``SERIAL``, ``LOCAL_SERIAL``, ``LOCAL_ONE`` 224 | 225 | ``scylladb.deletes.enabled`` 226 | 227 | Flag to determine if the connector should process deletes. 228 | The Kafka records with kafka record value as null will result in deletion of ScyllaDB record 229 | with the primary key present in Kafka record key. 230 | 231 | * Type: boolean 232 | * Default: true 233 | * Importance: high 234 | 235 | ``scylladb.execute.timeout.ms`` 236 | 237 | The timeout for executing a ScyllaDB statement. 238 | 239 | * Type: Long 240 | * Importance: Low 241 | * Valid Values: [0,...] 242 | * Default Value: 30000 243 | 244 | ``scylladb.ttl`` 245 | 246 | The retention period for the data in ScyllaDB. After this interval elapses, ScyllaDB will remove these records. If this configuration is not provided, the Sink Connector will perform insert operations in ScyllaDB without TTL setting. 247 | 248 | * Type: Int 249 | * Importance: Medium 250 | * Default Value: null 251 | 252 | ``scylladb.offset.storage.table.enable`` 253 | 254 | If true, Kafka consumer offsets will be additionally stored in ScyllaDB table. If false, connector will skip writing offset 255 | information into ScyllaDB. 256 | 257 | * Type: Boolean 258 | * Importance: Medium 259 | * Default Value: True 260 | 261 | ### ScyllaDB 262 | 263 | ``behavior.on.error`` 264 | 265 | Error handling behavior setting. Must be configured to one of the following: 266 | 267 | ``fail`` 268 | 269 | The Connector throws ConnectException and stops processing records when an error occurs while processing or inserting records into ScyllaDB. 270 | 271 | ``ignore`` 272 | 273 | Continues to process next set of records when error occurs while processing or inserting records into ScyllaDB. 274 | 275 | ``log`` 276 | 277 | Logs the error via connect-reporter when an error occurs while processing or inserting records into ScyllaDB and continues to process next set of records, available in the kafka topics. 278 | 279 | * Type: string 280 | * Default: FAIL 281 | * Valid Values: [FAIL, LOG, IGNORE] 282 | * Importance: medium 283 | 284 | 285 | ### Confluent Platform Configurations. 286 | 287 | ``tasks.max`` 288 | 289 | The maximum number of tasks to use for the connector that helps in parallelism. 290 | 291 | * Type:int 292 | * Default: 1 293 | * Importance: high 294 | 295 | ``topics`` 296 | 297 | The name of the topics to consume data from and write to ScyllaDB. 298 | 299 | * Type: list 300 | * Importance: high 301 | 302 | ``confluent.topic.bootstrap.servers`` 303 | 304 | A list of host/port pairs to use for establishing the initial connection to the Kafka cluster used for licensing. All servers in the cluster will be discovered from the initial connection. This list should be in the form host1:port1,host2:port2,…. Since these servers are just used for the initial connection to discover the full cluster membership (which may change dynamically), this list need not contain the full set of servers (you may want more than one, though, in case a server is down). 305 | 306 | * Type: list 307 | * Default: localhost:9092 308 | * Importance: high 309 | -------------------------------------------------------------------------------- /documentation/EXAMPLES.md: -------------------------------------------------------------------------------- 1 | # Example setups 2 | Below are examples of non-trivial connector configurations. 3 | For simpler step-by-step instructions with environment setup check out [quickstart](QUICKSTART.md) first. 4 | 5 | 6 | ## One topic to many tables 7 | 8 | This can be achieved by running multiple instances of the connector. 9 | 10 | #### Environment 11 | Running Kafka cluster and dockerized Scylla (contact point `172.17.0.2`) 12 | Test data generated using [Datagen Source Connector](https://www.confluent.io/hub/confluentinc/kafka-connect-datagen) 13 | with following configuration: 14 | ``` 15 | name = DatagenConnectorExample_1 16 | connector.class = io.confluent.kafka.connect.datagen.DatagenConnector 17 | kafka.topic = usersTopic 18 | quickstart = users 19 | ``` 20 | 21 | #### Connectors configuration 22 | We will use 2 connectors for this example. 23 | 24 | Connector1.properties: 25 | ``` 26 | name = ScyllaDbSinkConnectorExample_1 27 | connector.class = io.connect.scylladb.ScyllaDbSinkConnector 28 | transforms = createKey 29 | topics = usersTopic 30 | transforms.createKey.type = org.apache.kafka.connect.transforms.ValueToKey 31 | transforms.createKey.fields = userid 32 | scylladb.contact.points = 172.17.0.2 33 | scylladb.consistency.level = ONE 34 | scylladb.keyspace = example_ks 35 | scylladb.keyspace.replication.factor = 1 36 | scylladb.offset.storage.table = kafka_connect_offsets 37 | ``` 38 | Connector2.properties: 39 | ``` 40 | name = ScyllaDbSinkConnectorExample_2 41 | connector.class = io.connect.scylladb.ScyllaDbSinkConnector 42 | transforms = createKey, ChangeTopic 43 | topics = usersTopic 44 | transforms.createKey.type = org.apache.kafka.connect.transforms.ValueToKey 45 | transforms.createKey.fields = userid 46 | transforms.ChangeTopic.type = org.apache.kafka.connect.transforms.RegexRouter 47 | transforms.ChangeTopic.regex = usersTopic 48 | transforms.ChangeTopic.replacement = ChangedTopic 49 | scylladb.contact.points = 172.17.0.2 50 | scylladb.consistency.level = ONE 51 | scylladb.keyspace = example_ks 52 | scylladb.keyspace.replication.factor = 1 53 | scylladb.offset.storage.table = kafka_connect_offsets_2 54 | topic.ChangedTopic.example_ks.ChangedTopic.mapping = mappedUserIdCol=key.userid,mappedGenderCol=value.gender 55 | ``` 56 | This setup results in creation of 4 tables in `example_ks` keyspace. Two different for keeping offsets and two different for data. 57 | 58 | Connector1 creates `userstopic` which should look like table below. Keeps its offsets in `kafka_connect_offsets`. 59 | 
 userid | gender | regionid | registertime
60 | --------+--------+----------+---------------
61 |  User_3 |   MALE | Region_3 | 1497171901434
62 |  User_1 |  OTHER | Region_2 | 1515602353163
63 |  User_6 |  OTHER | Region_3 | 1512008940490
64 |  User_7 |   MALE | Region_7 | 1507294138815
65 |  User_2 | FEMALE | Region_2 | 1493737097490
66 |
67 | Connector2 uses RegexRouter SMT to change topic name to `changedtopic`. This results in creation of `changedtopic` table. Additionally it makes use of custom mapping property (`topic.ChangedTopic.example_ks...`) to define a different structure for this table. 68 | This results in following: 69 | 
cqlsh> SELECT * FROM example_ks.changedtopic 
70 |    ... ;
71 | 
72 |  mappedUserIdCol | mappedGenderCol
73 | -----------------+-----------------
74 |           User_3 |           OTHER
75 |           User_1 |           OTHER
76 |           User_6 |           OTHER
77 |           User_7 |            MALE
78 |
-------------------------------------------------------------------------------- /documentation/QUICKSTART.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | This quickstart will show how to setup the ScyllaDB Sink Connector against a Dockerized ScyllaDB. 4 | 5 | 6 | ## Preliminary Setup 7 | 8 | ###Docker Setup 9 | This [link](https://hub.docker.com/r/scylladb/scylla/) provides docker commands to bring up ScyllaDB. 10 | 11 | Command to start ScyllaDB docker container: 12 | 13 | ``` 14 | $ docker run --name some-scylla --hostname some-scylla -d scylladb/scylla 15 | ``` 16 | Running `docker ps` will show you the exposed ports, which should look something like the following: 17 | 18 | ``` 19 | $ docker ps 20 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 21 | 26cc6d47efe3 replace-with-image-name "/docker-entrypoint.…" 4 hours ago Up 23 seconds 0.0.0.0:32777->1883/tcp, 0.0.0.0:32776->9001/tcp anonymous_my_1 22 | ``` 23 | 24 | ### Confluent Platform Installation 25 | 26 | If you are new to Confluent then follow this [link](https://www.confluent.io/download) to download the Confluent Platform . 27 | 28 | 29 | 1. Click on DOWNLOAD FREE under Self managed software. 30 | 31 | 2. Click on Zip archive then fill the Email address then Accept the T&C and lastly click on Download Version 5.X.X. 32 | 33 | 3. Extract the downloaded file and paste it to the desired location. 34 | 35 | 4. Now follow this [link](https://docs.confluent.io/current/quickstart/ce-quickstart.html#ce-quickstart) to complete the installation. 36 | 37 | 38 | ### Manual Installation Of The Connector 39 | 40 | For manual installation, navigate to the following github link and clone the repository. 41 | 42 | ``https://github.com/scylladb/kafka-connect-scylladb`` 43 | 44 | Follow these steps to build the project: 45 | * Open source code folder in terminal. 46 | * Run the command ``mvn clean install``. 47 | * Run the Integration Tests in an IDE. If tests fail run ``mvn clean install -DskipTests``. 48 | 49 | Note: To run Integration Tests there is no need to run Confluent. Use docker-compose.yml file in the github repository and run the following command( it contains images to run kafka and other services): 50 | 51 | ``docker-compose -f docker-compose.yml up`` 52 | 53 | After completion of the above steps, a folder by the name of ‘components’ will be created in the target folder of the source code folder. 54 | The Connector's full package is present in ``{source-code-folder}/target/components/packages/ScyllaDB-kafka-connect-scylladb-`` 55 | 56 | Navigate to your Confluent Platform installation directory and place this folder in `{confluent-directory}/share/java`. 57 | In case of different Kafka Connect installations you can modify `plugin.path` property so that it includes Connector's package folder. 58 | 59 | ## Sink Connector 60 | 61 | The ScyllaDB sink connector is used to publish records from a Kafka topic into ScyllaDB. 62 | 63 | Adding a new connector plugin requires restarting Connect. Use the Confluent CLI to restart Connect: 64 | 65 | ``` 66 | $ confluent local stop && confluent local start 67 | Starting zookeeper 68 | zookeeper is [UP] 69 | Starting kafka 70 | kafka is [UP] 71 | Starting schema-registry 72 | schema-registry is [UP] 73 | Starting kafka-rest 74 | kafka-rest is [UP] 75 | Starting connect 76 | connect is [UP] 77 | ``` 78 | 79 | Check if the kafka-connect-scylladb connector plugin has been installed correctly and picked up by the plugin loader: 80 | 81 | ``` 82 | $ curl -sS localhost:8083/connector-plugins | jq .[].class | grep ScyllaDbSinkConnector 83 | ``` 84 | Your output should resemble: 85 | 86 | ``` 87 | "io.connect.scylladb.ScyllaDbSinkConnector" 88 | ``` 89 | 90 | #### Connector Configuration 91 | 92 | Save these configs in a file *kafka-connect-scylladb.json* and run the following command: 93 | 94 | ```json 95 | { 96 | "name" : "scylladb-sink-connector", 97 | "config" : { 98 | "connector.class" : "io.connect.scylladb.ScyllaDbSinkConnector", 99 | "tasks.max" : "1", 100 | "topics" : "topic1,topic2,topic3", 101 | "scylladb.contact.points" : "scylladb-hosts", 102 | "scylladb.keyspace" : "test" 103 | } 104 | } 105 | ``` 106 | 107 | Use this command to load the connector: 108 | 109 | ``` 110 | curl -s -X POST -H 'Content-Type: application/json' --data @kafka-connect-scylladb.json http://localhost:8083/connectors 111 | ``` 112 | 113 | Use the following command to update the configuration of existing connector. 114 | 115 | ``` 116 | curl -s -X PUT -H 'Content-Type: application/json' --data @kafka-connect-scylladb.json http://localhost:8083/connectors/scylladb/config 117 | ``` 118 | 119 | Once the Connector is up and running, use the command ``kafka-avro-console-producer`` to produce records(in AVRO format) into Kafka topic. 120 | 121 | Example: 122 | 123 | ``` 124 | kafka-avro-console-producer \ 125 | --broker-list localhost:9092 \ 126 | --topic topic1 \ 127 | --property parse.key=true \ 128 | --property key.schema='{"type":"record","name":"key_schema","fields":[{"name":"id","type":"int"}]}' \ 129 | --property "key.separator=$" \ 130 | --property value.schema='{"type":"record","name":"value_schema","fields":[{"name":"id","type":"int"},{"name":"firstName","type":"string"},{"name":"lastName","type":"string"}]}' 131 | {"id":1}${"id":1,"firstName":"first","lastName":"last"} 132 | ``` 133 | 134 | Output upon running the select query in ScyllaDB: 135 | 136 | ``` 137 | select * from test.topic1; 138 | 139 | id | firstname | lastname 140 | 141 | ----+-----------+---------- 142 | 143 | 1 | first | last 144 | ``` 145 | 146 | 147 | ## Modes in ScyllaDB 148 | 149 | ### Standard 150 | 151 | Use this command to load the connector in : 152 | 153 | ``` 154 | curl -s -X POST -H 'Content-Type: application/json' --data @kafka-connect-scylladb.json http://localhost:8083/connectors 155 | ``` 156 | 157 | This example will connect to ScyllaDB instance without authentication. 158 | 159 | Select one of the following configuration methods based on how you have deployed |kconnect-long|. 160 | Distributed Mode will the JSON / REST examples. Standalone mode will use the properties based 161 | example. 162 | 163 | **Note**: Each json record should consist of a schema and payload. 164 | 165 | 166 | **Distributed Mode JSON** 167 | 168 | ```json 169 | { 170 | "name" : "scylladb-sink-connector", 171 | "config" : { 172 | "connector.class" : "io.connect.scylladb.ScyllaDbSinkConnector", 173 | "tasks.max" : "1", 174 | "topics" : "topic1,topic2,topic3", 175 | "scylladb.contact.points" : "scylladb-hosts", 176 | "scylladb.keyspace" : "test", 177 | "key.converter" : "org.apache.kafka.connect.json.JsonConverter", 178 | "value.converter" : "org.apache.kafka.connect.json.JsonConverter", 179 | "key.converter.schemas.enable" : "true", 180 | "value.converter.schemas.enable" : "true", 181 | 182 | "transforms" : "createKey", 183 | "transforms.createKey.fields" : "[field-you-want-as-primary-key-in-scylla]", 184 | "transforms.createKey.type" : "org.apache.kafka.connect.transforms.ValueToKey" 185 | } 186 | } 187 | ``` 188 | 189 | **Standalone Mode Json** 190 | 191 | To load the connector in Standalone mode use: 192 | 193 | ``` 194 | confluent local load scylladb-sink-conector -- -d scylladb-sink-connector.properties 195 | ``` 196 | Use the following configs: 197 | 198 | ``` 199 | scylladb.class=io.connect.scylladb.ScyllaDbSinkConnector 200 | tasks.max=1 201 | topics=topic1,topic2,topic3 202 | scylladb.contact.points=cassandra 203 | scylladb.keyspace=test 204 | 205 | key.converter=org.apache.kafka.connect.json.JsonConverter 206 | value.converter=org.apache.kafka.connect.json.JsonConverter 207 | key.converter.schemas.enable=true 208 | value.converter.schemas.enable=true 209 | 210 | transforms=createKey 211 | transforms.createKey.fields=[field-you-want-as-primary-key-in-scylla] 212 | transforms.createKey.type=org.apache.kafka.connect.transforms.ValueToKey 213 | ``` 214 | 215 | Example: 216 | 217 | ``` 218 | kafka-console-producer --broker-list localhost:9092 --topic sample-topic 219 | >{"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"name"},{"type":"string","optional":true,"field":"department"}],"payload":{"id":10,"name":"John Doe10","department":"engineering"}}} 220 | ``` 221 | 222 | Run the select query to view the data: 223 | 224 | ``` 225 | Select * from keyspace_name.topic-name; 226 | ``` 227 | 228 | **Note**: To publish records in Avro Format use the following properties: 229 | 230 | ``` 231 | key.converter=io.confluent.connect.avro.AvroConverter 232 | key.converter.schema.registry.url=http://localhost:8081 233 | value.converter=io.confluent.connect.avro.AvroConverter 234 | value.converter.schema.registry.url=http://localhost:8081 235 | key.converter.schemas.enable=true 236 | value.converter.schemas.enable=true 237 | ``` 238 | 239 | ---------------------- 240 | Authentication 241 | ---------------------- 242 | 243 | This example will connect to ScyllaDB instance with security enabled and username / password authentication. 244 | 245 | 246 | Select one of the following configuration methods based on how you have deployed |kconnect-long|. 247 | Distributed Mode will the JSON / REST examples. Standalone mode will use the properties based 248 | example. 249 | 250 | 251 | **Distributed Mode** 252 | 253 | ```json 254 | { 255 | "name" : "scylladbSinkConnector", 256 | "config" : { 257 | "connector.class" : "io.connect.scylladb.ScyllaDbSinkConnector", 258 | "tasks.max" : "1", 259 | "topics" : "topic1,topic2,topic3", 260 | "scylladb.contact.points" : "cassandra", 261 | "scylladb.keyspace" : "test", 262 | "scylladb.security.enabled" : "true", 263 | "scylladb.username" : "example", 264 | "scylladb.password" : "password", 265 | **add other properties same as in the above example** 266 | } 267 | } 268 | ``` 269 | 270 | 271 | **Standalone Mode** 272 | 273 | ``` 274 | connector.class=io.connect.scylladb.ScyllaDbSinkConnector 275 | tasks.max=1 276 | topics=topic1,topic2,topic3 277 | scylladb.contact.points=cassandra 278 | scylladb.keyspace=test 279 | scylladb.ssl.enabled=true 280 | scylladb.username=example 281 | scylladb.password=password 282 | ``` 283 | 284 | ### Logging 285 | 286 | To check logs for the Confluent Platform use: 287 | 288 | ``` 289 | confluent local log -- [] --path 290 | ``` 291 | To check logs for Scylla: 292 | 293 | ``` 294 | $ docker logs some-scylla | tail 295 | ``` 296 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 4.0.0 5 | io.kafka.connect 6 | kafka-connect-scylladb 7 | kafka-connect-scylladb 8 | 1.1.1 9 | A Kafka Connect plugin for Scylla Database 10 | https://github.com/scylla/kafka-connect-scylladb 11 | 2020 12 | 13 | 14 | scm:git:https://github.com/scylla/kafka-connect-scylladb.git 15 | scm:git:git@github.com:scylla/kafka-connect-scylladb.git 16 | 17 | https://github.com/scylladb/kafka-connect-scylladb 18 | HEAD 19 | 20 | 21 | 22 | 2020 23 | 2020 24 | 0.11.1 25 | 1.2.0 26 | 3.0.0-M5 27 | 3.0.0-M5 28 | 2.4.0 29 | 5.2.0 30 | 4.17.0.0 31 | false 32 | 33 | 2.13.4.2 34 | 24.1.1-jre 35 | 36 | 37 | 38 | 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-assembly-plugin 43 | 3.2.0 44 | 45 | 46 | org.apache.maven.plugins 47 | maven-compiler-plugin 48 | 3.5.1 49 | true 50 | 51 | 1.8 52 | 1.8 53 | 54 | 55 | 56 | com.mycila 57 | license-maven-plugin 58 | 59 | 60 | src/test/docker/** 61 | config/* 62 | 63 | 64 | 65 | 66 | io.confluent 67 | kafka-connect-maven-plugin 68 | ${kafka.connect.maven.plugin.version} 69 | 70 | 71 | 72 | kafka-connect 73 | 74 | 75 | Kafka Connect Scylla Database Connector 76 | ${project.version}-preview 77 | ScyllaDB 78 | ScyllaDB. 79 | 80 | sink 81 | 82 | 83 | Scylla Database 84 | 85 | true 86 | 87 | 88 | 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-surefire-plugin 93 | ${surefire.version} 94 | 95 | org.apache.kafka.test.IntegrationTest 96 | false 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-failsafe-plugin 102 | ${failsafe.version} 103 | 104 | false 105 | ${skipIntegrationTests} 106 | **/*IT.java 107 | 108 | ${docker.host.address} 109 | 110 | 111 | 112 | 113 | 114 | integration-test 115 | verify 116 | 117 | 118 | 119 | 120 | 121 | org.jacoco 122 | jacoco-maven-plugin 123 | 124 | 125 | 126 | prepare-agent 127 | 128 | 129 | 130 | generate-code-coverage-report 131 | test 132 | 133 | report 134 | 135 | 136 | 137 | 138 | 139 | io.fabric8 140 | docker-maven-plugin 141 | 0.39.0 142 | 143 | 144 | Always 145 | 146 | 147 | composeImages 148 | fabric8/compose-demo:latest 149 | 150 | compose 151 | ${basedir} 152 | docker-compose.yml 153 | 154 | 155 | 156 | 157 | init - serving 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | start 168 | pre-integration-test 169 | 170 | build 171 | start 172 | 173 | 174 | 175 | stop 176 | post-integration-test 177 | 178 | stop 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | true 188 | src/main/resources 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | com.fasterxml.jackson.core 197 | jackson-databind 198 | ${jackson-databind.version} 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | io.confluent.kafka 208 | connect-utils 209 | [0.1.0,0.1.100) 210 | 211 | 212 | 213 | org.apache.kafka 214 | connect-api 215 | ${kafka.version} 216 | provided 217 | 218 | 219 | 220 | org.apache.kafka 221 | kafka-clients 222 | ${kafka.version} 223 | provided 224 | 225 | 226 | 227 | com.scylladb 228 | java-driver-core 229 | ${scylladb.version} 230 | 231 | 232 | 233 | com.scylladb 234 | java-driver-query-builder 235 | ${scylladb.version} 236 | 237 | 238 | 239 | com.scylladb 240 | java-driver-mapper-runtime 241 | ${scylladb.version} 242 | 243 | 244 | 245 | org.apache.kafka 246 | connect-runtime 247 | ${kafka.version} 248 | test 249 | 250 | 251 | org.apache.kafka 252 | connect-runtime 253 | ${kafka.version} 254 | test-jar 255 | test 256 | test 257 | 258 | 259 | 260 | org.apache.kafka 261 | kafka-clients 262 | ${kafka.version} 263 | test 264 | test-jar 265 | test 266 | 267 | 268 | 269 | junit 270 | junit 271 | 4.13.1 272 | test 273 | 274 | 275 | 276 | org.mockito 277 | mockito-core 278 | 2.23.4 279 | test 280 | 281 | 282 | 283 | org.junit.jupiter 284 | junit-jupiter-api 285 | ${junit.version} 286 | test 287 | 288 | 289 | 290 | org.lz4 291 | lz4-java 292 | 293 | 1.7.1 294 | 295 | 296 | 297 | org.xerial.snappy 298 | snappy-java 299 | 300 | 1.1.10.4 301 | 302 | 303 | 304 | 305 | 310 | 311 | 312 | 313 | 314 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/ClusterAddressTranslator.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Compose, an IBM company 2 | // 3 | // MIT License 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining 6 | // a copy of this software and associated documentation files (the 7 | // "Software"), to deal in the Software without restriction, including 8 | // without limitation the rights to use, copy, modify, merge, publish, 9 | // distribute, sublicense, and/or sell copies of the Software, and to 10 | // permit persons to whom the Software is furnished to do so, subject to 11 | // the following conditions: 12 | // 13 | // The above copyright notice and this permission notice shall be 14 | // included in all copies or substantial portions of the Software. 15 | // 16 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | package io.connect.scylladb; 25 | 26 | import java.net.InetSocketAddress; 27 | import java.util.Collection; 28 | import java.util.Collections; 29 | import java.util.HashMap; 30 | import java.util.Iterator; 31 | import java.util.Map; 32 | 33 | import com.datastax.oss.driver.api.core.addresstranslation.AddressTranslator; 34 | import com.fasterxml.jackson.core.JsonProcessingException; 35 | import com.fasterxml.jackson.databind.JsonNode; 36 | import com.fasterxml.jackson.databind.ObjectMapper; 37 | import org.slf4j.Logger; 38 | import org.slf4j.LoggerFactory; 39 | 40 | class ClusterAddressTranslator implements AddressTranslator { 41 | 42 | public Map addressMap = new HashMap<>(); 43 | private static final Logger log = LoggerFactory.getLogger(ClusterAddressTranslator.class); 44 | 45 | public void setMap(String addressMapString) throws JsonProcessingException { 46 | ObjectMapper objectMapper = new ObjectMapper(); 47 | JsonNode jsonNode = objectMapper.readTree(addressMapString); 48 | Iterator> entries = jsonNode.fields(); 49 | while(entries.hasNext()) { 50 | Map.Entry entry = entries.next(); 51 | addAddresses(entry.getKey(), entry.getValue().asText()); 52 | } 53 | } 54 | 55 | public void addAddresses(String internal, String external) { 56 | String[] internalhostport = internal.split(":"); 57 | String[] externalhostport = external.split(":"); 58 | InetSocketAddress internaladdress = new InetSocketAddress(internalhostport[0], Integer.parseInt(internalhostport[1])); 59 | InetSocketAddress externaladdress = new InetSocketAddress(externalhostport[0], Integer.parseInt(externalhostport[1])); 60 | addressMap.put(internaladdress, externaladdress); 61 | } 62 | 63 | public Collection getContactPoints() { 64 | return Collections.unmodifiableCollection(addressMap.values()); 65 | } 66 | 67 | @Override 68 | public InetSocketAddress translate(final InetSocketAddress inetSocketAddress) { 69 | return addressMap.getOrDefault(inetSocketAddress, inetSocketAddress); 70 | } 71 | 72 | @Override 73 | public void close() { 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/RecordConverter.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; 4 | import io.connect.scylladb.topictotable.TopicConfigs; 5 | import io.connect.scylladb.utils.ScyllaDbConstants; 6 | import org.apache.kafka.connect.data.Field; 7 | import org.apache.kafka.connect.data.Schema; 8 | import org.apache.kafka.connect.data.Struct; 9 | import org.apache.kafka.connect.errors.DataException; 10 | import org.apache.kafka.connect.header.Header; 11 | import org.apache.kafka.connect.sink.SinkRecord; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.math.BigDecimal; 16 | import java.math.BigInteger; 17 | import java.util.Date; 18 | import java.util.Iterator; 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | public abstract class RecordConverter { 23 | 24 | private static final Logger log = LoggerFactory.getLogger(RecordConverter.class); 25 | 26 | protected abstract T newValue(); 27 | 28 | protected abstract void setStringField(T result, String name, String value); 29 | 30 | protected abstract void setFloat32Field(T result, String name, Float value); 31 | 32 | protected abstract void setFloat64Field(T result, String name, Double value); 33 | 34 | protected abstract void setTimestampField(T result, String name, Date value); 35 | 36 | protected abstract void setDateField(T result, String name, Date value); 37 | 38 | protected abstract void setTimeField(T result, String name, Date value); 39 | 40 | protected abstract void setInt8Field(T result, String name, Byte value); 41 | 42 | protected abstract void setInt16Field(T result, String name, Short value); 43 | 44 | protected abstract void setInt32Field(T result, String name, Integer value); 45 | 46 | protected abstract void setInt64Field(T result, String name, Long value); 47 | 48 | protected abstract void setBytesField(T result, String name, byte[] value); 49 | 50 | protected abstract void setDecimalField(T result, String name, BigDecimal value); 51 | 52 | protected abstract void setBooleanField(T result, String name, Boolean value); 53 | 54 | protected abstract void setStructField(T result, String name, Struct value); 55 | 56 | protected abstract void setArray(T result, String name, Schema schema, List value); 57 | 58 | protected abstract void setMap(T result, String name, Schema schema, Map value); 59 | 60 | protected abstract void setNullField(T result, String name); 61 | 62 | public T convert(SinkRecord record, TopicConfigs topicConfigs, String operationType) { 63 | Object recordObject = ScyllaDbConstants.DELETE_OPERATION.equals(operationType) ? 64 | record.key() : record.value(); 65 | T result = this.newValue(); 66 | Map columnDetailsMap = null; 67 | Preconditions.checkNotNull(recordObject, 68 | (ScyllaDbConstants.DELETE_OPERATION.equals(operationType) ? "key " : "value ") + "cannot be null."); 69 | if (topicConfigs != null && topicConfigs.isScyllaColumnsMapped()) { 70 | columnDetailsMap = topicConfigs.getTableColumnMap(); 71 | Preconditions.checkNotNull(record.key(), "key cannot be null."); 72 | findRecordTypeAndConvert(result, record.key(), topicConfigs.getTablePartitionKeyMap()); 73 | for (Header header : record.headers()) { 74 | if (topicConfigs.getTableColumnMap().containsKey(header.key())) { 75 | TopicConfigs.KafkaScyllaColumnMapper headerKafkaScyllaColumnMapper = topicConfigs.getTableColumnMap().get(header.key()); 76 | parseStructAndSetInStatement(result, header.schema(), 77 | headerKafkaScyllaColumnMapper.getKafkaRecordField(), header.value(), 78 | headerKafkaScyllaColumnMapper.getScyllaColumnName()); 79 | } 80 | } 81 | } 82 | findRecordTypeAndConvert(result, recordObject, columnDetailsMap); 83 | return result; 84 | } 85 | 86 | void findRecordTypeAndConvert(T result, Object recordObject, 87 | Map columnDetailsMap) { 88 | if (recordObject instanceof Struct) { 89 | this.convertStruct(result, (Struct)recordObject, columnDetailsMap); 90 | } else { 91 | if (!(recordObject instanceof Map)) { 92 | throw new DataException(String.format("Only Schema (%s) or Schema less (%s) are supported. %s is not a supported type.", Struct.class.getName(), Map.class.getName(), recordObject.getClass().getName())); 93 | } 94 | 95 | this.convertMap(result, (Map)recordObject, columnDetailsMap); 96 | } 97 | } 98 | 99 | void convertMap(T result, Map value, Map columnDetailsMap) { 100 | Iterator valueIterator = value.keySet().iterator(); 101 | 102 | while(valueIterator.hasNext()) { 103 | Object key = valueIterator.next(); 104 | Preconditions.checkState(key instanceof String, "Map key must be a String."); 105 | String fieldName = (String)key; 106 | Object fieldValue = value.get(key); 107 | if (columnDetailsMap != null) { 108 | if (columnDetailsMap.containsKey(fieldName)) { 109 | fieldName = columnDetailsMap.get(fieldName).getScyllaColumnName(); 110 | } else { 111 | continue; 112 | } 113 | } 114 | 115 | try { 116 | if (null == fieldValue) { 117 | log.trace("convertStruct() - Setting '{}' to null.", fieldName); 118 | this.setNullField(result, fieldName); 119 | } else if (fieldValue instanceof String) { 120 | log.trace("convertStruct() - Processing '{}' as string.", fieldName); 121 | this.setStringField(result, fieldName, (String)fieldValue); 122 | } else if (fieldValue instanceof Byte) { 123 | log.trace("convertStruct() - Processing '{}' as int8.", fieldName); 124 | this.setInt8Field(result, fieldName, (Byte)fieldValue); 125 | } else if (fieldValue instanceof Short) { 126 | log.trace("convertStruct() - Processing '{}' as int16.", fieldName); 127 | this.setInt16Field(result, fieldName, (Short)fieldValue); 128 | } else if (fieldValue instanceof Integer) { 129 | log.trace("convertStruct() - Processing '{}' as int32.", fieldName); 130 | this.setInt32Field(result, fieldName, (Integer)fieldValue); 131 | } else if (fieldValue instanceof Long) { 132 | log.trace("convertStruct() - Processing '{}' as long.", fieldName); 133 | this.setInt64Field(result, fieldName, (Long)fieldValue); 134 | } else if (fieldValue instanceof BigInteger) { 135 | log.trace("convertStruct() - Processing '{}' as long.", fieldName); 136 | this.setInt64Field(result, fieldName, ((BigInteger)fieldValue).longValue()); 137 | } else if (fieldValue instanceof Double) { 138 | log.trace("convertStruct() - Processing '{}' as float64.", fieldName); 139 | this.setFloat64Field(result, fieldName, (Double)fieldValue); 140 | } else if (fieldValue instanceof Float) { 141 | log.trace("convertStruct() - Processing '{}' as float32.", fieldName); 142 | this.setFloat32Field(result, fieldName, (Float)fieldValue); 143 | } else if (fieldValue instanceof BigDecimal) { 144 | log.trace("convertStruct() - Processing '{}' as decimal.", fieldName); 145 | this.setDecimalField(result, fieldName, (BigDecimal)fieldValue); 146 | } else if (fieldValue instanceof Boolean) { 147 | log.trace("convertStruct() - Processing '{}' as boolean.", fieldName); 148 | this.setBooleanField(result, fieldName, (Boolean)fieldValue); 149 | } else if (fieldValue instanceof Date) { 150 | log.trace("convertStruct() - Processing '{}' as timestamp.", fieldName); 151 | this.setTimestampField(result, fieldName, (Date)fieldValue); 152 | } else if (fieldValue instanceof byte[]) { 153 | log.trace("convertStruct() - Processing '{}' as bytes.", fieldName); 154 | this.setBytesField(result, fieldName, (byte[])((byte[])fieldValue)); 155 | } else if (fieldValue instanceof List) { 156 | log.trace("convertStruct() - Processing '{}' as array.", fieldName); 157 | this.setArray(result, fieldName, (Schema)null, (List)fieldValue); 158 | } else { 159 | if (!(fieldValue instanceof Map)) { 160 | throw new DataException(String.format("%s is not a supported data type.", fieldValue.getClass().getName())); 161 | } 162 | 163 | log.trace("convertStruct() - Processing '{}' as map.", fieldName); 164 | this.setMap(result, fieldName, (Schema)null, (Map)fieldValue); 165 | } 166 | } catch (Exception ex) { 167 | throw new DataException(String.format("Exception thrown while processing field '%s'", fieldName), ex); 168 | } 169 | } 170 | 171 | } 172 | 173 | void convertStruct(T result, Struct struct, Map columnDetailsMap) { 174 | Schema schema = struct.schema(); 175 | Iterator fieldsIterator = schema.fields().iterator(); 176 | 177 | while(fieldsIterator.hasNext()) { 178 | Field field = (Field)fieldsIterator.next(); 179 | String fieldName = field.name(); 180 | log.trace("convertStruct() - Processing '{}'", field.name()); 181 | Object fieldValue = struct.get(field); 182 | if (columnDetailsMap != null) { 183 | if (columnDetailsMap.containsKey(fieldName)) { 184 | fieldName = columnDetailsMap.get(fieldName).getScyllaColumnName(); 185 | } else { 186 | continue; 187 | } 188 | } 189 | parseStructAndSetInStatement(result, schema, field, fieldValue, fieldName); 190 | } 191 | } 192 | 193 | void parseStructAndSetInStatement(T result, Schema schema, Field field, Object fieldValue, String fieldName) { 194 | try { 195 | if (null == fieldValue) { 196 | log.trace("convertStruct() - Setting '{}' to null.", fieldName); 197 | this.setNullField(result, fieldName); 198 | } else { 199 | log.trace("convertStruct() - Field '{}'.field().schema().type() = '{}'", fieldName, field.schema().type()); 200 | switch(field.schema().type()) { 201 | case STRING: 202 | log.trace("convertStruct() - Processing '{}' as string.", fieldName); 203 | this.setStringField(result, fieldName, (String)fieldValue); 204 | break; 205 | case INT8: 206 | log.trace("convertStruct() - Processing '{}' as int8.", fieldName); 207 | this.setInt8Field(result, fieldName, (Byte)fieldValue); 208 | break; 209 | case INT16: 210 | log.trace("convertStruct() - Processing '{}' as int16.", fieldName); 211 | this.setInt16Field(result, fieldName, (Short)fieldValue); 212 | break; 213 | case INT32: 214 | if ("org.apache.kafka.connect.data.Date".equals(field.schema().name())) { 215 | log.trace("convertStruct() - Processing '{}' as date.", fieldName); 216 | this.setDateField(result, fieldName, (Date)fieldValue); 217 | } else if ("org.apache.kafka.connect.data.Time".equals(field.schema().name())) { 218 | log.trace("convertStruct() - Processing '{}' as time.", fieldName); 219 | this.setTimeField(result, fieldName, (Date)fieldValue); 220 | } else { 221 | Integer int32Value = (Integer)fieldValue; 222 | log.trace("convertStruct() - Processing '{}' as int32.", fieldName); 223 | this.setInt32Field(result, fieldName, int32Value); 224 | } 225 | break; 226 | case INT64: 227 | if ("org.apache.kafka.connect.data.Timestamp".equals(field.schema().name())) { 228 | log.trace("convertStruct() - Processing '{}' as timestamp.", fieldName); 229 | this.setTimestampField(result, fieldName, (Date)fieldValue); 230 | } else { 231 | Long int64Value = (Long)fieldValue; 232 | log.trace("convertStruct() - Processing '{}' as int64.", fieldName); 233 | this.setInt64Field(result, fieldName, int64Value); 234 | } 235 | break; 236 | case BYTES: 237 | if ("org.apache.kafka.connect.data.Decimal".equals(field.schema().name())) { 238 | log.trace("convertStruct() - Processing '{}' as decimal.", fieldName); 239 | this.setDecimalField(result, fieldName, (BigDecimal)fieldValue); 240 | } else { 241 | byte[] bytes = (byte[])((byte[])fieldValue); 242 | log.trace("convertStruct() - Processing '{}' as bytes.", fieldName); 243 | this.setBytesField(result, fieldName, bytes); 244 | } 245 | break; 246 | case FLOAT32: 247 | log.trace("convertStruct() - Processing '{}' as float32.", fieldName); 248 | this.setFloat32Field(result, fieldName, (Float)fieldValue); 249 | break; 250 | case FLOAT64: 251 | log.trace("convertStruct() - Processing '{}' as float64.", fieldName); 252 | this.setFloat64Field(result, fieldName, (Double)fieldValue); 253 | break; 254 | case BOOLEAN: 255 | log.trace("convertStruct() - Processing '{}' as boolean.", fieldName); 256 | this.setBooleanField(result, fieldName, (Boolean)fieldValue); 257 | break; 258 | case STRUCT: 259 | log.trace("convertStruct() - Processing '{}' as struct.", fieldName); 260 | this.setStructField(result, fieldName, (Struct)fieldValue); 261 | break; 262 | case ARRAY: 263 | log.trace("convertStruct() - Processing '{}' as array.", fieldName); 264 | this.setArray(result, fieldName, schema, (List)fieldValue); 265 | break; 266 | case MAP: 267 | log.trace("convertStruct() - Processing '{}' as map.", fieldName); 268 | this.setMap(result, fieldName, schema, (Map)fieldValue); 269 | break; 270 | default: 271 | throw new DataException("Unsupported schema.type(): " + schema.type()); 272 | } 273 | } 274 | } catch (Exception ex) { 275 | throw new DataException(String.format("Exception thrown while processing field '%s'", fieldName), ex); 276 | } 277 | } 278 | } 279 | 280 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/RecordToBoundStatementConverter.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.datastax.oss.driver.api.core.cql.BoundStatement; 4 | import com.datastax.oss.driver.api.core.cql.PreparedStatement; 5 | import org.apache.kafka.connect.data.Schema; 6 | import org.apache.kafka.connect.data.Struct; 7 | 8 | import java.math.BigDecimal; 9 | import java.nio.ByteBuffer; 10 | import java.time.LocalDate; 11 | import java.time.LocalTime; 12 | import java.time.ZoneId; 13 | import java.util.Date; 14 | import java.util.List; 15 | import java.util.Map; 16 | import java.util.concurrent.TimeUnit; 17 | 18 | class RecordToBoundStatementConverter extends RecordConverter { 19 | private final PreparedStatement preparedStatement; 20 | 21 | static class State { 22 | 23 | public BoundStatement statement; 24 | public int parameters = 0; 25 | 26 | State(BoundStatement statement) { 27 | this.statement = statement; 28 | } 29 | } 30 | 31 | RecordToBoundStatementConverter(PreparedStatement preparedStatement) { 32 | this.preparedStatement = preparedStatement; 33 | } 34 | 35 | protected RecordToBoundStatementConverter.State newValue() { 36 | BoundStatement boundStatement = this.preparedStatement.bind(); 37 | return new State(boundStatement); 38 | } 39 | 40 | protected void setStringField( 41 | RecordToBoundStatementConverter.State state, 42 | String fieldName, 43 | String value 44 | ) { 45 | state.statement = state.statement.setString(fieldName, value); 46 | state.parameters++; 47 | } 48 | 49 | protected void setFloat32Field( 50 | RecordToBoundStatementConverter.State state, 51 | String fieldName, 52 | Float value 53 | ) { 54 | state.statement = state.statement.setFloat(fieldName, value); 55 | state.parameters++; 56 | } 57 | 58 | protected void setFloat64Field( 59 | RecordToBoundStatementConverter.State state, 60 | String fieldName, 61 | Double value 62 | ) { 63 | state.statement = state.statement.setDouble(fieldName, value); 64 | state.parameters++; 65 | } 66 | 67 | protected void setTimestampField( 68 | RecordToBoundStatementConverter.State state, 69 | String fieldName, 70 | Date value 71 | ) { 72 | state.statement = state.statement.setInstant(fieldName, value.toInstant()); 73 | state.parameters++; 74 | } 75 | 76 | protected void setDateField( 77 | RecordToBoundStatementConverter.State state, 78 | String fieldName, 79 | Date value 80 | ) { 81 | state.statement = state.statement.setLocalDate(fieldName, LocalDate.from(value.toInstant().atZone(ZoneId.systemDefault()).toLocalDate())); 82 | state.parameters++; 83 | } 84 | 85 | protected void setTimeField( 86 | RecordToBoundStatementConverter.State state, 87 | String fieldName, 88 | Date value 89 | ) { 90 | final long nanoseconds = TimeUnit.NANOSECONDS.convert(value.getTime(), TimeUnit.MILLISECONDS); 91 | state.statement = state.statement.setLocalTime(fieldName, LocalTime.ofNanoOfDay(nanoseconds)); 92 | 93 | state.parameters++; 94 | } 95 | 96 | protected void setInt8Field( 97 | RecordToBoundStatementConverter.State state, 98 | String fieldName, 99 | Byte value 100 | ) { 101 | state.statement = state.statement.setByte(fieldName, value); 102 | state.parameters++; 103 | } 104 | 105 | protected void setInt16Field( 106 | RecordToBoundStatementConverter.State state, 107 | String fieldName, 108 | Short value 109 | ) { 110 | state.statement = state.statement.setShort(fieldName, value); 111 | state.parameters++; 112 | } 113 | 114 | protected void setInt32Field( 115 | RecordToBoundStatementConverter.State state, 116 | String fieldName, 117 | Integer value 118 | ) { 119 | state.statement = state.statement.setInt(fieldName, value); 120 | state.parameters++; 121 | } 122 | 123 | protected void setInt64Field( 124 | RecordToBoundStatementConverter.State state, 125 | String fieldName, 126 | Long value 127 | ) { 128 | state.statement = state.statement.setLong(fieldName, value); 129 | state.parameters++; 130 | } 131 | 132 | protected void setBytesField( 133 | RecordToBoundStatementConverter.State state, 134 | String fieldName, 135 | byte[] value 136 | ) { 137 | state.statement = state.statement.setByteBuffer(fieldName, ByteBuffer.wrap(value)); 138 | state.parameters++; 139 | } 140 | 141 | protected void setDecimalField( 142 | RecordToBoundStatementConverter.State state, 143 | String fieldName, 144 | BigDecimal value 145 | ) { 146 | state.statement = state.statement.setBigDecimal(fieldName, value); 147 | state.parameters++; 148 | } 149 | 150 | protected void setBooleanField( 151 | RecordToBoundStatementConverter.State state, 152 | String fieldName, 153 | Boolean value 154 | ) { 155 | state.statement = state.statement.setBool(fieldName, value); 156 | state.parameters++; 157 | } 158 | 159 | protected void setStructField( 160 | RecordToBoundStatementConverter.State state, 161 | String fieldName, 162 | Struct value 163 | ) { 164 | throw new UnsupportedOperationException(); 165 | } 166 | 167 | protected void setArray( 168 | RecordToBoundStatementConverter.State state, 169 | String fieldName, 170 | Schema schema, 171 | List value 172 | ) { 173 | state.statement = state.statement.setList(fieldName, value, Object.class); 174 | state.parameters++; 175 | } 176 | 177 | protected void setMap( 178 | RecordToBoundStatementConverter.State state, 179 | String fieldName, 180 | Schema schema, 181 | Map value 182 | ) { 183 | state.statement = state.statement.setMap(fieldName, value, Object.class, Object.class); 184 | state.parameters++; 185 | } 186 | 187 | protected void setNullField( 188 | RecordToBoundStatementConverter.State state, 189 | String fieldName 190 | ) { 191 | state.statement = state.statement.setToNull(fieldName); 192 | state.parameters++; 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/ScyllaDbSession.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.datastax.oss.driver.api.core.cql.AsyncResultSet; 4 | import com.datastax.oss.driver.api.core.cql.BoundStatement; 5 | import com.datastax.oss.driver.api.core.cql.ResultSet; 6 | import com.datastax.oss.driver.api.core.cql.Statement; 7 | import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; 8 | import io.connect.scylladb.topictotable.TopicConfigs; 9 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 10 | import org.apache.kafka.common.TopicPartition; 11 | import org.apache.kafka.connect.sink.SinkRecord; 12 | 13 | import java.io.Closeable; 14 | import java.util.Map; 15 | import java.util.Set; 16 | import java.util.concurrent.CompletionStage; 17 | 18 | public interface ScyllaDbSession extends Closeable { 19 | 20 | /** 21 | * Execute a statement 22 | */ 23 | ResultSet executeStatement(Statement statement); 24 | 25 | /** 26 | * Execute a statement asynchronously 27 | */ 28 | CompletionStage executeStatementAsync(Statement statement); 29 | 30 | /** 31 | * Execute a query 32 | */ 33 | ResultSet executeQuery(String query); 34 | 35 | /** 36 | * Lookup metadata for a keyspace. 37 | * @param keyspaceName name of the keyspace 38 | */ 39 | KeyspaceMetadata keyspaceMetadata(String keyspaceName); 40 | 41 | /** 42 | * Check if a keyspace exists. 43 | * @param keyspaceName name of the keyspace 44 | */ 45 | boolean keyspaceExists(String keyspaceName); 46 | 47 | /** 48 | * Lookup metadata for a table. 49 | * @param tableName name of the table 50 | */ 51 | TableMetadata.Table tableMetadata(String tableName); 52 | 53 | /** 54 | * Check if a table exists. 55 | * @param tableName name of the table 56 | */ 57 | boolean tableExists(String tableName); 58 | 59 | /** 60 | * Ensure that a table has a specified schema. 61 | * @param tableName name of the table 62 | * @param sinkRecord which will have keySchema that will be used for the primary key and 63 | * valueSchema that will be used for the rest of the table. 64 | * @param topicConfigs class containing mapping details for the record 65 | */ 66 | void createOrAlterTable(String tableName, SinkRecord sinkRecord, TopicConfigs topicConfigs); 67 | 68 | /** 69 | * Flag to determine if the session is valid. 70 | */ 71 | boolean isValid(); 72 | 73 | /** 74 | * Method is used to mark a session as invalid. 75 | */ 76 | void setInvalid(); 77 | 78 | /** 79 | * Method will return a RecordToBoundStatementConverter for a delete the supplied table. 80 | * @param tableName table to return the RecordToBoundStatementConverter for 81 | * @return RecordToBoundStatementConverter that can be used for the record. 82 | */ 83 | RecordToBoundStatementConverter delete(String tableName); 84 | 85 | /** 86 | * Method will return a RecordToBoundStatementConverter for an insert the supplied table. 87 | * @param tableName table to return the RecordToBoundStatementConverter for 88 | * @param topicConfigs class containing mapping details for the record 89 | * @return RecordToBoundStatementConverter that can be used for the record. 90 | */ 91 | RecordToBoundStatementConverter insert(String tableName, TopicConfigs topicConfigs); 92 | 93 | /** 94 | * Method generates a BoundStatement, that inserts the offset metadata 95 | * for a given topic and partition. 96 | * @param topicPartition topic and partition for the offset 97 | * @param metadata offset metadata to be inserted 98 | * @return statement that inserts the provided offset to Scylla. 99 | */ 100 | BoundStatement getInsertOffsetStatement(TopicPartition topicPartition, OffsetAndMetadata metadata); 101 | 102 | /** 103 | * Method is used to load offsets from storage in ScyllaDb 104 | * @param assignment The assignment of TopicPartitions that have been assigned to this task. 105 | * @return The offsets by TopicPartition based on the assignment. 106 | */ 107 | Map loadOffsets(Set assignment); 108 | 109 | /** 110 | * Callback that is fired when a table has changed. 111 | * @param keyspace Keyspace for the table change. 112 | * @param tableName Table name for the change. 113 | */ 114 | void onTableChanged(String keyspace, String tableName); 115 | 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/ScyllaDbSessionFactory.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | 5 | import com.datastax.oss.driver.api.core.CqlSession; 6 | import com.datastax.oss.driver.api.core.CqlSessionBuilder; 7 | import com.datastax.oss.driver.api.core.config.DefaultDriverOption; 8 | import com.datastax.oss.driver.api.core.config.DriverConfigLoader; 9 | import com.datastax.oss.driver.api.core.config.ProgrammaticDriverConfigLoaderBuilder; 10 | import com.datastax.oss.driver.api.core.type.codec.registry.MutableCodecRegistry; 11 | import com.datastax.oss.driver.internal.core.loadbalancing.DcInferringLoadBalancingPolicy; 12 | import com.datastax.oss.driver.internal.core.ssl.DefaultSslEngineFactory; 13 | import com.datastax.oss.driver.internal.core.type.codec.registry.DefaultCodecRegistry; 14 | import io.connect.scylladb.codec.ConvenienceCodecs; 15 | import io.connect.scylladb.codec.StringDurationCodec; 16 | import io.connect.scylladb.codec.StringInetCodec; 17 | import io.connect.scylladb.codec.StringTimeUuidCodec; 18 | import io.connect.scylladb.codec.StringUuidCodec; 19 | import io.connect.scylladb.codec.StringVarintCodec; 20 | import org.apache.kafka.connect.errors.ConnectException; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.io.File; 25 | import java.io.FileInputStream; 26 | import java.io.IOException; 27 | import java.io.InputStream; 28 | import java.net.InetSocketAddress; 29 | import java.security.KeyStore; 30 | import java.security.KeyStoreException; 31 | import java.security.NoSuchAlgorithmException; 32 | import java.security.cert.CertificateException; 33 | import java.util.Arrays; 34 | 35 | public class ScyllaDbSessionFactory { 36 | 37 | private static final Logger log = LoggerFactory.getLogger(ScyllaDbSessionFactory.class); 38 | private static final MutableCodecRegistry CODEC_REGISTRY = new DefaultCodecRegistry("ScyllaDbSessionFactory.CodecRegistry"); 39 | 40 | static { 41 | // Register custom codec once at class loading time; duplicates will be logged via warning 42 | CODEC_REGISTRY.register(StringUuidCodec.INSTANCE); 43 | CODEC_REGISTRY.register(StringTimeUuidCodec.INSTANCE); 44 | CODEC_REGISTRY.register(StringInetCodec.INSTANCE); 45 | CODEC_REGISTRY.register(StringVarintCodec.INSTANCE); 46 | CODEC_REGISTRY.register(StringDurationCodec.INSTANCE); 47 | CODEC_REGISTRY.register(ConvenienceCodecs.ALL_INSTANCES); 48 | } 49 | 50 | public ScyllaDbSession newSession(ScyllaDbSinkConnectorConfig config) { 51 | 52 | ProgrammaticDriverConfigLoaderBuilder driverConfigLoaderBuilder = DriverConfigLoader.programmaticBuilder(); 53 | CqlSessionBuilder sessionBuilder = CqlSession.builder().withCodecRegistry(CODEC_REGISTRY); 54 | try { 55 | configureAddressTranslator(config, sessionBuilder, driverConfigLoaderBuilder); 56 | } catch (JsonProcessingException e) { 57 | log.info("Failed to configure address translator, provide a valid JSON string " + 58 | "with external network address and port mapped to private network " + 59 | "address and port."); 60 | configurePublicContactPoints(config, sessionBuilder); 61 | } 62 | 63 | driverConfigLoaderBuilder.withClass(DefaultDriverOption.LOAD_BALANCING_POLICY_CLASS, DcInferringLoadBalancingPolicy.class); 64 | if (!config.loadBalancingLocalDc.isEmpty()) { 65 | sessionBuilder.withLocalDatacenter(config.loadBalancingLocalDc); 66 | } else { 67 | log.warn("`scylladb.loadbalancing.localdc` has not been configured, " 68 | + "which is recommended configuration in case of more than one DC."); 69 | } 70 | if (config.securityEnabled) { 71 | sessionBuilder.withAuthCredentials(config.username, config.password); 72 | } 73 | 74 | if (config.sslEnabled) { 75 | driverConfigLoaderBuilder 76 | .withClass(DefaultDriverOption.SSL_ENGINE_FACTORY_CLASS, DefaultSslEngineFactory.class) 77 | .withBoolean(DefaultDriverOption.SSL_HOSTNAME_VALIDATION, config.sslHostnameVerificationEnabled); 78 | 79 | if (null != config.trustStorePath) { 80 | log.info("Configuring Driver ({}) to use Truststore {}", DefaultDriverOption.SSL_TRUSTSTORE_PATH.getPath(), config.trustStorePath); 81 | driverConfigLoaderBuilder 82 | .withString(DefaultDriverOption.SSL_TRUSTSTORE_PATH, config.trustStorePath.getAbsolutePath()) 83 | .withString(DefaultDriverOption.SSL_TRUSTSTORE_PASSWORD, String.valueOf(config.trustStorePassword)); 84 | } 85 | 86 | if (null != config.keyStorePath) { 87 | log.info("Configuring Driver ({}) to use Keystore {}", DefaultDriverOption.SSL_KEYSTORE_PATH.getPath(), config.keyStorePath); 88 | driverConfigLoaderBuilder 89 | .withString(DefaultDriverOption.SSL_KEYSTORE_PATH, config.keyStorePath.getAbsolutePath()) 90 | .withString(DefaultDriverOption.SSL_KEYSTORE_PASSWORD, String.valueOf(config.keyStorePassword)); 91 | } 92 | 93 | if (config.cipherSuites.size() > 0) { 94 | driverConfigLoaderBuilder 95 | .withStringList(DefaultDriverOption.SSL_CIPHER_SUITES, config.cipherSuites); 96 | } 97 | } 98 | 99 | driverConfigLoaderBuilder.withString(DefaultDriverOption.PROTOCOL_COMPRESSION, config.compression); 100 | 101 | log.info("Creating session"); 102 | sessionBuilder.withConfigLoader(driverConfigLoaderBuilder.build()); 103 | final CqlSession session = sessionBuilder.build(); 104 | return new ScyllaDbSessionImpl(config, session); 105 | } 106 | 107 | private void configurePublicContactPoints(ScyllaDbSinkConnectorConfig config, CqlSessionBuilder sessionBuilder) { 108 | log.info("Configuring public contact points={}", config.contactPoints); 109 | String[] contactPointsArray = config.contactPoints.split(","); 110 | for (String contactPoint : contactPointsArray) { 111 | if (contactPoint == null) { 112 | throw new NullPointerException("One of provided contact points is null"); 113 | } 114 | sessionBuilder.addContactPoint(new InetSocketAddress(contactPoint, config.port)); 115 | } 116 | } 117 | 118 | private void configureAddressTranslator(ScyllaDbSinkConnectorConfig config, CqlSessionBuilder sessionBuilder, ProgrammaticDriverConfigLoaderBuilder configBuilder) throws JsonProcessingException { 119 | log.info("Trying to configure address translator for private network address and port."); 120 | ClusterAddressTranslator translator = new ClusterAddressTranslator(); 121 | translator.setMap(config.contactPoints); 122 | sessionBuilder.addContactPoints(translator.getContactPoints()); 123 | configBuilder.withClass(DefaultDriverOption.ADDRESS_TRANSLATOR_CLASS, ClusterAddressTranslator.class); 124 | } 125 | 126 | private KeyStore createKeyStore(File path, char[] password) { 127 | KeyStore keyStore; 128 | try { 129 | keyStore = KeyStore.getInstance("JKS"); 130 | try (InputStream inputStream = new FileInputStream(path)) { 131 | keyStore.load(inputStream, password); 132 | } catch (IOException e) { 133 | throw new ConnectException("Exception while reading keystore", e); 134 | } catch (CertificateException | NoSuchAlgorithmException e) { 135 | throw new ConnectException("Exception while loading keystore", e); 136 | } 137 | } catch (KeyStoreException e) { 138 | throw new ConnectException("Exception while creating keystore", e); 139 | } 140 | return keyStore; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/ScyllaDbSessionImpl.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.datastax.oss.driver.api.core.CqlIdentifier; 4 | import com.datastax.oss.driver.api.core.CqlSession; 5 | import com.datastax.oss.driver.api.core.cql.AsyncResultSet; 6 | import com.datastax.oss.driver.api.core.cql.BoundStatement; 7 | import com.datastax.oss.driver.api.core.cql.PreparedStatement; 8 | import com.datastax.oss.driver.api.core.cql.ResultSet; 9 | import com.datastax.oss.driver.api.core.cql.Row; 10 | import com.datastax.oss.driver.api.core.cql.Statement; 11 | import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; 12 | import com.datastax.oss.driver.api.querybuilder.QueryBuilder; 13 | import com.datastax.oss.driver.api.querybuilder.delete.Delete; 14 | import com.datastax.oss.driver.api.querybuilder.delete.DeleteSelection; 15 | import com.datastax.oss.driver.api.querybuilder.insert.InsertInto; 16 | import com.datastax.oss.driver.api.querybuilder.insert.RegularInsert; 17 | import com.datastax.oss.driver.api.querybuilder.select.Select; 18 | import io.connect.scylladb.topictotable.TopicConfigs; 19 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 20 | import org.apache.kafka.common.TopicPartition; 21 | import org.apache.kafka.connect.sink.SinkRecord; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.io.IOException; 26 | import java.util.HashMap; 27 | import java.util.Map; 28 | import java.util.Optional; 29 | import java.util.Set; 30 | import java.util.concurrent.CompletionStage; 31 | import java.util.function.Function; 32 | 33 | class ScyllaDbSessionImpl implements ScyllaDbSession { 34 | private static final Logger log = LoggerFactory.getLogger(ScyllaDbSessionImpl.class); 35 | 36 | private final ScyllaDbSinkConnectorConfig config; 37 | private final CqlSession session; 38 | private final ScyllaDbSchemaBuilder schemaBuilder; 39 | private boolean sessionValid = true; 40 | private final Map tableMetadataCache; 41 | private final Map deleteStatementCache; 42 | private final Map insertStatementCache; 43 | 44 | ScyllaDbSessionImpl(ScyllaDbSinkConnectorConfig config, CqlSession session) { 45 | this.session = session; 46 | this.config = config; 47 | this.schemaBuilder = new ScyllaDbSchemaBuilder(this, config); 48 | this.tableMetadataCache = new HashMap<>(); 49 | this.deleteStatementCache = new HashMap<>(); 50 | this.insertStatementCache = new HashMap<>(); 51 | } 52 | 53 | @Override 54 | public ResultSet executeStatement(Statement statement) { 55 | log.trace("executeStatement() - Executing statement\n{}", statement); 56 | return this.session.execute(statement); 57 | } 58 | 59 | @Override 60 | public CompletionStage executeStatementAsync(Statement statement) { 61 | log.trace("executeStatement() - Executing statement\n{}", statement); 62 | return this.session.executeAsync(statement); 63 | } 64 | 65 | @Override 66 | public ResultSet executeQuery(String query) { 67 | log.trace("executeQuery() - Executing query\n{}", query); 68 | return this.session.execute(query); 69 | } 70 | 71 | @Override 72 | public KeyspaceMetadata keyspaceMetadata(String keyspaceName){ 73 | log.trace("keyspaceMetadata() - keyspaceName = '{}'", keyspaceName); 74 | return session.getMetadata().getKeyspace(keyspaceName).get(); 75 | } 76 | 77 | @Override 78 | public boolean keyspaceExists(String keyspaceName){ 79 | return keyspaceMetadata(keyspaceName) != null; 80 | } 81 | 82 | @Override 83 | public TableMetadata.Table tableMetadata(String tableName) { 84 | log.trace("tableMetadata() - tableName = '{}'", tableName); 85 | TableMetadata.Table result = this.tableMetadataCache.get(tableName); 86 | 87 | if (null == result) { 88 | final KeyspaceMetadata keyspaceMetadata = session.getMetadata().getKeyspace(config.keyspace).get(); 89 | final Optional tableMetadata = keyspaceMetadata.getTable(tableName); 90 | if (tableMetadata.isPresent()) { 91 | result = new TableMetadataImpl.TableImpl(tableMetadata.get()); 92 | this.tableMetadataCache.put(tableName, result); 93 | } else { 94 | log.warn("Could not find metadata for table {} in keyspace {}. Are you sure the table exists?", tableName, keyspaceMetadata.getName().asCql(false)); 95 | result = null; 96 | } 97 | } 98 | 99 | return result; 100 | } 101 | 102 | @Override 103 | public boolean tableExists(String tableName) { 104 | return null != tableMetadata(tableName); 105 | } 106 | 107 | @Override 108 | public void createOrAlterTable(String tableName, SinkRecord record, TopicConfigs topicConfigs) { 109 | this.schemaBuilder.build(tableName, record, topicConfigs); 110 | } 111 | 112 | @Override 113 | public boolean isValid() { 114 | return this.sessionValid; 115 | } 116 | 117 | @Override 118 | public void setInvalid() { 119 | this.sessionValid = false; 120 | } 121 | 122 | 123 | @Override 124 | public RecordToBoundStatementConverter delete(String tableName) { 125 | return this.deleteStatementCache.computeIfAbsent( 126 | tableName, 127 | new Function() { 128 | @Override 129 | public RecordToBoundStatementConverter apply(String tableName) { 130 | DeleteSelection statementStart = QueryBuilder.deleteFrom(config.keyspace, tableName); 131 | Delete statement = null; 132 | TableMetadata.Table tableMetadata = tableMetadata(tableName); 133 | for (TableMetadata.Column columnMetadata : tableMetadata.primaryKey()) { 134 | if (statement == null) { 135 | statement = statementStart.whereColumn(columnMetadata.getName()).isEqualTo(QueryBuilder.bindMarker(columnMetadata.getName())); 136 | } 137 | else { 138 | statement = statement.whereColumn(columnMetadata.getName()).isEqualTo(QueryBuilder.bindMarker(columnMetadata.getName())); 139 | } 140 | } 141 | assert statement != null; 142 | log.debug("delete() - Preparing statement. '{}'", statement.asCql()); 143 | PreparedStatement preparedStatement = session.prepare(statement.build()); 144 | return new RecordToBoundStatementConverter(preparedStatement); 145 | } 146 | } 147 | ); 148 | } 149 | 150 | private PreparedStatement createInsertPreparedStatement(String tableName, TopicConfigs topicConfigs) { 151 | InsertInto insertInto = QueryBuilder.insertInto(config.keyspace, tableName); 152 | RegularInsert regularInsert = null; 153 | TableMetadata.Table tableMetadata = tableMetadata(tableName); 154 | for (TableMetadata.Column columnMetadata : tableMetadata.columns()) { 155 | if (regularInsert == null) { 156 | regularInsert = insertInto.value(CqlIdentifier.fromInternal(columnMetadata.getName()), QueryBuilder.bindMarker(columnMetadata.getName())); 157 | } else { 158 | regularInsert = regularInsert.value(CqlIdentifier.fromInternal(columnMetadata.getName()), QueryBuilder.bindMarker(columnMetadata.getName())); 159 | } 160 | } 161 | assert regularInsert != null; 162 | log.debug("insert() - Preparing statement. '{}'", regularInsert.asCql()); 163 | if (topicConfigs != null) { 164 | return (topicConfigs.getTtl() == null) ? session.prepare(regularInsert.build()) : 165 | session.prepare(regularInsert.usingTtl(topicConfigs.getTtl()).build()); 166 | } else { 167 | return (config.ttl == null) ? session.prepare(regularInsert.build()) : 168 | session.prepare(regularInsert.usingTtl(config.ttl).build()); 169 | } 170 | } 171 | 172 | @Override 173 | public RecordToBoundStatementConverter insert(String tableName, TopicConfigs topicConfigs) { 174 | if (topicConfigs != null && topicConfigs.getTtl() != null) { 175 | PreparedStatement preparedStatement = createInsertPreparedStatement(tableName, topicConfigs); 176 | return new RecordToBoundStatementConverter(preparedStatement); 177 | } else { 178 | return this.insertStatementCache.computeIfAbsent( 179 | tableName, 180 | new Function() { 181 | @Override 182 | public RecordToBoundStatementConverter apply(String tableName) { 183 | PreparedStatement preparedStatement = createInsertPreparedStatement(tableName, topicConfigs); 184 | return new RecordToBoundStatementConverter(preparedStatement); 185 | } 186 | } 187 | ); 188 | } 189 | } 190 | 191 | private PreparedStatement offsetPreparedStatement; 192 | 193 | @Override 194 | public BoundStatement getInsertOffsetStatement( 195 | TopicPartition topicPartition, 196 | OffsetAndMetadata metadata) { 197 | if (null == this.offsetPreparedStatement) { 198 | this.offsetPreparedStatement = 199 | createInsertPreparedStatement(this.config.offsetStorageTable, null); 200 | } 201 | log.debug( 202 | "getAddOffsetsStatement() - Setting offset to {}:{}:{}", 203 | topicPartition.topic(), 204 | topicPartition.partition(), 205 | metadata.offset() 206 | ); 207 | BoundStatement statement = offsetPreparedStatement.bind(); 208 | statement = statement.setString("topic", topicPartition.topic()); 209 | statement = statement.setInt("partition", topicPartition.partition()); 210 | statement = statement.setLong("offset", metadata.offset()); 211 | return statement; 212 | } 213 | 214 | @Override 215 | public Map loadOffsets(Set assignment) { 216 | Map result = new HashMap<>(); 217 | if (null != assignment && !assignment.isEmpty()) { 218 | final Select partitionQuery = QueryBuilder.selectFrom(this.config.keyspace, this.config.offsetStorageTable) 219 | .column("offset") 220 | .whereColumn("topic").isEqualTo(QueryBuilder.bindMarker("topic")) 221 | .whereColumn("partition").isEqualTo(QueryBuilder.bindMarker("partition")); 222 | log.debug("loadOffsets() - Preparing statement. {}", partitionQuery.asCql()); 223 | final PreparedStatement preparedStatement = this.session.prepare(partitionQuery.build()); 224 | 225 | for (final TopicPartition topicPartition : assignment) { 226 | log.debug("loadOffsets() - Querying for {}", topicPartition); 227 | BoundStatement boundStatement = preparedStatement.bind(); 228 | boundStatement = boundStatement.setString("topic", topicPartition.topic()); 229 | boundStatement = boundStatement.setInt("partition", topicPartition.partition()); 230 | 231 | ResultSet resultSet = this.executeStatement(boundStatement); 232 | Row row = resultSet.one(); 233 | if (null != row) { 234 | long offset = row.getLong("offset"); 235 | log.info("Found offset of {} for {}", offset, topicPartition); 236 | result.put(topicPartition, offset); 237 | } 238 | } 239 | } 240 | 241 | return result; 242 | } 243 | 244 | @Override 245 | public void close() throws IOException { 246 | this.session.close(); 247 | } 248 | 249 | @Override 250 | public void onTableChanged(String keyspace, String tableName) { 251 | this.tableMetadataCache.remove(tableName); 252 | this.deleteStatementCache.remove(tableName); 253 | this.insertStatementCache.remove(tableName); 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/ScyllaDbSinkConnector.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.Objects; 9 | 10 | import com.datastax.oss.driver.api.core.type.DataTypes; 11 | import com.datastax.oss.driver.api.querybuilder.SchemaBuilder; 12 | import com.datastax.oss.driver.api.querybuilder.schema.CreateKeyspace; 13 | import com.datastax.oss.driver.api.querybuilder.schema.CreateTable; 14 | import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; 15 | import io.connect.scylladb.utils.VersionUtil; 16 | import org.apache.kafka.common.config.ConfigDef; 17 | import org.apache.kafka.common.config.Config; 18 | import org.apache.kafka.common.config.ConfigValue; 19 | import org.apache.kafka.connect.connector.Task; 20 | import org.apache.kafka.connect.errors.ConnectException; 21 | import org.apache.kafka.connect.sink.SinkConnector; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | /** 26 | * Sink connector class for Scylla Database. 27 | */ 28 | public class ScyllaDbSinkConnector extends SinkConnector { 29 | 30 | private static final Logger log = LoggerFactory.getLogger(ScyllaDbSinkConnector.class); 31 | 32 | ScyllaDbSinkConnectorConfig config; 33 | 34 | /** 35 | * Start the Connector. The method will establish a ScyllaDB session 36 | * and perform the following:
    37 | *
  1. Create a keyspace with the name under scylladb.keyspace configuration, 38 | * if scylladb.keyspace.create.enabled is set to true. 39 | *
  2. Create a table for managing offsets in ScyllaDB with the name under 40 | * scylladb.offset.storage.table configuration, 41 | * if scylladb.offset.storage.table.enable is set to true. 42 | *
43 | */ 44 | @Override 45 | public void start(Map settings) { 46 | 47 | config = new ScyllaDbSinkConnectorConfig(settings); 48 | ScyllaDbSessionFactory sessionFactory = new ScyllaDbSessionFactory(); 49 | 50 | try (ScyllaDbSession session = sessionFactory.newSession(config)) { 51 | 52 | if (config.keyspaceCreateEnabled) { 53 | CreateKeyspace createKeyspace = SchemaBuilder.createKeyspace(config.keyspace) 54 | .ifNotExists() 55 | .withReplicationOptions(ImmutableMap.of( 56 | "class", (Object) "SimpleStrategy", 57 | "replication_factor", config.keyspaceReplicationFactor 58 | )) 59 | .withDurableWrites(true); 60 | session.executeStatement(createKeyspace.build()); 61 | } 62 | if (config.offsetEnabledInScyllaDB) { 63 | final CreateTable createOffsetStorage = SchemaBuilder 64 | .createTable(config.keyspace, config.offsetStorageTable) 65 | .ifNotExists() 66 | .withPartitionKey("topic", DataTypes.TEXT) 67 | .withPartitionKey("partition", DataTypes.INT) 68 | .withColumn("offset", DataTypes.BIGINT); 69 | session.executeStatement(createOffsetStorage.build()); 70 | } 71 | 72 | } catch (IOException ex) { 73 | //TODO: improve error handling for both cases 74 | throw new ConnectException("Failed to start the Connector.", ex); 75 | } 76 | } 77 | 78 | @Override 79 | public List> taskConfigs(int maxTasks) { 80 | List> configs = new ArrayList<>(); 81 | Map taskProps = new HashMap<>(); 82 | taskProps.putAll(config.originalsStrings()); 83 | for (int i = 0; i < maxTasks; i++) { 84 | configs.add(taskProps); 85 | } 86 | return configs; 87 | } 88 | 89 | @Override 90 | public void stop() { 91 | log.info("Stopping ScyllaDB Sink Connector."); 92 | } 93 | 94 | @Override 95 | public ConfigDef config() { 96 | return ScyllaDbSinkConnectorConfig.config(); 97 | } 98 | 99 | @Override 100 | public Class taskClass() { 101 | return ScyllaDbSinkTask.class; 102 | } 103 | 104 | @Override 105 | public String version() { 106 | return VersionUtil.getVersion(); 107 | } 108 | 109 | @Override 110 | public Config validate(Map connectorConfigs){ 111 | // Do the usual validation, field by field. 112 | Config result = super.validate(connectorConfigs); 113 | 114 | ConfigValue userConfig = getConfigValue(result, ScyllaDbSinkConnectorConfig.USERNAME_CONFIG); 115 | ConfigValue passwordConfig = getConfigValue(result, ScyllaDbSinkConnectorConfig.PASSWORD_CONFIG); 116 | if (userConfig.value() == null && passwordConfig.value() != null) { 117 | userConfig.addErrorMessage("Username not provided, even though password is."); 118 | } 119 | if (userConfig.value() != null && passwordConfig.value() == null) { 120 | userConfig.addErrorMessage("Password not provided, even though username is."); 121 | } 122 | 123 | ConfigValue securityEnable = getConfigValue(result, ScyllaDbSinkConnectorConfig.SECURITY_ENABLE_CONFIG); 124 | if (Objects.equals(securityEnable.value(), true)) { 125 | if (userConfig.value() == null) { 126 | userConfig.addErrorMessage("Username is required with security enabled."); 127 | } 128 | if (passwordConfig.value() == null) { 129 | passwordConfig.addErrorMessage("Password is required with security enabled."); 130 | } 131 | } 132 | 133 | boolean hasErrors = result.configValues().stream().anyMatch(c -> !(c.errorMessages().isEmpty()) ); 134 | if (!hasErrors) 135 | { 136 | config = new ScyllaDbSinkConnectorConfig(connectorConfigs); 137 | // Make trial connection 138 | ScyllaDbSessionFactory sessionFactory = new ScyllaDbSessionFactory(); 139 | try (ScyllaDbSession session = sessionFactory.newSession(config)) { 140 | if (!session.isValid()) { 141 | ConfigValue contactPoints = getConfigValue(result, ScyllaDbSinkConnectorConfig.CONTACT_POINTS_CONFIG); 142 | contactPoints.addErrorMessage("Session is invalid."); 143 | } else { 144 | ConfigValue keyspaceCreateConfig = getConfigValue(result, ScyllaDbSinkConnectorConfig.KEYSPACE_CREATE_ENABLED_CONFIG); 145 | ConfigValue keyspaceConfig = getConfigValue(result, ScyllaDbSinkConnectorConfig.KEYSPACE_CONFIG); 146 | if (config.keyspaceCreateEnabled) { 147 | if (keyspaceConfig.value() == null) { 148 | // This is possibly not needed, since keyspace should be always required non-null. 149 | keyspaceConfig.addErrorMessage("Scylla keyspace name is required when keyspace creation is enabled."); 150 | } 151 | } else { 152 | // Check if keyspace exists: 153 | if (!session.keyspaceExists(config.keyspace)) { 154 | keyspaceCreateConfig.addErrorMessage("Seems provided keyspace is not present. Did you mean to set " 155 | + ScyllaDbSinkConnectorConfig.KEYSPACE_CREATE_ENABLED_CONFIG 156 | + " to true?"); 157 | keyspaceConfig.addErrorMessage("Keyspace " + config.keyspace + " not found."); 158 | } 159 | } 160 | } 161 | } catch (Exception ex) { 162 | ConfigValue contactPoints = getConfigValue(result, ScyllaDbSinkConnectorConfig.CONTACT_POINTS_CONFIG); 163 | contactPoints.addErrorMessage("Failed to establish trial session: " + ex.getMessage()); 164 | log.error("Validation ended with exception", ex); 165 | } 166 | } 167 | return result; 168 | } 169 | 170 | private ConfigValue getConfigValue(Config config, String configName){ 171 | return config.configValues().stream() 172 | .filter(value -> value.name().equals(configName) ) 173 | .findFirst().get(); 174 | } 175 | } -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/ScyllaDbSinkTask.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.Collection; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Set; 10 | import java.util.concurrent.CompletionStage; 11 | import java.util.concurrent.TimeUnit; 12 | import java.util.concurrent.TimeoutException; 13 | import java.util.stream.Collectors; 14 | 15 | import com.datastax.oss.driver.api.core.AllNodesFailedException; 16 | import com.datastax.oss.driver.api.core.cql.AsyncResultSet; 17 | import com.datastax.oss.driver.api.core.cql.BoundStatement; 18 | import io.connect.scylladb.utils.VersionUtil; 19 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 20 | import org.apache.kafka.common.TopicPartition; 21 | import org.apache.kafka.connect.errors.ConnectException; 22 | import org.apache.kafka.connect.errors.DataException; 23 | import org.apache.kafka.connect.errors.RetriableException; 24 | import org.apache.kafka.connect.sink.SinkRecord; 25 | import org.apache.kafka.connect.sink.SinkTask; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | 30 | /** 31 | * Task class for ScyllaDB Sink Connector. 32 | */ 33 | public class ScyllaDbSinkTask extends SinkTask { 34 | 35 | private static final Logger log = LoggerFactory.getLogger(ScyllaDbSinkTask.class); 36 | 37 | private ScyllaDbSinkConnectorConfig config; 38 | private Map topicOffsets; 39 | ScyllaDbSession session; 40 | 41 | /** 42 | * Starts the sink task. 43 | * If scylladb.offset.storage.table.enable is set to true, 44 | * the task will load offsets for each Kafka topic-partition from 45 | * ScyllaDB offset table into task context. 46 | */ 47 | @Override 48 | public void start(Map settings) { 49 | this.config = new ScyllaDbSinkConnectorConfig(settings); 50 | if (config.isOffsetEnabledInScyllaDb()) { 51 | Set assignment = context.assignment(); 52 | this.session = getValidSession(); 53 | Map offsets = session.loadOffsets(assignment); 54 | if (!offsets.isEmpty()) { 55 | context.offset(offsets); 56 | } 57 | } 58 | topicOffsets = new HashMap<>(); 59 | } 60 | 61 | /* 62 | * Returns a ScyllaDB session. 63 | * Creates a session, if not already exists. 64 | * In case the when session is not valid, 65 | * it closes the existing session and creates a new one. 66 | */ 67 | private ScyllaDbSession getValidSession() { 68 | 69 | ScyllaDbSessionFactory sessionFactory = new ScyllaDbSessionFactory(); 70 | 71 | if (session == null) { 72 | log.info("Creating ScyllaDb Session."); 73 | session = sessionFactory.newSession(this.config); 74 | } 75 | 76 | if (!session.isValid()) { 77 | log.warn("ScyllaDb Session is invalid. Closing and creating new."); 78 | close(); 79 | session = sessionFactory.newSession(this.config); 80 | } 81 | return session; 82 | } 83 | 84 | /** 85 | *
    86 | *
  1. Validates the kafka records. 87 | *
  2. Writes or deletes records from Kafka topic into ScyllaDB. 88 | *
  3. Requests to commit the records when the scyllaDB operations are successful. 89 | *
90 | */ 91 | @Override 92 | public void put(Collection records) { 93 | final List> futures = new ArrayList<>(records.size()); 94 | 95 | for (SinkRecord record : records) { 96 | try { 97 | ScyllaDbSinkTaskHelper scyllaDbSinkTaskHelper = new ScyllaDbSinkTaskHelper(config, getValidSession()); 98 | scyllaDbSinkTaskHelper.validateRecord(record); 99 | 100 | BoundStatement boundStatement = scyllaDbSinkTaskHelper.getBoundStatementForRecord(record); 101 | log.trace("put() - Executing Bound Statement {} for {}:{}:{}", 102 | boundStatement.getPreparedStatement().getQuery(), 103 | record.topic(), 104 | record.kafkaPartition(), 105 | record.kafkaOffset() 106 | ); 107 | 108 | // Commit offset in the case of successful processing of sink record 109 | topicOffsets.put(new TopicPartition(record.topic(), record.kafkaPartition()), 110 | new OffsetAndMetadata(record.kafkaOffset() + 1)); 111 | 112 | CompletionStage resultSetFuture = this.getValidSession().executeStatementAsync(boundStatement); 113 | futures.add(resultSetFuture); 114 | } catch (DataException | NullPointerException ex) { 115 | handleErrors(record, ex); 116 | } 117 | } 118 | 119 | if (!futures.isEmpty()) { 120 | try { 121 | log.debug("put() - Checking future(s)"); 122 | for (CompletionStage future : futures) { 123 | AsyncResultSet resultSet = 124 | future.toCompletableFuture().get(this.config.statementTimeoutMs, TimeUnit.MILLISECONDS); 125 | } 126 | context.requestCommit(); 127 | // TODO : Log the records that fail in Queue/Kafka Topic. 128 | } catch (AllNodesFailedException ex) { 129 | log.debug("put() - Setting clusterValid = false", ex); 130 | getValidSession().setInvalid(); 131 | throw new RetriableException(ex); 132 | } catch (TimeoutException ex) { 133 | log.error("put() - TimeoutException.", ex); 134 | throw new RetriableException(ex); 135 | } catch (Exception ex) { 136 | log.error("put() - Unknown exception. Setting clusterValid = false", ex); 137 | getValidSession().setInvalid(); 138 | throw new RetriableException(ex); 139 | } 140 | } 141 | } 142 | 143 | /** 144 | * If scylladb.offset.storage.table.enable is set to true, 145 | * updates offsets in ScyllaDB table. 146 | * Else, assumes all the records in previous @put call were successfully 147 | * written in to ScyllaDB and returns the same offsets. 148 | */ 149 | @Override 150 | public Map preCommit( 151 | Map currentOffsets 152 | ) { 153 | if (config.isOffsetEnabledInScyllaDb()) { 154 | try { 155 | log.debug("flush() - Flushing offsets to {}", this.config.offsetStorageTable); 156 | List> insertFutures = currentOffsets.entrySet().stream() 157 | .map(e -> this.getValidSession().getInsertOffsetStatement(e.getKey(), e.getValue())) 158 | .map(s -> getValidSession().executeStatementAsync(s)) 159 | .collect(Collectors.toList()); 160 | 161 | for (CompletionStage future : insertFutures) { 162 | future.toCompletableFuture().get(this.config.statementTimeoutMs, TimeUnit.MILLISECONDS); 163 | } 164 | } catch (AllNodesFailedException ex) { 165 | log.debug("put() - Setting clusterValid = false", ex); 166 | getValidSession().setInvalid(); 167 | throw new RetriableException(ex); 168 | } catch (Exception ex) { 169 | log.error("put() - Unknown exception. Setting clusterValid = false", ex); 170 | getValidSession().setInvalid(); 171 | throw new RetriableException(ex); 172 | } 173 | } 174 | return currentOffsets; 175 | } 176 | 177 | /** 178 | * handle error based on configured behavior on error. 179 | */ 180 | private void handleErrors(SinkRecord record, Exception ex) { 181 | if (config.behaviourOnError == ScyllaDbSinkConnectorConfig.BehaviorOnError.FAIL) { 182 | throw new ConnectException("Exception occurred while " 183 | + "extracting records from Kafka Sink Records.", ex); 184 | } else if (config.behaviourOnError == ScyllaDbSinkConnectorConfig.BehaviorOnError.LOG) { 185 | log.warn("Exception occurred while extracting records from Kafka Sink Records, " 186 | + "ignoring and processing next set of records.", ex); 187 | } else { 188 | log.trace("Exception occurred while extracting records from Kafka Sink Records, " 189 | + "ignoring and processing next set of records.", ex); 190 | } 191 | // Commit offset in the case when BehaviorOnError is not FAIL. 192 | topicOffsets.put(new TopicPartition(record.topic(), record.kafkaPartition()), 193 | new OffsetAndMetadata(record.kafkaOffset() + 1)); 194 | } 195 | 196 | /** 197 | * Closes the ScyllaDB session and proceeds to closing sink task. 198 | */ 199 | @Override 200 | public void stop() { 201 | close(); 202 | } 203 | 204 | // Visible for testing 205 | void close() { 206 | if (null != this.session) { 207 | log.info("Closing getValidSession"); 208 | try { 209 | this.session.close(); 210 | } catch (IOException ex) { 211 | log.error("Exception thrown while closing ScyllaDB session.", ex); 212 | } 213 | this.session = null; 214 | } 215 | } 216 | 217 | @Override 218 | public String version() { 219 | return VersionUtil.getVersion(); 220 | } 221 | } -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/ScyllaDbSinkTaskHelper.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.datastax.oss.driver.api.core.cql.BoundStatement; 4 | import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; 5 | import io.connect.scylladb.topictotable.TopicConfigs; 6 | import io.connect.scylladb.utils.ScyllaDbConstants; 7 | import org.apache.kafka.common.TopicPartition; 8 | import org.apache.kafka.connect.data.Struct; 9 | import org.apache.kafka.connect.errors.DataException; 10 | import org.apache.kafka.connect.sink.SinkRecord; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.Map; 15 | 16 | public class ScyllaDbSinkTaskHelper { 17 | 18 | private static final Logger log = LoggerFactory.getLogger(ScyllaDbSinkTaskHelper.class); 19 | 20 | private ScyllaDbSinkConnectorConfig scyllaDbSinkConnectorConfig; 21 | private ScyllaDbSession session; 22 | private Map topicPartitionRecordSizeMap; 23 | 24 | 25 | public ScyllaDbSinkTaskHelper(ScyllaDbSinkConnectorConfig scyllaDbSinkConnectorConfig, 26 | ScyllaDbSession session) { 27 | this.scyllaDbSinkConnectorConfig = scyllaDbSinkConnectorConfig; 28 | this.session = session; 29 | } 30 | 31 | public void validateRecord(SinkRecord record) { 32 | if (null == record.key()) { 33 | throw new DataException( 34 | "Record with a null key was encountered. This connector requires that records " 35 | + "from Kafka contain the keys for the ScyllaDb table. Please use a " 36 | + "transformation like org.apache.kafka.connect.transforms.ValueToKey " 37 | + "to create a key with the proper fields." 38 | ); 39 | } 40 | 41 | if (!(record.key() instanceof Struct) && !(record.key() instanceof Map)) { 42 | throw new DataException( 43 | "Key must be a struct or map. This connector requires that records from Kafka " 44 | + "contain the keys for the ScyllaDb table. Please use a transformation like " 45 | + "org.apache.kafka.connect.transforms.ValueToKey to create a key with the " 46 | + "proper fields." 47 | ); 48 | } 49 | } 50 | 51 | public BoundStatement getBoundStatementForRecord(SinkRecord record) { 52 | final String tableName = record.topic().replaceAll("\\.", "_").replaceAll("-", "_"); 53 | BoundStatement boundStatement = null; 54 | TopicConfigs topicConfigs = null; 55 | if (scyllaDbSinkConnectorConfig.topicWiseConfigs.containsKey(tableName)) { 56 | topicConfigs = scyllaDbSinkConnectorConfig.topicWiseConfigs.get(tableName); 57 | if (topicConfigs.getMappingStringForTopic() != null && !topicConfigs.isScyllaColumnsMapped()) { 58 | topicConfigs.setTablePartitionAndColumnValues(record); 59 | } 60 | topicConfigs.setTtlAndTimeStampIfAvailable(record); 61 | } 62 | if (null == record.value()) { 63 | boolean deletionEnabled = topicConfigs != null 64 | ? topicConfigs.isDeletesEnabled() : scyllaDbSinkConnectorConfig.deletesEnabled; 65 | if (deletionEnabled) { 66 | if (this.session.tableExists(tableName)) { 67 | final RecordToBoundStatementConverter boundStatementConverter = this.session.delete(tableName); 68 | final RecordToBoundStatementConverter.State state = boundStatementConverter.convert(record, null, ScyllaDbConstants.DELETE_OPERATION); 69 | Preconditions.checkState( 70 | state.parameters > 0, 71 | "key must contain the columns in the primary key." 72 | ); 73 | boundStatement = state.statement; 74 | } else { 75 | log.warn("put() - table '{}' does not exist. Skipping delete.", tableName); 76 | } 77 | } else { 78 | throw new DataException( 79 | String.format("Record with null value found for the key '%s'. If you are trying to delete the record set " 80 | + "scylladb.deletes.enabled = true or topic.my_topic.my_ks.my_table.deletesEnabled = true in " 81 | + "your connector configuration.", 82 | record.key())); 83 | } 84 | } else { 85 | this.session.createOrAlterTable(tableName, record, topicConfigs); 86 | final RecordToBoundStatementConverter boundStatementConverter = this.session.insert(tableName, topicConfigs); 87 | final RecordToBoundStatementConverter.State state = boundStatementConverter.convert(record, topicConfigs, ScyllaDbConstants.INSERT_OPERATION); 88 | boundStatement = state.statement; 89 | } 90 | 91 | if (topicConfigs != null) { 92 | log.trace("Topic mapped Consistency level : " + topicConfigs.getConsistencyLevel() 93 | + ", Record/Topic mapped timestamp : " + topicConfigs.getTimeStamp()); 94 | boundStatement = boundStatement.setConsistencyLevel(topicConfigs.getConsistencyLevel()); 95 | boundStatement = boundStatement.setQueryTimestamp(topicConfigs.getTimeStamp()); 96 | } else { 97 | boundStatement = boundStatement.setConsistencyLevel(this.scyllaDbSinkConnectorConfig.consistencyLevel); 98 | // Timestamps in Kafka (record.timestamp()) are in millisecond precision, 99 | // while Scylla expects a microsecond precision: 1 ms = 1000 us. 100 | boundStatement = boundStatement.setQueryTimestamp(record.timestamp() * 1000); 101 | } 102 | return boundStatement; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/TableMetadata.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.datastax.oss.driver.api.core.type.DataType; 4 | 5 | import java.util.List; 6 | 7 | interface TableMetadata { 8 | 9 | interface Column { 10 | 11 | String getName(); 12 | 13 | DataType getType(); 14 | } 15 | 16 | interface Table { 17 | /** 18 | * Keyspace for the table. 19 | * 20 | * @return Keyspace for the table. 21 | */ 22 | String keyspace(); 23 | 24 | /** 25 | * Method is used to return the metadata for a column. 26 | * 27 | * @param columnName getName of the column to return metadata for. 28 | * @return Null if the column does not exist. 29 | */ 30 | Column columnMetadata(String columnName); 31 | 32 | /** 33 | * Method is used to return all of the columns for a table. 34 | * 35 | * @return List of columns for the table. 36 | */ 37 | List columns(); 38 | 39 | /** 40 | * Method is used to return the primary key columns for a table. 41 | * @return List of columns in the primary key. 42 | */ 43 | List primaryKey(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/TableMetadataImpl.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; 4 | import com.datastax.oss.driver.api.core.type.DataType; 5 | import com.datastax.oss.driver.shaded.guava.common.base.MoreObjects; 6 | import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.Map; 11 | import java.util.TreeMap; 12 | import java.util.stream.Collectors; 13 | 14 | class TableMetadataImpl { 15 | static class ColumnImpl implements TableMetadata.Column { 16 | final ColumnMetadata columnMetadata; 17 | 18 | ColumnImpl(ColumnMetadata columnMetadata) { 19 | this.columnMetadata = columnMetadata; 20 | } 21 | 22 | @Override 23 | public String getName() { 24 | return this.columnMetadata.getName().toString(); 25 | } 26 | 27 | @Override 28 | public DataType getType() { 29 | return this.columnMetadata.getType(); 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return MoreObjects.toStringHelper(this) 35 | .add("name", this.columnMetadata.getName()) 36 | .add("type", this.columnMetadata.getType().asCql(true, true)) 37 | .toString(); 38 | } 39 | } 40 | 41 | static class TableImpl implements TableMetadata.Table { 42 | final String name; 43 | final String keyspace; 44 | final com.datastax.oss.driver.api.core.metadata.schema.TableMetadata tableMetadata; 45 | final Map columns; 46 | final List primaryKey; 47 | 48 | TableImpl(com.datastax.oss.driver.api.core.metadata.schema.TableMetadata tableMetadata) { 49 | this.tableMetadata = tableMetadata; 50 | this.name = this.tableMetadata.getName().asInternal(); 51 | this.keyspace = this.tableMetadata.getKeyspace().asInternal(); 52 | this.primaryKey = this.tableMetadata.getPrimaryKey() 53 | .stream() 54 | .map(ColumnImpl::new) 55 | .collect(Collectors.toList()); 56 | List allColumns = new ArrayList<>(); 57 | allColumns.addAll( 58 | this.tableMetadata.getColumns().values().stream() 59 | .map(ColumnImpl::new) 60 | .collect(Collectors.toList()) 61 | ); 62 | this.columns = allColumns.stream() 63 | .collect(Collectors.toMap( 64 | TableMetadata.Column::getName, 65 | c -> c, 66 | (o, n) -> n, 67 | () -> new TreeMap<>(String.CASE_INSENSITIVE_ORDER) 68 | )); 69 | } 70 | 71 | @Override 72 | public String keyspace() { 73 | return this.keyspace; 74 | } 75 | 76 | @Override 77 | public TableMetadata.Column columnMetadata(String columnName) { 78 | return this.columns.get(columnName); 79 | } 80 | 81 | @Override 82 | public List columns() { 83 | return ImmutableList.copyOf(this.columns.values()); 84 | } 85 | 86 | @Override 87 | public List primaryKey() { 88 | return this.primaryKey; 89 | } 90 | 91 | @Override 92 | public String toString() { 93 | return MoreObjects.toStringHelper(this) 94 | .add("keyspace", this.keyspace) 95 | .add("name", this.name) 96 | .add("columns", this.columns) 97 | .add("primaryKey", this.primaryKey) 98 | .toString(); 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/codec/ConvenienceCodecs.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.codec; 2 | 3 | import com.datastax.oss.driver.api.core.type.codec.MappingCodec; 4 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 5 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 6 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 7 | import edu.umd.cs.findbugs.annotations.Nullable; 8 | 9 | import java.lang.reflect.Type; 10 | import java.math.BigDecimal; 11 | import java.math.BigInteger; 12 | import java.time.Instant; 13 | import java.util.ArrayList; 14 | import java.util.Arrays; 15 | import java.util.Date; 16 | import java.util.List; 17 | 18 | public class ConvenienceCodecs { 19 | public static class ByteToBigintCodec extends MappingCodec { 20 | 21 | public static final ByteToBigintCodec INSTANCE = new ByteToBigintCodec(); 22 | 23 | public ByteToBigintCodec() { super(TypeCodecs.BIGINT, GenericType.BYTE); } 24 | 25 | @Nullable 26 | @Override 27 | protected Byte innerToOuter(@Nullable Long value) { 28 | return value == null ? null : value.byteValue(); 29 | } 30 | 31 | @Nullable 32 | @Override 33 | protected Long outerToInner(@Nullable Byte value) { 34 | return value == null ? null : value.longValue(); 35 | } 36 | } 37 | 38 | public static class ShortToBigintCodec extends MappingCodec { 39 | 40 | public static final ShortToBigintCodec INSTANCE = new ShortToBigintCodec(); 41 | 42 | public ShortToBigintCodec() { super(TypeCodecs.BIGINT, GenericType.SHORT); } 43 | 44 | @Nullable 45 | @Override 46 | protected Short innerToOuter(@Nullable Long value) { 47 | return value == null ? null : value.shortValue(); 48 | } 49 | 50 | @Nullable 51 | @Override 52 | protected Long outerToInner(@Nullable Short value) { 53 | return value == null ? null : value.longValue(); 54 | } 55 | } 56 | 57 | public static class IntegerToBigintCodec extends MappingCodec { 58 | 59 | public static final IntegerToBigintCodec INSTANCE = new IntegerToBigintCodec(); 60 | 61 | public IntegerToBigintCodec() { super(TypeCodecs.BIGINT, GenericType.INTEGER); } 62 | 63 | @Nullable 64 | @Override 65 | protected Integer innerToOuter(@Nullable Long value) { 66 | return value == null ? null : value.intValue(); 67 | } 68 | 69 | @Nullable 70 | @Override 71 | protected Long outerToInner(@Nullable Integer value) { 72 | return value == null ? null : value.longValue(); 73 | } 74 | } 75 | 76 | public static class FloatToDecimalCodec extends MappingCodec { 77 | 78 | public static final FloatToDecimalCodec INSTANCE = new FloatToDecimalCodec(); 79 | 80 | public FloatToDecimalCodec() { super(TypeCodecs.DECIMAL, GenericType.FLOAT); } 81 | 82 | @Nullable 83 | @Override 84 | protected Float innerToOuter(@Nullable BigDecimal value) { 85 | return value == null ? null : value.floatValue(); 86 | } 87 | 88 | @Nullable 89 | @Override 90 | protected BigDecimal outerToInner(@Nullable Float value) { 91 | return value == null ? null : BigDecimal.valueOf(value.doubleValue()); 92 | } 93 | } 94 | 95 | public static class DoubleToDecimalCodec extends MappingCodec { 96 | 97 | public static final DoubleToDecimalCodec INSTANCE = new DoubleToDecimalCodec(); 98 | 99 | public DoubleToDecimalCodec() { super(TypeCodecs.DECIMAL, GenericType.DOUBLE); } 100 | 101 | @Nullable 102 | @Override 103 | protected Double innerToOuter(@Nullable BigDecimal value) { 104 | return value == null ? null : value.doubleValue(); 105 | } 106 | 107 | @Nullable 108 | @Override 109 | protected BigDecimal outerToInner(@Nullable Double value) { 110 | return value == null ? null : BigDecimal.valueOf(value); 111 | } 112 | } 113 | 114 | public static class IntegerToDecimalCodec extends MappingCodec { 115 | 116 | public static final IntegerToDecimalCodec INSTANCE = new IntegerToDecimalCodec(); 117 | 118 | public IntegerToDecimalCodec() { super(TypeCodecs.DECIMAL, GenericType.INTEGER); } 119 | 120 | @Nullable 121 | @Override 122 | protected Integer innerToOuter(@Nullable BigDecimal value) { 123 | return value == null ? null : value.intValue(); 124 | } 125 | 126 | @Nullable 127 | @Override 128 | protected BigDecimal outerToInner(@Nullable Integer value) { 129 | return value == null ? null : BigDecimal.valueOf(value.longValue()); 130 | } 131 | } 132 | 133 | public static class LongToDecimalCodec extends MappingCodec { 134 | 135 | public static final LongToDecimalCodec INSTANCE = new LongToDecimalCodec(); 136 | 137 | public LongToDecimalCodec() { super(TypeCodecs.DECIMAL, GenericType.LONG); } 138 | 139 | @Nullable 140 | @Override 141 | protected Long innerToOuter(@Nullable BigDecimal value) { 142 | return value == null ? null : value.longValueExact(); 143 | } 144 | 145 | @Nullable 146 | @Override 147 | protected BigDecimal outerToInner(@Nullable Long value) { 148 | return value == null ? null : BigDecimal.valueOf(value); 149 | } 150 | } 151 | 152 | public static class FloatToDoubleCodec extends MappingCodec { 153 | 154 | public static final FloatToDoubleCodec INSTANCE = new FloatToDoubleCodec(); 155 | 156 | public FloatToDoubleCodec() { super(TypeCodecs.DOUBLE, GenericType.FLOAT); } 157 | 158 | @Nullable 159 | @Override 160 | protected Float innerToOuter(@Nullable Double value) { 161 | return value == null ? null : value.floatValue(); 162 | } 163 | 164 | @Nullable 165 | @Override 166 | protected Double outerToInner(@Nullable Float value) { 167 | return value == null ? null : value.doubleValue(); 168 | } 169 | } 170 | 171 | public static class ByteToIntCodec extends MappingCodec { 172 | 173 | public static final ByteToIntCodec INSTANCE = new ByteToIntCodec(); 174 | 175 | public ByteToIntCodec() { super(TypeCodecs.INT, GenericType.BYTE); } 176 | 177 | @Nullable 178 | @Override 179 | protected Byte innerToOuter(@Nullable Integer value) { 180 | return value == null ? null : value.byteValue(); 181 | } 182 | 183 | @Nullable 184 | @Override 185 | protected Integer outerToInner(@Nullable Byte value) { 186 | return value == null ? null : value.intValue(); 187 | } 188 | } 189 | 190 | public static class ShortToIntCodec extends MappingCodec { 191 | 192 | public static final ShortToIntCodec INSTANCE = new ShortToIntCodec(); 193 | 194 | public ShortToIntCodec() { super(TypeCodecs.INT, GenericType.SHORT); } 195 | 196 | @Nullable 197 | @Override 198 | protected Short innerToOuter(@Nullable Integer value) { 199 | return value == null ? null : value.shortValue(); 200 | } 201 | 202 | @Nullable 203 | @Override 204 | protected Integer outerToInner(@Nullable Short value) { 205 | return value == null ? null : value.intValue(); 206 | } 207 | } 208 | 209 | public static class LongToIntCodec extends MappingCodec { 210 | 211 | public static final LongToIntCodec INSTANCE = new LongToIntCodec(); 212 | 213 | public LongToIntCodec() { super(TypeCodecs.INT, GenericType.LONG); } 214 | 215 | @Nullable 216 | @Override 217 | protected Long innerToOuter(@Nullable Integer value) { 218 | return value == null ? null : value.longValue(); 219 | } 220 | 221 | @Nullable 222 | @Override 223 | protected Integer outerToInner(@Nullable Long value) { 224 | return value == null ? null : value.intValue(); 225 | } 226 | } 227 | 228 | public static class StringToTinyintCodec extends MappingCodec { 229 | 230 | public static final StringToTinyintCodec INSTANCE = new StringToTinyintCodec(); 231 | 232 | public StringToTinyintCodec() { super(TypeCodecs.TINYINT, GenericType.STRING); } 233 | 234 | @Nullable 235 | @Override 236 | protected String innerToOuter(@Nullable Byte value) { 237 | return value == null ? null : value.toString(); 238 | } 239 | 240 | @Nullable 241 | @Override 242 | protected Byte outerToInner(@Nullable String value) { 243 | return value == null ? null : Byte.parseByte(value); 244 | } 245 | } 246 | 247 | public static class StringToSmallintCodec extends MappingCodec { 248 | 249 | public static final StringToSmallintCodec INSTANCE = new StringToSmallintCodec(); 250 | 251 | public StringToSmallintCodec() { super(TypeCodecs.SMALLINT, GenericType.STRING); } 252 | 253 | @Nullable 254 | @Override 255 | protected String innerToOuter(@Nullable Short value) { 256 | return value == null ? null : value.toString(); 257 | } 258 | 259 | @Nullable 260 | @Override 261 | protected Short outerToInner(@Nullable String value) { 262 | return value == null ? null : Short.parseShort(value); 263 | } 264 | } 265 | 266 | public static class StringToIntCodec extends MappingCodec { 267 | 268 | public static final StringToIntCodec INSTANCE = new StringToIntCodec(); 269 | 270 | public StringToIntCodec() { super(TypeCodecs.INT, GenericType.STRING); } 271 | 272 | @Nullable 273 | @Override 274 | protected String innerToOuter(@Nullable Integer value) { 275 | return value == null ? null : value.toString(); 276 | } 277 | 278 | @Nullable 279 | @Override 280 | protected Integer outerToInner(@Nullable String value) { 281 | return value == null ? null : Integer.parseInt(value); 282 | } 283 | } 284 | 285 | public static class StringToBigintCodec extends MappingCodec { 286 | 287 | public static final StringToBigintCodec INSTANCE = new StringToBigintCodec(); 288 | 289 | public StringToBigintCodec() { super(TypeCodecs.BIGINT, GenericType.STRING); } 290 | 291 | @Nullable 292 | @Override 293 | protected String innerToOuter(@Nullable Long value) { 294 | return value == null ? null : value.toString(); 295 | } 296 | 297 | @Nullable 298 | @Override 299 | protected Long outerToInner(@Nullable String value) { 300 | return value == null ? null : Long.parseLong(value); 301 | } 302 | } 303 | 304 | public static class ByteToSmallintCodec extends MappingCodec { 305 | 306 | public static final ByteToSmallintCodec INSTANCE = new ByteToSmallintCodec(); 307 | 308 | public ByteToSmallintCodec() { super(TypeCodecs.SMALLINT, GenericType.BYTE); } 309 | 310 | @Nullable 311 | @Override 312 | protected Byte innerToOuter(@Nullable Short value) { 313 | return value == null ? null : value.byteValue(); 314 | } 315 | 316 | @Nullable 317 | @Override 318 | protected Short outerToInner(@Nullable Byte value) { 319 | return value == null ? null : value.shortValue(); 320 | } 321 | } 322 | 323 | public static class LongToVarintCodec extends MappingCodec { 324 | 325 | public static final LongToVarintCodec INSTANCE = new LongToVarintCodec(); 326 | 327 | public LongToVarintCodec() { super(TypeCodecs.VARINT, GenericType.LONG); } 328 | 329 | @Nullable 330 | @Override 331 | protected Long innerToOuter(@Nullable BigInteger value) { 332 | return value == null ? null : value.longValue(); 333 | } 334 | 335 | @Nullable 336 | @Override 337 | protected BigInteger outerToInner(@Nullable Long value) { 338 | return value == null ? null : BigInteger.valueOf(value); 339 | } 340 | } 341 | 342 | public static class StringToTimestampCodec extends MappingCodec { 343 | 344 | public static final StringToTimestampCodec INSTANCE = new StringToTimestampCodec(); 345 | 346 | public StringToTimestampCodec() { super(TypeCodecs.TIMESTAMP, GenericType.STRING); } 347 | 348 | @Nullable 349 | @Override 350 | protected String innerToOuter(@Nullable Instant value) { 351 | return value == null ? null : value.toString(); 352 | } 353 | 354 | @Nullable 355 | @Override 356 | protected Instant outerToInner(@Nullable String value) { 357 | return value == null ? null : Instant.parse(value); 358 | } 359 | } 360 | 361 | public static final List> ALL_INSTANCES = new ArrayList<>(Arrays.asList( 362 | ByteToBigintCodec.INSTANCE, 363 | ShortToBigintCodec.INSTANCE, 364 | IntegerToBigintCodec.INSTANCE, 365 | FloatToDecimalCodec.INSTANCE, 366 | DoubleToDecimalCodec.INSTANCE, 367 | IntegerToDecimalCodec.INSTANCE, 368 | LongToDecimalCodec.INSTANCE, 369 | FloatToDoubleCodec.INSTANCE, 370 | ByteToIntCodec.INSTANCE, 371 | ShortToIntCodec.INSTANCE, 372 | LongToIntCodec.INSTANCE, 373 | StringToTinyintCodec.INSTANCE, 374 | StringToSmallintCodec.INSTANCE, 375 | StringToIntCodec.INSTANCE, 376 | StringToBigintCodec.INSTANCE, 377 | ByteToSmallintCodec.INSTANCE, 378 | LongToVarintCodec.INSTANCE, 379 | StringToTimestampCodec.INSTANCE 380 | )); 381 | } 382 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/codec/StringAbstractCodec.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.codec; 2 | 3 | import com.datastax.oss.driver.api.core.ProtocolVersion; 4 | import com.datastax.oss.driver.api.core.type.DataType; 5 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 6 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 7 | import edu.umd.cs.findbugs.annotations.NonNull; 8 | import edu.umd.cs.findbugs.annotations.Nullable; 9 | 10 | import java.nio.ByteBuffer; 11 | import java.util.Objects; 12 | 13 | public class StringAbstractCodec implements TypeCodec { 14 | protected final TypeCodec TCodec; 15 | protected final DataType dataType; 16 | 17 | protected StringAbstractCodec(DataType dataType, TypeCodec baseCodec) { 18 | Objects.requireNonNull(baseCodec); 19 | this.TCodec = baseCodec; 20 | this.dataType = dataType; 21 | } 22 | 23 | protected T parseAsT(String value) { 24 | return TCodec.parse(value); 25 | } 26 | 27 | @Override 28 | public String parse(String value) { 29 | try { 30 | T t = parseAsT(value); 31 | return t != null ? t.toString() : null; 32 | } catch (IllegalArgumentException e) { 33 | throw new IllegalArgumentException(String.format("Cannot parse string value from \"%s\"", value), e); 34 | } 35 | } 36 | 37 | @NonNull 38 | @Override 39 | public GenericType getJavaType() { 40 | return GenericType.STRING; 41 | } 42 | 43 | @NonNull 44 | @Override 45 | public DataType getCqlType() { 46 | return dataType; 47 | } 48 | 49 | @Nullable 50 | @Override 51 | public ByteBuffer encode(@Nullable String value, @NonNull ProtocolVersion protocolVersion) { 52 | T t = parseAsT(value); 53 | return t == null ? null : TCodec.encode(t, protocolVersion); 54 | } 55 | 56 | @Nullable 57 | @Override 58 | public String decode(@Nullable ByteBuffer bytes, @NonNull ProtocolVersion protocolVersion) { 59 | T t = TCodec.decode(bytes, protocolVersion); 60 | return t == null ? null : t.toString(); 61 | } 62 | 63 | @NonNull 64 | @Override 65 | public String format(String value) { 66 | return value == null ? "NULL" : value; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/codec/StringDurationCodec.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.codec; 2 | 3 | import com.datastax.oss.driver.api.core.data.CqlDuration; 4 | import com.datastax.oss.driver.api.core.type.DataTypes; 5 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 6 | 7 | public class StringDurationCodec extends StringAbstractCodec { 8 | public static final StringDurationCodec INSTANCE = new StringDurationCodec(); 9 | 10 | public StringDurationCodec() { 11 | super(DataTypes.DURATION, TypeCodecs.DURATION); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/codec/StringInetCodec.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.codec; 2 | 3 | import com.datastax.oss.driver.api.core.type.DataTypes; 4 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 5 | 6 | import java.net.InetAddress; 7 | 8 | public class StringInetCodec extends StringAbstractCodec { 9 | public static final StringInetCodec INSTANCE = new StringInetCodec(); 10 | public StringInetCodec() { 11 | super(DataTypes.INET, TypeCodecs.INET); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/codec/StringTimeUuidCodec.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.codec; 2 | 3 | import com.datastax.oss.driver.api.core.ProtocolVersion; 4 | import com.datastax.oss.driver.api.core.type.DataType; 5 | import com.datastax.oss.driver.api.core.type.DataTypes; 6 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 7 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 8 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 9 | import edu.umd.cs.findbugs.annotations.NonNull; 10 | import edu.umd.cs.findbugs.annotations.Nullable; 11 | 12 | import java.nio.ByteBuffer; 13 | import java.util.UUID; 14 | 15 | public class StringTimeUuidCodec implements TypeCodec { 16 | public static final StringTimeUuidCodec INSTANCE = new StringTimeUuidCodec(); 17 | 18 | @NonNull 19 | @Override 20 | public GenericType getJavaType() { 21 | return GenericType.STRING; 22 | } 23 | 24 | @NonNull 25 | @Override 26 | public DataType getCqlType() { 27 | return DataTypes.TIMEUUID; 28 | } 29 | 30 | @Nullable 31 | @Override 32 | public ByteBuffer encode(@Nullable String value, @NonNull ProtocolVersion protocolVersion) { 33 | UUID uuid = TypeCodecs.TIMEUUID.parse(value); 34 | return uuid == null ? null : TypeCodecs.TIMEUUID.encode(uuid, protocolVersion); 35 | } 36 | 37 | @Nullable 38 | @Override 39 | public String decode(@Nullable ByteBuffer bytes, @NonNull ProtocolVersion protocolVersion) { 40 | UUID uuid = TypeCodecs.TIMEUUID.decode(bytes, protocolVersion); 41 | return uuid == null ? null : uuid.toString(); 42 | } 43 | 44 | @NonNull 45 | @Override 46 | public String format(@Nullable String value) { 47 | return value == null ? "NULL" : value; 48 | } 49 | 50 | @Nullable 51 | @Override 52 | public String parse(@Nullable String value) { 53 | try { 54 | UUID uuid = TypeCodecs.TIMEUUID.parse(value); 55 | return uuid != null ? uuid.toString() : null; 56 | } catch (IllegalArgumentException e) { 57 | throw new IllegalArgumentException(String.format("Cannot parse string TIMEUUID value from \"%s\"", value), e); 58 | } 59 | 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/codec/StringUuidCodec.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.codec; 2 | 3 | import com.datastax.oss.driver.api.core.ProtocolVersion; 4 | import com.datastax.oss.driver.api.core.type.DataType; 5 | import com.datastax.oss.driver.api.core.type.DataTypes; 6 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 7 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 8 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 9 | import edu.umd.cs.findbugs.annotations.NonNull; 10 | import edu.umd.cs.findbugs.annotations.Nullable; 11 | 12 | import java.nio.ByteBuffer; 13 | import java.util.UUID; 14 | 15 | public class StringUuidCodec implements TypeCodec { 16 | public static final StringUuidCodec INSTANCE = new StringUuidCodec(); 17 | 18 | @NonNull 19 | @Override 20 | public GenericType getJavaType() { 21 | return GenericType.STRING; 22 | } 23 | 24 | @NonNull 25 | @Override 26 | public DataType getCqlType() { 27 | return DataTypes.UUID; 28 | } 29 | 30 | @Nullable 31 | @Override 32 | public ByteBuffer encode(@Nullable String value, @NonNull ProtocolVersion protocolVersion) { 33 | UUID uuid = TypeCodecs.UUID.parse(value); 34 | return uuid == null ? null : TypeCodecs.UUID.encode(uuid, protocolVersion); 35 | } 36 | 37 | @Nullable 38 | @Override 39 | public String decode(@Nullable ByteBuffer bytes, @NonNull ProtocolVersion protocolVersion) { 40 | UUID uuid = TypeCodecs.UUID.decode(bytes, protocolVersion); 41 | return uuid == null ? null : uuid.toString(); 42 | } 43 | 44 | @NonNull 45 | @Override 46 | public String format(@Nullable String value) { 47 | return value == null ? "NULL" : value; 48 | } 49 | 50 | @Nullable 51 | @Override 52 | public String parse(@Nullable String value) { 53 | try { 54 | UUID uuid = TypeCodecs.UUID.parse(value); 55 | return uuid != null ? uuid.toString() : null; 56 | } catch (IllegalArgumentException e) { 57 | throw new IllegalArgumentException(String.format("Cannot parse string UUID value from \"%s\"", value), e); 58 | } 59 | 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/codec/StringVarintCodec.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.codec; 2 | 3 | import com.datastax.oss.driver.api.core.type.DataTypes; 4 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 5 | 6 | import java.math.BigInteger; 7 | 8 | public class StringVarintCodec extends StringAbstractCodec { 9 | public static final StringVarintCodec INSTANCE = new StringVarintCodec(); 10 | 11 | public StringVarintCodec() { 12 | super(DataTypes.VARINT, TypeCodecs.VARINT); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/topictotable/TopicConfigs.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.topictotable; 2 | 3 | import com.datastax.oss.driver.api.core.ConsistencyLevel; 4 | import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; 5 | import com.datastax.oss.driver.shaded.guava.common.base.Joiner; 6 | import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; 7 | import io.connect.scylladb.ScyllaDbSinkConnectorConfig; 8 | import org.apache.kafka.connect.data.Field; 9 | import org.apache.kafka.connect.data.Schema; 10 | import org.apache.kafka.connect.data.Struct; 11 | import org.apache.kafka.connect.errors.DataException; 12 | import org.apache.kafka.connect.header.Header; 13 | import org.apache.kafka.connect.sink.SinkRecord; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.util.Arrays; 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | import java.util.stream.Collectors; 21 | 22 | public class TopicConfigs { 23 | 24 | private static final Logger log = LoggerFactory.getLogger(TopicConfigs.class); 25 | private String mappingStringForTopic; 26 | private Map tablePartitionKeyMap; 27 | private Map tableColumnMap; 28 | private ConsistencyLevel consistencyLevel = null; 29 | private String ttlMappedField; 30 | private Integer ttl; 31 | private String timeStampMappedField; 32 | private Long timeStamp; 33 | private boolean deletesEnabled; 34 | private boolean isScyllaColumnsMapped; 35 | 36 | public TopicConfigs(Map configsMapForTheTopic, 37 | ScyllaDbSinkConnectorConfig scyllaDbSinkConnectorConfig) { 38 | this.tablePartitionKeyMap = new HashMap<>(); 39 | this.tableColumnMap = new HashMap<>(); 40 | this.consistencyLevel = scyllaDbSinkConnectorConfig.consistencyLevel; 41 | this.ttl = scyllaDbSinkConnectorConfig.ttl; 42 | this.deletesEnabled = scyllaDbSinkConnectorConfig.deletesEnabled; 43 | if (configsMapForTheTopic.containsKey("mapping")) { 44 | this.mappingStringForTopic = configsMapForTheTopic.get("mapping"); 45 | } 46 | if (configsMapForTheTopic.containsKey("deletesEnabled")) { 47 | String deleteEnabledValue = configsMapForTheTopic.get("deletesEnabled"); 48 | if ("true".equalsIgnoreCase(deleteEnabledValue) || "false".equalsIgnoreCase(deleteEnabledValue)) { 49 | this.deletesEnabled = Boolean.parseBoolean(deleteEnabledValue); 50 | } else { 51 | throw new DataException( 52 | String.format("%s is not a valid value for deletesEnabled. Valid values are : true, false", 53 | deleteEnabledValue 54 | ) 55 | ); 56 | } 57 | } 58 | try { 59 | if (configsMapForTheTopic.containsKey("ttlSeconds")) { 60 | this.ttl = Integer.parseInt(configsMapForTheTopic.get("ttlSeconds")); 61 | } 62 | if (configsMapForTheTopic.containsKey("consistencyLevel")) { 63 | this.consistencyLevel = DefaultConsistencyLevel.valueOf(configsMapForTheTopic.get("consistencyLevel")); 64 | } 65 | } catch (NumberFormatException e) { 66 | throw new DataException( 67 | String.format("The setting ttlSeconds must be of type Integer. %s is not a supported type", 68 | configsMapForTheTopic.get("ttlSeconds").getClass().getName())); 69 | } catch (IllegalArgumentException e) { 70 | throw new DataException( 71 | String.format("%s is not a valid value for consistencyLevel. Valid values are %s", 72 | configsMapForTheTopic.get("consistencyLevel"), Arrays.toString(DefaultConsistencyLevel.values())) 73 | ); 74 | } 75 | } 76 | 77 | public void setTablePartitionAndColumnValues(SinkRecord record) { 78 | for (String mappedEntry : this.mappingStringForTopic.split(",")) { 79 | String[] columnNameMap = mappedEntry.split("="); 80 | String recordField = columnNameMap[1].split("\\.").length > 0 81 | ? columnNameMap[1].split("\\.")[1] : ""; 82 | String scyllaColumnName = columnNameMap[0].trim(); 83 | KafkaScyllaColumnMapper kafkaScyllaColumnMapper = new KafkaScyllaColumnMapper(scyllaColumnName); 84 | if (columnNameMap[1].startsWith("key.")) { 85 | if (record.keySchema() != null) { 86 | kafkaScyllaColumnMapper.kafkaRecordField = getFiledForNameFromSchema(record.keySchema(), recordField, "record.keySchema()"); 87 | } 88 | this.tablePartitionKeyMap.put(recordField, kafkaScyllaColumnMapper); 89 | } else if (columnNameMap[1].startsWith("value.")) { 90 | Field valueField = null; 91 | if (record.valueSchema() != null) { 92 | valueField = getFiledForNameFromSchema(record.valueSchema(), recordField, "record.valueSchema()"); 93 | } 94 | if (scyllaColumnName.equals("__ttl")) { 95 | ttlMappedField = recordField; 96 | } else if (scyllaColumnName.equals("__timestamp")) { 97 | timeStampMappedField = recordField; 98 | } else { 99 | kafkaScyllaColumnMapper.kafkaRecordField = valueField; 100 | this.tableColumnMap.put(recordField, kafkaScyllaColumnMapper); 101 | } 102 | } else if (columnNameMap[1].startsWith("header.")) { 103 | int index = 0; 104 | for (Header header : record.headers()) { 105 | if (header.key().equals(recordField)) { 106 | if (header.schema().type().isPrimitive()) { 107 | kafkaScyllaColumnMapper.kafkaRecordField = new Field(header.key(), index, header.schema()); 108 | tableColumnMap.put(recordField, kafkaScyllaColumnMapper); 109 | index++; 110 | } else { 111 | throw new IllegalArgumentException(String.format("Header schema type should be of primitive type. " 112 | + "%s schema type is not allowed in header.", header.schema().type().getName())); 113 | } 114 | } 115 | } 116 | } else { 117 | throw new IllegalArgumentException("field name must start with 'key.', 'value.' or 'header.'."); 118 | } 119 | } 120 | this.isScyllaColumnsMapped = true; 121 | } 122 | 123 | private Field getFiledForNameFromSchema(Schema schema, String name, String schemaType) { 124 | Field schemaField = schema.field(name); 125 | if (null == schemaField) { 126 | throw new DataException( 127 | String.format( 128 | schemaType + " must contain all of key fields mentioned in the " 129 | + "'topic.my_topic.my_ks.my_table.mapping' config. " + schemaType 130 | + "is missing field '%s'. " + schemaType + " is used by the connector " 131 | + "to persist data to the table in ScyllaDb. Here are " 132 | + "the available fields for " + schemaType + "(%s).", 133 | name, 134 | Joiner.on(", ").join( 135 | schema.fields().stream().map(Field::name).collect(Collectors.toList()) 136 | ) 137 | ) 138 | ); 139 | } 140 | return schemaField; 141 | } 142 | 143 | public void setTtlAndTimeStampIfAvailable(SinkRecord record) { 144 | // Timestamps in Kafka (record.timestamp()) are in millisecond precision, 145 | // while Scylla expects a microsecond precision: 1 ms = 1000 us. 146 | this.timeStamp = record.timestamp() * 1000; 147 | if (timeStampMappedField != null) { 148 | Object timeStampValue = getValueOfField(record.value(), timeStampMappedField); 149 | if (timeStampValue instanceof Long) { 150 | this.timeStamp = (Long) timeStampValue; 151 | } else { 152 | throw new DataException( 153 | String.format("TimeStamp should be of type Long. But record provided for %s is of type %s", 154 | timeStampMappedField, timeStampValue.getClass().getName() 155 | )); 156 | } 157 | } 158 | if (ttlMappedField != null) { 159 | Object ttlValue = getValueOfField(record.value(), ttlMappedField); 160 | if (ttlValue instanceof Integer) { 161 | this.ttl = (Integer) ttlValue; 162 | } else { 163 | throw new DataException( 164 | String.format("TTL should be of type Integer. But record provided for %s is of type %s", 165 | ttlMappedField, ttlValue.getClass().getName() 166 | )); 167 | } 168 | } 169 | } 170 | 171 | public Object getValueOfField(Object value, String field) { 172 | Preconditions.checkNotNull(value, "value cannot be null."); 173 | if (value instanceof Struct) { 174 | return ((Struct)value).get(field); 175 | } else { 176 | if (!(value instanceof Map)) { 177 | throw new DataException(String.format("Only Schema (%s) or Schema less (%s) are supported. %s is not a supported type.", Struct.class.getName(), Map.class.getName(), value.getClass().getName())); 178 | } 179 | return ((Map)value).get(field); 180 | } 181 | } 182 | 183 | public Map getTablePartitionKeyMap() { 184 | return tablePartitionKeyMap; 185 | } 186 | 187 | public Map getTableColumnMap() { 188 | return tableColumnMap; 189 | } 190 | 191 | public ConsistencyLevel getConsistencyLevel() { 192 | return consistencyLevel; 193 | } 194 | 195 | public String getTtlMappedField() { 196 | return ttlMappedField; 197 | } 198 | 199 | public Integer getTtl() { 200 | return ttl; 201 | } 202 | 203 | public Long getTimeStamp() { 204 | return timeStamp; 205 | } 206 | 207 | public boolean isScyllaColumnsMapped() { 208 | return isScyllaColumnsMapped; 209 | } 210 | 211 | public void setScyllaColumnsMappedFalse() { 212 | this.isScyllaColumnsMapped = false; 213 | } 214 | 215 | public String getMappingStringForTopic() { 216 | return mappingStringForTopic; 217 | } 218 | 219 | public boolean isDeletesEnabled() { 220 | return deletesEnabled; 221 | } 222 | 223 | public class KafkaScyllaColumnMapper { 224 | private String scyllaColumnName; 225 | private Field kafkaRecordField; 226 | 227 | KafkaScyllaColumnMapper(String scyllaColumnName) { 228 | this.scyllaColumnName = scyllaColumnName; 229 | } 230 | 231 | public String getScyllaColumnName() { 232 | return scyllaColumnName; 233 | } 234 | 235 | public Field getKafkaRecordField() { 236 | return kafkaRecordField; 237 | } 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/utils/ListRecommender.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.utils; 2 | 3 | //import jdk.nashorn.internal.ir.LiteralNode; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | public class ListRecommender implements ConfigDef.Recommender { 10 | private final List validValues; 11 | 12 | public ListRecommender(List validValues){ 13 | this.validValues = validValues; 14 | } 15 | 16 | @Override 17 | public List validValues(String s, Map map) { 18 | return validValues; 19 | } 20 | 21 | @Override 22 | public boolean visible(String s, Map map) { 23 | return true; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/utils/NullOrReadableFile.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.utils; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | 5 | import java.io.File; 6 | 7 | public class NullOrReadableFile implements ConfigDef.Validator { 8 | @Override 9 | public void ensureValid(String s, Object o) { 10 | if(o != null){ 11 | File file = new File(o.toString()); 12 | if(!file.canRead()){ 13 | throw new IllegalArgumentException("Cannot read file '" + o + "' provided through '" + s + "'."); 14 | } 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/utils/ScyllaDbConstants.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.utils; 2 | 3 | public class ScyllaDbConstants { 4 | 5 | public static final String DELETE_OPERATION = "delete"; 6 | 7 | public static final String INSERT_OPERATION = "insert"; 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/utils/VersionUtil.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.utils; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.util.Properties; 9 | 10 | public class VersionUtil { 11 | private static final Logger log = LoggerFactory.getLogger(VersionUtil.class); 12 | private static final Properties VERSION = loadVersion(); 13 | 14 | private static Properties loadVersion() { 15 | // Load the version from version.properties resource file. 16 | InputStream inputStream = VersionUtil.class.getClassLoader().getResourceAsStream("io/connect/scylladb/version.properties"); 17 | Properties props = new Properties(); 18 | try { 19 | props.load(inputStream); 20 | } catch (IOException e) { 21 | log.error("Error loading the connector version"); 22 | } 23 | return props; 24 | } 25 | 26 | public static String getVersion() { 27 | return VERSION.getProperty("version"); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/io/connect/scylladb/utils/VisibleIfEqual.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.utils; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | 5 | import java.util.Collections; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | public class VisibleIfEqual implements ConfigDef.Recommender { 10 | 11 | private final String configName; 12 | private final Object expectedValue; 13 | private final List validValues; 14 | 15 | 16 | public VisibleIfEqual(String configName, Object expectedValue){ 17 | this(configName, expectedValue, Collections.emptyList()); 18 | } 19 | 20 | public VisibleIfEqual(String configName, Object expectedValue, List validValues){ 21 | this.configName = configName; 22 | this.expectedValue = expectedValue; 23 | this.validValues = validValues; 24 | } 25 | 26 | @Override 27 | public List validValues(String s, Map map) { 28 | return validValues; 29 | } 30 | 31 | @Override 32 | public boolean visible(String s, Map map) { 33 | Object value = map.get(configName); 34 | if(value == null) { 35 | return false; 36 | } 37 | return value.equals(expectedValue); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/resources/io/connect/scylladb/version.properties: -------------------------------------------------------------------------------- 1 | version=${project.version} 2 | -------------------------------------------------------------------------------- /src/test/docker/configA/README.md: -------------------------------------------------------------------------------- 1 | This file contains Docker Compose files for the "configA" environment. 2 | Change this as necessary to reflect a configuration for Scylla Database. -------------------------------------------------------------------------------- /src/test/docker/configA/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | my: 3 | image: replace-with-image-name 4 | ports: 5 | - 1234 6 | volumes: 7 | - ./external.conf:/some/config/external.conf -------------------------------------------------------------------------------- /src/test/docker/configA/external.conf: -------------------------------------------------------------------------------- 1 | # This file is used to configure the external system in the Docker container 2 | # See docker-compose.yml -------------------------------------------------------------------------------- /src/test/docker/configB/README.md: -------------------------------------------------------------------------------- 1 | This file contains Docker Compose files for the "configB" environment. 2 | Change this as necessary to reflect a configuration for Scylla Database. -------------------------------------------------------------------------------- /src/test/docker/configB/docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | my: 3 | image: replace-with-image-name 4 | ports: 5 | - 1234 6 | volumes: 7 | - ./external.conf:/some/config/external.conf -------------------------------------------------------------------------------- /src/test/docker/configB/external.conf: -------------------------------------------------------------------------------- 1 | # This file is used to configure the external system in the Docker container 2 | # See docker-compose.yml -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/ClusterAddressTranslatorTest.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import org.junit.Test; 5 | 6 | public class ClusterAddressTranslatorTest { 7 | 8 | @Test 9 | public void setMapTest() throws JsonProcessingException { 10 | ClusterAddressTranslator addressTranslator = new ClusterAddressTranslator(); 11 | final String ADDRESS_MAP_STRING = "{\"10.0.24.69:9042\": \"sl-eu-lon-2-portal.3.dblayer.com:15227\", " 12 | + "\"10.0.24.71:9042\": \"sl-eu-lon-2-portal.2.dblayer.com:15229\", " 13 | + "\"10.0.24.70:9042\": \"sl-eu-lon-2-portal.1.dblayer.com:15228\"}"; // String from CONTACT_POINTS_DOC 14 | addressTranslator.setMap(ADDRESS_MAP_STRING); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/ScyllaDbSinkConnectorConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | import static org.junit.Assert.assertNotNull; 11 | 12 | public class ScyllaDbSinkConnectorConfigTest { 13 | 14 | Map settings; 15 | ScyllaDbSinkConnectorConfig config; 16 | 17 | @Before 18 | public void before() { 19 | settings = new HashMap<>(); 20 | settings.put(ScyllaDbSinkConnectorConfig.KEYSPACE_CONFIG, "scylladb"); 21 | config = null; 22 | } 23 | 24 | @Test 25 | public void shouldAcceptValidConfig() { 26 | settings.put(ScyllaDbSinkConnectorConfig.PORT_CONFIG, "9042"); 27 | config = new ScyllaDbSinkConnectorConfig(settings); 28 | assertNotNull(config); 29 | } 30 | 31 | @Test 32 | public void shouldUseDefaults() { 33 | config = new ScyllaDbSinkConnectorConfig(settings); 34 | assertEquals(true, config.keyspaceCreateEnabled); 35 | } 36 | 37 | //TODO: Add more tests 38 | } -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/ScyllaDbSinkConnectorTest.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | import static org.junit.Assert.assertFalse; 12 | import static org.junit.Assert.assertNotNull; 13 | import static org.junit.Assert.assertTrue; 14 | import static org.mockito.Mockito.mock; 15 | import static org.mockito.Mockito.verify; 16 | 17 | public class ScyllaDbSinkConnectorTest { 18 | 19 | Map settings; 20 | ScyllaDbSinkConnector connector; 21 | 22 | @Before 23 | public void before() { 24 | settings = new HashMap<>(); 25 | connector = new ScyllaDbSinkConnector(); 26 | //adding required configurations 27 | settings.put(ScyllaDbSinkConnectorConfig.KEYSPACE_CONFIG, "scylladb"); 28 | } 29 | //TODO: failing need to check 30 | //@Test 31 | public void shouldReturnNonNullVersion() { 32 | System.out.println(connector.version()); 33 | assertNotNull(connector.version()); 34 | } 35 | 36 | @Test 37 | public void shouldStartWithoutError() { 38 | startConnector(); 39 | } 40 | 41 | @Test 42 | public void shouldReturnSinkTask() { 43 | assertEquals(ScyllaDbSinkTask.class, connector.taskClass()); 44 | } 45 | 46 | @Test 47 | public void shouldGenerateValidTaskConfigs() { 48 | startConnector(); 49 | //TODO: Change this logic to reflect expected behavior of your connector 50 | List> taskConfigs = connector.taskConfigs(1); 51 | assertTrue("zero task configs provided", !taskConfigs.isEmpty()); 52 | for (Map taskConfig : taskConfigs) { 53 | assertEquals(settings, taskConfig); 54 | } 55 | } 56 | 57 | @Test 58 | public void shouldStartAndStop() { 59 | startConnector(); 60 | connector.stop(); 61 | } 62 | 63 | @Test 64 | public void shouldNotHaveNullConfigDef() { 65 | // ConfigDef objects don't have an overridden equals() method; just make sure it's non-null 66 | assertNotNull(connector.config()); 67 | } 68 | 69 | //TODO: failing need to check 70 | //@Test 71 | public void version() { 72 | assertNotNull(connector.version()); 73 | assertFalse(connector.version().equals("0.0.0.0")); 74 | assertTrue(connector.version().matches("^(\\d+\\.)?(\\d+\\.)?(\\*|\\d+)(-\\w+)?$")); 75 | } 76 | 77 | 78 | protected void startConnector() { 79 | connector.config = new ScyllaDbSinkConnectorConfig(settings); 80 | //connector.doStart(); 81 | } 82 | } -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/ScyllaDbSinkTaskTest.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb; 2 | 3 | import org.apache.kafka.connect.errors.ConnectException; 4 | import org.apache.kafka.connect.data.Schema; 5 | import org.apache.kafka.connect.sink.SinkRecord; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | import java.util.Collections; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import static org.junit.Assert.assertFalse; 14 | import static org.junit.Assert.assertNotNull; 15 | import static org.junit.Assert.assertTrue; 16 | 17 | public class ScyllaDbSinkTaskTest { 18 | 19 | Map settings; 20 | ScyllaDbSinkTask task; 21 | 22 | @Before 23 | public void before() { 24 | settings = new HashMap<>(); 25 | task = new ScyllaDbSinkTask(); 26 | } 27 | 28 | static final String KAFKA_TOPIC = "topic"; 29 | 30 | //TODO: failing need to check 31 | //@Test 32 | public void shouldReturnNonNullVersion() { 33 | assertNotNull(task.version()); 34 | } 35 | 36 | @Test 37 | public void shouldStopAndDisconnect() { 38 | task.stop(); 39 | //TODO: Ensure the task stopped 40 | } 41 | 42 | //TODO: failing need to check 43 | //@Test(expected = ConnectException.class) 44 | public void shouldFailWithInvalidRecord() { 45 | SinkRecord record = new SinkRecord( 46 | KAFKA_TOPIC, 47 | 1, 48 | Schema.STRING_SCHEMA, 49 | "Sample key", 50 | Schema.STRING_SCHEMA, 51 | "Sample value", 52 | 1L 53 | ); 54 | 55 | // Ensure that the exception is translated into a ConnectException 56 | task.put(Collections.singleton(record)); 57 | } 58 | 59 | //TODO: failing need to check 60 | //@Test 61 | public void version() { 62 | assertNotNull(task.version()); 63 | assertFalse(task.version().equals("0.0.0.0")); 64 | assertTrue(task.version().matches("^(\\d+\\.)?(\\d+\\.)?(\\*|\\d+)(-\\w+)?$")); 65 | } 66 | } -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/integration/RowValidator.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.integration; 2 | 3 | import com.datastax.oss.driver.api.core.CqlIdentifier; 4 | import com.datastax.oss.driver.api.querybuilder.QueryBuilder; 5 | import com.datastax.oss.driver.api.querybuilder.relation.Relation; 6 | import com.datastax.oss.driver.api.querybuilder.select.Select; 7 | import com.datastax.oss.driver.api.querybuilder.select.SelectFrom; 8 | import com.google.common.base.Preconditions; 9 | import org.apache.kafka.connect.data.Struct; 10 | import org.apache.kafka.connect.sink.SinkRecord; 11 | 12 | import java.util.Map; 13 | 14 | import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; 15 | import static io.connect.scylladb.integration.TestDataUtil.asMap; 16 | 17 | class RowValidator { 18 | final String table; 19 | final Map key; 20 | final Map value; 21 | final boolean rowExists; 22 | 23 | 24 | RowValidator(String table, Map key, Map value) { 25 | Preconditions.checkNotNull(key); 26 | Preconditions.checkState(!key.isEmpty()); 27 | this.table = table; 28 | this.key = key; 29 | this.value = value; 30 | this.rowExists = null != this.value; 31 | } 32 | 33 | public static RowValidator of(String table, Map key, Map value) { 34 | return new RowValidator(table, key, value); 35 | } 36 | 37 | public static RowValidator of(String table, Struct keyStruct, Struct valueStruct) { 38 | Map key = asMap(keyStruct); 39 | Map value = asMap(valueStruct); 40 | return new RowValidator(table, key, value); 41 | } 42 | 43 | public static Map toMapChecked(Object o) { 44 | Map result; 45 | if (o instanceof Map) { 46 | result = (Map) o; 47 | } else if (o instanceof Struct) { 48 | result = asMap((Struct) o); 49 | } else if (null == o) { 50 | result = null; 51 | } else { 52 | throw new UnsupportedOperationException("Must be a struct or map"); 53 | } 54 | return result; 55 | } 56 | 57 | public static RowValidator of(SinkRecord record) { 58 | Map key = toMapChecked(record.key()); 59 | Map value = toMapChecked(record.value()); 60 | return new RowValidator(record.topic(), key, value); 61 | } 62 | 63 | 64 | @Override 65 | public String toString() { 66 | Select select = QueryBuilder.selectFrom(table).all(); 67 | for (Map.Entry e : key.entrySet()) { 68 | select = select.whereColumn(CqlIdentifier.fromInternal(e.getKey())).isEqualTo(literal(e.getValue())); 69 | } 70 | 71 | return select.asCql(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/integration/SinkRecordUtil.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.integration; 2 | 3 | import com.google.common.base.Preconditions; 4 | import org.apache.kafka.common.record.TimestampType; 5 | import org.apache.kafka.connect.data.Schema; 6 | import org.apache.kafka.connect.data.SchemaAndValue; 7 | import org.apache.kafka.connect.data.Struct; 8 | import org.apache.kafka.connect.errors.DataException; 9 | import org.apache.kafka.connect.sink.SinkRecord; 10 | 11 | public class SinkRecordUtil { 12 | 13 | public static final int PARTITION = 1; 14 | public static final long OFFSET = 91283741L; 15 | public static final long TIMESTAMP = 1530286549123L; 16 | 17 | public static SinkRecord delete(String topic, Struct key) { 18 | Preconditions.checkNotNull(key, "key cannot be null."); 19 | return delete(topic, new SchemaAndValue(key.schema(), key)); 20 | } 21 | 22 | public static SinkRecord delete(String topic, Schema keySchema, Object key) { 23 | return delete(topic, new SchemaAndValue(keySchema, key)); 24 | } 25 | 26 | public static SinkRecord delete(String topic, SchemaAndValue key) { 27 | Preconditions.checkNotNull(topic, "topic cannot be null"); 28 | if (null == key) { 29 | throw new DataException("key cannot be null."); 30 | } 31 | if (null == key.value()) { 32 | throw new DataException("key values cannot be null."); 33 | } 34 | 35 | return new SinkRecord( 36 | topic, 37 | PARTITION, 38 | key.schema(), 39 | key.value(), 40 | null, 41 | null, 42 | OFFSET, 43 | TIMESTAMP, 44 | TimestampType.CREATE_TIME 45 | ); 46 | } 47 | 48 | public static SinkRecord write(String topic, Struct key, Struct value) { 49 | return write( 50 | topic, 51 | new SchemaAndValue(key.schema(), key), 52 | new SchemaAndValue(value.schema(), value) 53 | ); 54 | } 55 | 56 | public static SinkRecord write(String topic, Schema keySchema, Object key, Schema valueSchema, Object value) { 57 | return write( 58 | topic, 59 | new SchemaAndValue(keySchema, key), 60 | new SchemaAndValue(valueSchema, value) 61 | ); 62 | } 63 | 64 | public static SinkRecord write(String topic, SchemaAndValue key, SchemaAndValue value) { 65 | Preconditions.checkNotNull(topic, "topic cannot be null"); 66 | Preconditions.checkNotNull(key, "key cannot be null."); 67 | Preconditions.checkNotNull(key.value(), "key values cannot be null."); 68 | Preconditions.checkNotNull(value, "value cannot be null."); 69 | Preconditions.checkNotNull(value.value(), "value values cannot be null."); 70 | 71 | return new SinkRecord( 72 | topic, 73 | PARTITION, 74 | key.schema(), 75 | key.value(), 76 | value.schema(), 77 | value.value(), 78 | OFFSET, 79 | TIMESTAMP, 80 | TimestampType.CREATE_TIME 81 | ); 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/integration/TestDataUtil.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.integration; 2 | 3 | import com.google.common.base.Preconditions; 4 | import com.google.common.base.Strings; 5 | import org.apache.kafka.connect.data.*; 6 | 7 | import java.util.*; 8 | 9 | public class TestDataUtil { 10 | private TestDataUtil() { 11 | 12 | } 13 | 14 | public static SchemaAndValue asSchemaAndValue(Struct struct) { 15 | Preconditions.checkNotNull(struct, "struct cannot be null."); 16 | return new SchemaAndValue(struct.schema(), struct); 17 | } 18 | 19 | public static Map asMap(Struct struct) { 20 | Preconditions.checkNotNull(struct, "struct cannot be null."); 21 | Map result = new LinkedHashMap<>(struct.schema().fields().size()); 22 | 23 | for (Field field : struct.schema().fields()) { 24 | final Object value; 25 | if (Schema.Type.STRUCT == field.schema().type()) { 26 | Struct s = struct.getStruct(field.name()); 27 | value = asMap(s); 28 | } else { 29 | value = struct.get(field); 30 | } 31 | result.put(field.name(), value); 32 | } 33 | 34 | return result; 35 | } 36 | 37 | 38 | static class FieldState { 39 | final String name; 40 | final Schema.Type type; 41 | final boolean isOptional; 42 | final Object value; 43 | 44 | private FieldState(String name, Schema.Type type, boolean isOptional, Object value) { 45 | this.name = name; 46 | this.type = type; 47 | this.isOptional = isOptional; 48 | this.value = value; 49 | } 50 | 51 | static FieldState of(String name, Schema.Type type, boolean isOptional, Object value) { 52 | return new FieldState(name, type, isOptional, value); 53 | } 54 | 55 | } 56 | 57 | private static Struct struct(String name, List fields) { 58 | final SchemaBuilder builder = SchemaBuilder.struct(); 59 | if (!Strings.isNullOrEmpty(name)) { 60 | builder.name(name); 61 | } 62 | 63 | for (FieldState field : fields) { 64 | final Schema schema; 65 | if (Schema.Type.STRUCT == field.type) { 66 | Preconditions.checkNotNull( 67 | field.value, 68 | "%s field.value cannot be null. Struct is needed to infer schema of nested struct." 69 | ); 70 | Struct struct = (Struct) field.value; 71 | schema = struct.schema(); 72 | } else { 73 | SchemaBuilder fieldBuilder = SchemaBuilder.type(field.type); 74 | if (field.isOptional) { 75 | fieldBuilder.optional(); 76 | } 77 | schema = fieldBuilder.build(); 78 | } 79 | builder.field(field.name, schema); 80 | } 81 | 82 | final Schema schema = builder.build(); 83 | final Struct struct = new Struct(schema); 84 | 85 | for (FieldState field : fields) { 86 | struct.put(field.name, field.value); 87 | } 88 | 89 | struct.validate(); 90 | return struct; 91 | } 92 | 93 | public static Struct struct( 94 | String name, 95 | String f1, 96 | Schema.Type t1, 97 | boolean o1, 98 | Object v1 99 | ) { 100 | return struct( 101 | name, 102 | Collections.singletonList( 103 | FieldState.of(f1, t1, o1, v1) 104 | ) 105 | ); 106 | } 107 | 108 | public static Struct struct( 109 | String name, 110 | String f1, 111 | Schema.Type t1, 112 | boolean o1, 113 | Object v1, 114 | String f2, 115 | Schema.Type t2, 116 | boolean o2, 117 | Object v2 118 | ) { 119 | return struct( 120 | name, 121 | Arrays.asList( 122 | FieldState.of(f1, t1, o1, v1), 123 | FieldState.of(f2, t2, o2, v2) 124 | ) 125 | ); 126 | } 127 | 128 | public static Struct struct( 129 | String name, 130 | String f1, 131 | Schema.Type t1, 132 | boolean o1, 133 | Object v1, 134 | String f2, 135 | Schema.Type t2, 136 | boolean o2, 137 | Object v2, 138 | String f3, 139 | Schema.Type t3, 140 | boolean o3, 141 | Object v3 142 | ) { 143 | return struct( 144 | name, 145 | Arrays.asList( 146 | FieldState.of(f1, t1, o1, v1), 147 | FieldState.of(f2, t2, o2, v2), 148 | FieldState.of(f3, t3, o3, v3) 149 | ) 150 | ); 151 | } 152 | 153 | public static Struct struct( 154 | String name, 155 | String f1, 156 | Schema.Type t1, 157 | boolean o1, 158 | Object v1, 159 | String f2, 160 | Schema.Type t2, 161 | boolean o2, 162 | Object v2, 163 | String f3, 164 | Schema.Type t3, 165 | boolean o3, 166 | Object v3, 167 | String f4, 168 | Schema.Type t4, 169 | boolean o4, 170 | Object v4 171 | ) { 172 | return struct( 173 | name, 174 | Arrays.asList( 175 | FieldState.of(f1, t1, o1, v1), 176 | FieldState.of(f2, t2, o2, v2), 177 | FieldState.of(f3, t3, o3, v3), 178 | FieldState.of(f4, t4, o4, v4) 179 | ) 180 | ); 181 | } 182 | 183 | public static Struct struct( 184 | String name, 185 | String f1, 186 | Schema.Type t1, 187 | boolean o1, 188 | Object v1, 189 | String f2, 190 | Schema.Type t2, 191 | boolean o2, 192 | Object v2, 193 | String f3, 194 | Schema.Type t3, 195 | boolean o3, 196 | Object v3, 197 | String f4, 198 | Schema.Type t4, 199 | boolean o4, 200 | Object v4, 201 | String f5, 202 | Schema.Type t5, 203 | boolean o5, 204 | Object v5 205 | ) { 206 | return struct( 207 | name, 208 | Arrays.asList( 209 | FieldState.of(f1, t1, o1, v1), 210 | FieldState.of(f2, t2, o2, v2), 211 | FieldState.of(f3, t3, o3, v3), 212 | FieldState.of(f4, t4, o4, v4), 213 | FieldState.of(f5, t5, o5, v5) 214 | ) 215 | ); 216 | } 217 | 218 | public static Struct struct( 219 | String name, 220 | String f1, 221 | Schema.Type t1, 222 | boolean o1, 223 | Object v1, 224 | String f2, 225 | Schema.Type t2, 226 | boolean o2, 227 | Object v2, 228 | String f3, 229 | Schema.Type t3, 230 | boolean o3, 231 | Object v3, 232 | String f4, 233 | Schema.Type t4, 234 | boolean o4, 235 | Object v4, 236 | String f5, 237 | Schema.Type t5, 238 | boolean o5, 239 | Object v5, 240 | String f6, 241 | Schema.Type t6, 242 | boolean o6, 243 | Object v6 244 | ) { 245 | return struct( 246 | name, 247 | Arrays.asList( 248 | FieldState.of(f1, t1, o1, v1), 249 | FieldState.of(f2, t2, o2, v2), 250 | FieldState.of(f3, t3, o3, v3), 251 | FieldState.of(f4, t4, o4, v4), 252 | FieldState.of(f5, t5, o5, v5), 253 | FieldState.of(f6, t6, o6, v6) 254 | ) 255 | ); 256 | } 257 | 258 | public static Struct struct( 259 | String name, 260 | String f1, 261 | Schema.Type t1, 262 | boolean o1, 263 | Object v1, 264 | String f2, 265 | Schema.Type t2, 266 | boolean o2, 267 | Object v2, 268 | String f3, 269 | Schema.Type t3, 270 | boolean o3, 271 | Object v3, 272 | String f4, 273 | Schema.Type t4, 274 | boolean o4, 275 | Object v4, 276 | String f5, 277 | Schema.Type t5, 278 | boolean o5, 279 | Object v5, 280 | String f6, 281 | Schema.Type t6, 282 | boolean o6, 283 | Object v6, 284 | String f7, 285 | Schema.Type t7, 286 | boolean o7, 287 | Object v7 288 | ) { 289 | return struct( 290 | name, 291 | Arrays.asList( 292 | FieldState.of(f1, t1, o1, v1), 293 | FieldState.of(f2, t2, o2, v2), 294 | FieldState.of(f3, t3, o3, v3), 295 | FieldState.of(f4, t4, o4, v4), 296 | FieldState.of(f5, t5, o5, v5), 297 | FieldState.of(f6, t6, o6, v6), 298 | FieldState.of(f7, t7, o7, v7) 299 | ) 300 | ); 301 | } 302 | 303 | public static Struct struct( 304 | String name, 305 | String f1, 306 | Schema.Type t1, 307 | boolean o1, 308 | Object v1, 309 | String f2, 310 | Schema.Type t2, 311 | boolean o2, 312 | Object v2, 313 | String f3, 314 | Schema.Type t3, 315 | boolean o3, 316 | Object v3, 317 | String f4, 318 | Schema.Type t4, 319 | boolean o4, 320 | Object v4, 321 | String f5, 322 | Schema.Type t5, 323 | boolean o5, 324 | Object v5, 325 | String f6, 326 | Schema.Type t6, 327 | boolean o6, 328 | Object v6, 329 | String f7, 330 | Schema.Type t7, 331 | boolean o7, 332 | Object v7, 333 | String f8, 334 | Schema.Type t8, 335 | boolean o8, 336 | Object v8 337 | ) { 338 | return struct( 339 | name, 340 | Arrays.asList( 341 | FieldState.of(f1, t1, o1, v1), 342 | FieldState.of(f2, t2, o2, v2), 343 | FieldState.of(f3, t3, o3, v3), 344 | FieldState.of(f4, t4, o4, v4), 345 | FieldState.of(f5, t5, o5, v5), 346 | FieldState.of(f6, t6, o6, v6), 347 | FieldState.of(f7, t7, o7, v7), 348 | FieldState.of(f8, t8, o8, v8) 349 | ) 350 | ); 351 | } 352 | 353 | public static Struct struct( 354 | String name, 355 | String f1, 356 | Schema.Type t1, 357 | boolean o1, 358 | Object v1, 359 | String f2, 360 | Schema.Type t2, 361 | boolean o2, 362 | Object v2, 363 | String f3, 364 | Schema.Type t3, 365 | boolean o3, 366 | Object v3, 367 | String f4, 368 | Schema.Type t4, 369 | boolean o4, 370 | Object v4, 371 | String f5, 372 | Schema.Type t5, 373 | boolean o5, 374 | Object v5, 375 | String f6, 376 | Schema.Type t6, 377 | boolean o6, 378 | Object v6, 379 | String f7, 380 | Schema.Type t7, 381 | boolean o7, 382 | Object v7, 383 | String f8, 384 | Schema.Type t8, 385 | boolean o8, 386 | Object v8, 387 | String f9, 388 | Schema.Type t9, 389 | boolean o9, 390 | Object v9 391 | ) { 392 | return struct( 393 | name, 394 | Arrays.asList( 395 | FieldState.of(f1, t1, o1, v1), 396 | FieldState.of(f2, t2, o2, v2), 397 | FieldState.of(f3, t3, o3, v3), 398 | FieldState.of(f4, t4, o4, v4), 399 | FieldState.of(f5, t5, o5, v5), 400 | FieldState.of(f6, t6, o6, v6), 401 | FieldState.of(f7, t7, o7, v7), 402 | FieldState.of(f8, t8, o8, v8), 403 | FieldState.of(f9, t9, o9, v9) 404 | ) 405 | ); 406 | } 407 | 408 | public static Struct struct( 409 | String name, 410 | String f1, 411 | Schema.Type t1, 412 | boolean o1, 413 | Object v1, 414 | String f2, 415 | Schema.Type t2, 416 | boolean o2, 417 | Object v2, 418 | String f3, 419 | Schema.Type t3, 420 | boolean o3, 421 | Object v3, 422 | String f4, 423 | Schema.Type t4, 424 | boolean o4, 425 | Object v4, 426 | String f5, 427 | Schema.Type t5, 428 | boolean o5, 429 | Object v5, 430 | String f6, 431 | Schema.Type t6, 432 | boolean o6, 433 | Object v6, 434 | String f7, 435 | Schema.Type t7, 436 | boolean o7, 437 | Object v7, 438 | String f8, 439 | Schema.Type t8, 440 | boolean o8, 441 | Object v8, 442 | String f9, 443 | Schema.Type t9, 444 | boolean o9, 445 | Object v9, 446 | String f10, 447 | Schema.Type t10, 448 | boolean o10, 449 | Object v10 450 | ) { 451 | return struct( 452 | name, 453 | Arrays.asList( 454 | FieldState.of(f1, t1, o1, v1), 455 | FieldState.of(f2, t2, o2, v2), 456 | FieldState.of(f3, t3, o3, v3), 457 | FieldState.of(f4, t4, o4, v4), 458 | FieldState.of(f5, t5, o5, v5), 459 | FieldState.of(f6, t6, o6, v6), 460 | FieldState.of(f7, t7, o7, v7), 461 | FieldState.of(f8, t8, o8, v8), 462 | FieldState.of(f9, t9, o9, v9), 463 | FieldState.of(f10, t10, o10, v10) 464 | ) 465 | ); 466 | } 467 | } 468 | -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/integration/codec/StringTimeUuidCodecTest.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.integration.codec; 2 | 3 | import com.datastax.oss.driver.api.core.ProtocolVersion; 4 | import io.connect.scylladb.codec.StringTimeUuidCodec; 5 | import org.junit.jupiter.api.AfterEach; 6 | import org.junit.jupiter.api.Assertions; 7 | import org.junit.jupiter.api.BeforeEach; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.nio.ByteBuffer; 11 | import java.util.UUID; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertEquals; 14 | 15 | class StringTimeUuidCodecTest { 16 | 17 | private static final String NON_UUID_STR = "some-non-uuid-string"; 18 | 19 | private static final UUID UUID_V1 = UUID.fromString("5fc03087-d265-11e7-b8c6-83e29cd24f4c"); 20 | private static final UUID UUID_V4 = UUID.randomUUID(); 21 | private static final String UUID_STR1 = UUID_V1.toString(); 22 | private static final String UUID_STR2 = UUID_V4.toString(); 23 | 24 | private StringTimeUuidCodec codec; 25 | 26 | @BeforeEach 27 | void setUp() { 28 | codec = new StringTimeUuidCodec(); 29 | } 30 | 31 | @AfterEach 32 | void tearDown() { 33 | } 34 | 35 | @Test 36 | public void shouldBeUuidType1() { 37 | assertEquals(1, UUID_V1.version()); 38 | } 39 | 40 | @Test 41 | public void shouldFormatNullString() { 42 | assertEquals("NULL", codec.format(null)); 43 | } 44 | 45 | @Test 46 | public void shouldFormatEmptyString() { 47 | assertEquals("", codec.format("")); 48 | } 49 | 50 | @Test 51 | public void shouldFormatUuidString() { 52 | assertEquals(UUID_STR1, codec.format(UUID_STR1)); 53 | assertEquals(UUID_STR2, codec.format(UUID_STR2)); 54 | } 55 | 56 | @Test 57 | public void shouldFormatNonUuidString() { 58 | assertEquals(NON_UUID_STR, codec.format(NON_UUID_STR)); 59 | } 60 | 61 | @Test 62 | public void shouldSerializeAndDeserializeNullString() { 63 | assertSerializeAndDeserialize(null); 64 | } 65 | 66 | @Test 67 | public void shouldSerializeAndDeserializeUuidType1Strings() { 68 | assertSerializeAndDeserialize(UUID_STR1); 69 | } 70 | 71 | @Test 72 | public void shouldFailToSerializeNonType1Strings() { 73 | Assertions.assertThrows(IllegalArgumentException.class, () -> { 74 | assertSerializeAndDeserialize(UUID_STR2); 75 | }); 76 | } 77 | 78 | @Test 79 | public void shouldFailToSerializeNonUuidString() { 80 | Assertions.assertThrows(IllegalArgumentException.class, () -> { 81 | codec.encode(NON_UUID_STR, ProtocolVersion.V4); 82 | }); 83 | } 84 | 85 | protected void assertSerializeAndDeserialize(String uuidStr) { 86 | ByteBuffer buffer = codec.encode(uuidStr, ProtocolVersion.V4); 87 | String deserialized = codec.decode(buffer, ProtocolVersion.V4); 88 | assertEquals(uuidStr, deserialized); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/io/connect/scylladb/integration/codec/StringUuidCodecTest.java: -------------------------------------------------------------------------------- 1 | package io.connect.scylladb.integration.codec; 2 | 3 | import com.datastax.oss.driver.api.core.ProtocolVersion; 4 | import io.connect.scylladb.codec.StringUuidCodec; 5 | import org.junit.jupiter.api.AfterEach; 6 | import org.junit.jupiter.api.Assertions; 7 | import org.junit.jupiter.api.BeforeEach; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.nio.ByteBuffer; 11 | import java.util.UUID; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertEquals; 14 | 15 | class StringUuidCodecTest { 16 | 17 | private static final String NON_UUID_STR = "some-non-uuid-string"; 18 | 19 | private static final String UUID_STR1 = UUID.randomUUID().toString(); 20 | private static final String UUID_STR2 = UUID.randomUUID().toString(); 21 | private static final String UUID_STR3 = UUID.randomUUID().toString(); 22 | 23 | private StringUuidCodec codec; 24 | 25 | @BeforeEach 26 | void setUp() { 27 | codec = new StringUuidCodec(); 28 | } 29 | 30 | @AfterEach 31 | void tearDown() { 32 | } 33 | 34 | @Test 35 | public void shouldFormatNullString() { 36 | assertEquals("NULL", codec.format(null)); 37 | } 38 | 39 | @Test 40 | public void shouldFormatEmptyString() { 41 | assertEquals("", codec.format("")); 42 | } 43 | 44 | @Test 45 | public void shouldFormatUuidString() { 46 | assertEquals(UUID_STR1, codec.format(UUID_STR1)); 47 | assertEquals(UUID_STR2, codec.format(UUID_STR2)); 48 | assertEquals(UUID_STR3, codec.format(UUID_STR3)); 49 | } 50 | 51 | @Test 52 | public void shouldFormatNonUuidString() { 53 | assertEquals(NON_UUID_STR, codec.format(NON_UUID_STR)); 54 | } 55 | 56 | @Test 57 | public void shouldSerializeAndDeserializeNullString() { 58 | assertSerializeAndDeserialize(null); 59 | } 60 | 61 | @Test 62 | public void shouldSerializeAndDeserializeUuidStrings() { 63 | assertSerializeAndDeserialize(UUID_STR1); 64 | assertSerializeAndDeserialize(UUID_STR2); 65 | assertSerializeAndDeserialize(UUID_STR3); 66 | } 67 | 68 | @Test 69 | public void shouldFailToSerializeNonUuidString() { 70 | Assertions.assertThrows(IllegalArgumentException.class, () -> { 71 | codec.encode(NON_UUID_STR, ProtocolVersion.V4); 72 | }); 73 | } 74 | 75 | protected void assertSerializeAndDeserialize(String uuidStr) { 76 | ByteBuffer buffer = codec.encode(uuidStr, ProtocolVersion.V4); 77 | String deserialized = codec.decode(buffer, ProtocolVersion.V4); 78 | assertEquals(uuidStr, deserialized); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | 2 | log4j.rootLogger=ERROR, stdout 3 | 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=[%d] (%t) %p %m (%c:%L)%n 7 | 8 | log4j.logger.org.apache.zookeeper=ERROR 9 | log4j.logger.org.I0Itec.zkclient=ERROR 10 | log4j.logger.org.reflections=ERROR 11 | log4j.logger.org.eclipse.jetty=ERROR 12 | 13 | #log4j.logger.io.confluent.connect.scylladb=ERROR 14 | --------------------------------------------------------------------------------