├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README-JP.md ├── README.md ├── common └── cassandra │ └── conf-dir │ └── resources │ └── cassandra │ └── conf │ ├── cassandra-rackdc.properties │ └── cassandra.yaml ├── conf └── dynamo-cassandra-proxy.yaml.template ├── docker-compose.yaml ├── docs ├── Summary.md ├── Translators-JP.md ├── Translators.md └── images │ ├── 1.png │ ├── 2.png │ └── gke.png ├── gke └── proxy-suite.yaml ├── k8s-local └── proxy-suite.yaml ├── migrate ├── .idea │ ├── compiler.xml │ ├── encodings.xml │ ├── hydra.xml │ ├── misc.xml │ ├── modules.xml │ ├── sbt.xml │ ├── scala_compiler.xml │ └── workspace.xml ├── README-JP.md ├── README.md ├── pom.xml ├── pom.xml.dse ├── rundump.sh ├── runit.sh ├── runit.sh.oss └── src │ └── main │ ├── resources │ └── create_schema.cql │ └── scala │ └── com │ └── datastax │ └── powertools │ └── DynamoReader.scala ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── datastax │ │ └── powertools │ │ └── dcp │ │ ├── DCProxyApplication.java │ │ ├── DCProxyConfiguration.java │ │ ├── DynamoDSETranslator.java │ │ ├── DynamoDSETranslatorJSONBlob.java │ │ ├── TranslatorType.java │ │ ├── api │ │ ├── DynamoDBResponse.java │ │ └── DynamoStatementType.java │ │ ├── managed │ │ ├── ddbstreams │ │ │ ├── DynamoStreamsManager.java │ │ │ ├── StreamsRecordProcessor.java │ │ │ └── StreamsRecordProcessorFactory.java │ │ ├── dse │ │ │ ├── CassandraManager.java │ │ │ ├── CassandraStatements.java │ │ │ ├── DockerHelper.java │ │ │ └── TableDef.java │ │ └── dynamodb │ │ │ └── DynamoManager.java │ │ └── resources │ │ └── DCProxyResource.java └── resources │ └── application.conf └── test └── java └── com └── datastax └── powertools └── dcp ├── AbstractDCPTest.java ├── CRUDTest.java ├── DynamoDSETranslatorJSONBlobTest.java ├── example ├── CatalogIntegrationTest.java └── CatalogItem.java └── resources └── DynamoDBResourceTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | conf/dynamo-cassandra-proxy.yaml 2 | logs/ 3 | 4 | .idea 5 | 6 | # Created by https://www.gitignore.io/api/java,osx,emacs,intellij,maven 7 | 8 | ### Java ### 9 | *.class 10 | 11 | # Mobile Tools for Java (J2ME) 12 | .mtj.tmp/ 13 | 14 | # Package Files # 15 | *.jar 16 | *.war 17 | *.ear 18 | 19 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 20 | hs_err_pid* 21 | 22 | 23 | ### OSX ### 24 | .DS_Store 25 | .AppleDouble 26 | .LSOverride 27 | 28 | # Icon must end with two \r 29 | Icon 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | 49 | 50 | ### Emacs ### 51 | # -*- mode: gitignore; -*- 52 | *~ 53 | \#*\# 54 | /.emacs.desktop 55 | /.emacs.desktop.lock 56 | *.elc 57 | auto-save-list 58 | tramp 59 | .\#* 60 | 61 | # Org-mode 62 | .org-id-locations 63 | *_archive 64 | 65 | # flymake-mode 66 | *_flymake.* 67 | 68 | # eshell files 69 | /eshell/history 70 | /eshell/lastdir 71 | 72 | # elpa packages 73 | /elpa/ 74 | 75 | # reftex files 76 | *.rel 77 | 78 | # AUCTeX auto folder 79 | /auto/ 80 | 81 | # cask packages 82 | .cask/ 83 | dist/ 84 | 85 | # Flycheck 86 | flycheck_*.el 87 | 88 | # server auth directory 89 | /server/ 90 | 91 | # projectiles files 92 | .projectile 93 | 94 | ### Intellij ### 95 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 96 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 97 | 98 | # User-specific stuff: 99 | .idea/workspace.xml 100 | .idea/tasks.xml 101 | .idea/dictionaries 102 | .idea/vcs.xml 103 | .idea/jsLibraryMappings.xml 104 | 105 | # Sensitive or high-churn files: 106 | .idea/dataSources.ids 107 | .idea/dataSources.xml 108 | .idea/dataSources.local.xml 109 | .idea/sqlDataSources.xml 110 | .idea/dynamic.xml 111 | .idea/uiDesigner.xml 112 | 113 | # Gradle: 114 | .idea/gradle.xml 115 | .idea/libraries 116 | 117 | # Mongo Explorer plugin: 118 | .idea/mongoSettings.xml 119 | 120 | ## File-based project format: 121 | *.iws 122 | 123 | ## Plugin-specific files: 124 | 125 | # IntelliJ 126 | /out/ 127 | 128 | # mpeltonen/sbt-idea plugin 129 | .idea_modules/ 130 | 131 | # JIRA plugin 132 | atlassian-ide-plugin.xml 133 | 134 | # Crashlytics plugin (for Android Studio and IntelliJ) 135 | com_crashlytics_export_strings.xml 136 | crashlytics.properties 137 | crashlytics-build.properties 138 | fabric.properties 139 | 140 | ### Intellij Patch ### 141 | *.iml 142 | 143 | 144 | ### Maven ### 145 | target/ 146 | pom.xml.tag 147 | pom.xml.releaseBackup 148 | pom.xml.versionsBackup 149 | pom.xml.next 150 | release.properties 151 | dependency-reduced-pom.xml 152 | buildNumber.properties 153 | .mvn/timing.properties 154 | 155 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk11 4 | services: 5 | - docker 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11 2 | 3 | COPY target/dynamodb-cassandra-proxy-0.1.0.jar /opt/dynamo-cassandra-proxy/dynamodb-cassandra-proxy-0.1.0.jar 4 | COPY conf/dynamo-cassandra-proxy.yaml.template /opt/dynamo-cassandra-proxy/dynamo-cassandra-proxy.yaml 5 | 6 | CMD java -jar /opt/dynamo-cassandra-proxy/dynamodb-cassandra-proxy-0.1.0.jar server /opt/dynamo-cassandra-proxy/dynamo-cassandra-proxy.yaml 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README-JP.md: -------------------------------------------------------------------------------- 1 | # dynamo-cassandra-proxy 2 | 3 | `dynamo-cassandra-proxy` は、アプリケーションと Apache Cassandra の間に座る、スケーラブルなプロキシレイヤーとして構成されます。 4 | 5 | このプロキシレイヤーは、DynamoDB SDK との互換性を提供します。そうすることで、既存の DynamoDB アプリケーションに変更を加えることなく Cassandra を対象としたデータの読み書きが可能になります。 6 | 7 | また、DynamoDB Streams 経由で、DynamoDB のテーブルを Cassandra 側と同期させる機能もサポートします。 8 | 9 | ## 構成設定 10 | 11 | 以下のように、テンプレートに基づいて yaml を作成します。 12 | 13 | cp conf/dynamo-cassandra-proxy.yaml.template conf/dynamo-cassandra-proxy.yaml 14 | 15 | 以下は、このプロキシがサポートするオプションです。 16 | 17 | | オプション | 説明 | 18 | | -------- | ---------- | 19 | |streamsEnabled| true に設定すると、プロキシは既存の DynamoDB テーブルからのライブデータの取り込みを有効します| 20 | |dynamoRegion| 上記のストリーミングを有効にした場合にのみ必要。DynamoDB テーブルが格納さているリージョン| 21 | |dyanmoAccessKey| 上記のストリーミングを有効にした場合にのみ必要。DynamoDB Streams への接続に使用| 22 | |dyanmoSecretKey| 上記のストリーミングを有効にした場合にのみ必要。DynamoDB Streams への接続に使用| 23 | |awsDynamodbEndpoint| 上記のストリーミングを有効にした場合にのみ必要。DynamoDB Streams への接続に使用| 24 | |contactPoints| Apache Cassandra(TM) クラスターへの接続に使用するコンタクトポイント。下記の docker オプションを使用する場合は、localhost のままにします| 25 | |dockerCassandra| true に設定すると、ローカルの docker で Cassandra を起動します。docker デーモンがインストールされ、実行されていることと、使用するユーザーが `docker ps` の実行権限を持っていることを確認してください| 26 | 27 | 28 | ## ローカルでの実行 29 | 30 | クローン: 31 | 32 | git clone git@github.com:datastax/dynamo-cassandra-proxy.git 33 | 34 | ビルド: 35 | 36 | mvn package 37 | 38 | プロキシの実行: プロキシを自分で用意した Cassandra クラスターに接続するにしろ、yaml ファルの中で cassandraDocker オプションを利用してプロキシに Cassandra ノードを立ち上げてもらうにしろ、ローカルでコードを実行するには、以下のようします。 39 | 40 | java -Ddw.contactPoints="$contactPoints" -cp target/dynamodb-cassandra-proxy-0.1.0.jar com.datastax.powertools.dcp.DCProxyApplication server conf/dynamo-cassandra-proxy.yaml 41 | 42 | プロキシは起動すると、8080 番のポートをリッスンします。そうしたら、DynamoDB アプリケーションを SDK の中で `<ホスト名>:8080` に向けます。接続文字列の参考例を以下に示します（Java の場合）。 43 | 44 | ClientConfiguration config = new ClientConfiguration(); 45 | config.setMaxConnections(dynamodbMaxConnections);; 46 | String dynamodbEndpoint = "localhost:8080" 47 | String signinRegion = "dummy" 48 | AwsClientBuilder.EndpointConfiguration endpointConfiguration = new AwsClientBuilder 49 | .EndpointConfiguration(protocol + "://" + dynamodbEndpoint, signinRegion); 50 | ddbBuilder = AmazonDynamoDBClientBuilder.standard() 51 | .withClientConfiguration(config) 52 | .withEndpointConfiguration(endpointConfiguration); 53 | 54 | メモ： `MaxConnections` は、AWS SDK をごく基本レベルの性能を超えて動作するようにする主要な鍵となります。参考までに、この値を最大 50 まで上げていきながらテストしましたが、ミディアムサイズの環境で、ほぼリニアにスケールすることを確認しています。ベンチマークなどのために Cassandra クラスターを飽和させることを目的としている場合には、この値を大きくして見てください。 55 | 56 | 57 | ## docker-compose を利用した実行 58 | 59 | 以下のコマンドでビルドします。 60 | 61 | mvn package 62 | 63 | docker コンテナをビルドして実行します。 64 | 65 | docker-compose up 66 | 67 | 68 | ## ローカルの Kubernetes での実行 69 | 70 | Cassandra 用の config map の設定: 71 | 72 | kubectl create configmap cassandra-config \ 73 | --from-file=common/cassandra/conf-dir/resources/cassandra/conf 74 | 75 | k8s yaml の適用: 76 | 77 | kubectl apply -f k8s-local/proxy-suite.yaml 78 | 79 | この時点で、pod は以下のようになっているはずです。 80 | 81 | ``` 82 | $ kubectl get pods [2:34:13] 83 | NAME READY STATUS RESTARTS AGE 84 | cassandra-0 1/1 Running 0 2m35s 85 | cassandra-1 1/1 Running 0 168s 86 | cassandra-3 1/1 Running 0 123s 87 | dynamo-cass-proxy-0 1/1 Running 4 63s 88 | ``` 89 | 90 | 実行したデプロイを終了するには、以下のようにします。 91 | 92 | kubectl delete -f k8s-local/proxy-suite.yaml 93 | 94 | 95 | ## コントリビューション 96 | 97 | Translator を用意するのも手始めとしては良いかもしれません。 98 | Translator の詳細については、[docs 内の Translators](docs/Translators-JP.md) をご覧ください。 99 | 100 | ## MVP（実用最小限の実装）のロードマップ: 101 | 102 | 印が入っているところは現時点の実装済機能: 103 | 104 | - [x] CreateTable - json_blob で実装 105 | - [x] DeleteItem - json_blob で実装 106 | - [ ] DeleteTable 107 | - [x] GetItem - json_blob で実装 108 | - [x] PutItem - json_blob で実装 109 | - [ ] Query - json_blob で単純なケースを実装 110 | - [ ] Scan 111 | - [x] ハイブリッド機能 - DynamoDB から Cassandra へ 112 | - [ ] ハイブリッド機能 - Cassandra から DynamoDB へ 113 | 114 | 未実装の**その他の機能** 115 | 116 | - UpdateItem 117 | - BatchGetItem 118 | - BatchWriteItem 119 | - DescribeStream 120 | - DescribeTable 121 | - DescribeLimits 122 | - DescribeTimeToLive 123 | - GetRecords 124 | - GetShardIterator 125 | - ListStreams 126 | - ListTables 127 | - ListTagsOfResource 128 | - TagResource 129 | - UntagResource 130 | - UpdateTable 131 | - UpdateTimeToLive 132 | 133 | ## ライセンス 134 | 本プロジェクトは、Apache Public License 2.0 のもとにライセンスします 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dynamo-cassandra-proxy 2 | 3 | Build state: 4 | ![build](https://api.travis-ci.org/datastax/dynamo-cassandra-proxy.svg?branch=master) 5 | 6 | `dynamo-cassandra-proxy` consists of a scalable proxy layer that sits between your app and Apache Cassandra. [See the design summary in docs](docs/Summary.md) 7 | 8 | It provides compatibility with the DynamoDB SDK which allows existing DynamoDB applications to read and write data to Cassandra without application changes. 9 | 10 | It also supports the ability to sync DynamoDB tables with cassandra via DynamoDB Streams. 11 | 12 | ## Config 13 | 14 | Create your yaml based on the template: 15 | 16 | cp conf/dynamo-cassandra-proxy.yaml.template conf/dynamo-cassandra-proxy.yaml 17 | 18 | The following are the options supported by the proxy: 19 | 20 | | Option | Description| 21 | | ------ | ---------- | 22 | |streamsEnabled| When set to true it enables the proxy to pull live data from an existing dynamodb table| 23 | |dynamoRegion| Only needed when streaming is enabled, region your dynamodb table is in | 24 | |dyanmoAccessKey| Only needed when streaming is enabled, used to connect to dynamodb streams| 25 | |dyanmoSecretKey| Only needed when streaming is enabled, used to connect to dynamodb streams| 26 | |awsDynamodbEndpoint| Only needed when streaming is enabled, used to connect to dynamodb streams| 27 | |contactPoints| Contact points to connect to Apache Cassandra(TM) cluster. If you are using the docker option just leave localhost| 28 | |dockerCassandra| When set to true it will stand up Cassandra in your local docker. Ensure the docker deamon is installed and running and your user has access to run `docker ps`| 29 | 30 | 31 | ## To run locally 32 | 33 | Clone: 34 | 35 | git clone git@github.com:datastax/dynamo-cassandra-proxy.git 36 | 37 | Build: 38 | 39 | mvn package 40 | 41 | Run the app: whether you point the proxy at your own c* cluster or you rely on the proxy to stand up a cassandra node using the cassandraDocker option in the yaml. You can run the code locally by: 42 | 43 | java -Ddw.contactPoints="$contactPoints" -cp target/dynamodb-cassandra-proxy-0.1.0.jar com.datastax.powertools.dcp.DCProxyApplication server conf/dynamo-cassandra-proxy.yaml 44 | 45 | The proxy will come up and listen on port :8080. In your dynamodb application, just point your app to `:8080` in the SDK. A sample connection string (in Java) should look as follows: 46 | 47 | ClientConfiguration config = new ClientConfiguration(); 48 | config.setMaxConnections(dynamodbMaxConnections);; 49 | String dynamodbEndpoint = "localhost:8080" 50 | String signinRegion = "dummy" 51 | AwsClientBuilder.EndpointConfiguration endpointConfiguration = new AwsClientBuilder 52 | .EndpointConfiguration(protocol + "://" + dynamodbEndpoint, signinRegion); 53 | ddbBuilder = AmazonDynamoDBClientBuilder.standard() 54 | .withClientConfiguration(config) 55 | .withEndpointConfiguration(endpointConfiguration); 56 | 57 | 58 | Note, `MaxConnections` is the main lever to get the AWS SDK to perform beyond very basic levels. We have tested this up to the maxium of 50 and it appears to scale almost linearly all the way up on a medum sized box. If you are doing some benchmarking and are looking to try to saturate a cassandra cluster, crank this value up. 59 | 60 | ## To run via docker-compose 61 | 62 | Build the app 63 | 64 | mvn package 65 | 66 | Build and run the docker containerst st 67 | 68 | docker-compose up 69 | 70 | 71 | ## To run in local k8s 72 | 73 | Set up cassandra config map: 74 | 75 | kubectl create configmap cassandra-config \ 76 | --from-file=common/cassandra/conf-dir/resources/cassandra/conf 77 | 78 | Apply k8s yaml: 79 | 80 | kubectl apply -f k8s-local/proxy-suite.yaml 81 | 82 | At this point, your pods should look as follows: 83 | 84 | ``` 85 | $ kubectl get pods [2:34:13] 86 | NAME READY STATUS RESTARTS AGE 87 | cassandra-0 1/1 Running 0 2m35s 88 | cassandra-1 1/1 Running 0 168s 89 | cassandra-3 1/1 Running 0 123s 90 | dynamo-cass-proxy-0 1/1 Running 4 63s 91 | ``` 92 | 93 | To terminate your deploymet run: 94 | 95 | kubectl delete -f k8s-local/proxy-suite.yaml 96 | 97 | ## To run on GKE 98 | 99 | ### Create cluster 100 | 101 | If you do not have a GKE cluster yet, [create one per the gcloud docs](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster) or use this sample commands: 102 | 103 | 104 | create: 105 | 106 | gcloud container clusters create dynamo-proxy-cluster \ 107 | --cluster-version=1.12.5-gke.10 --zone us-west1-b \ 108 | --machine-type n1-standard-4 --num-nodes 1 109 | 110 | configure kubectl: 111 | 112 | gcloud container clusters get-credentials k8-12-5-10-gke-n1-std-4 --zone us-west1-b 113 | 114 | ### Deploy the proxy 115 | 116 | 117 | Set up configMap 118 | 119 | kubectl create configmap cassandra-config \ 120 | --from-file=common/cassandra/conf-dir/resources/cassandra/conf 121 | 122 | Apply k8s yaml: 123 | 124 | kubectl apply -f gke/proxy-suite.yaml 125 | 126 | The workload will appear on your google console like so (ensure you're viewing the right project): 127 | 128 | ![gke](docs/images/gke.png) 129 | 130 | To terminate your deploymet run: 131 | 132 | kubectl delete -f gke/proxy-suite.yaml 133 | 134 | 135 | ## Contributing 136 | 137 | A good place to start might be fleshing out your own Translator. 138 | For details on translators see [Translators in the docs](docs/Translators.md) 139 | 140 | ## MVP Roadmap: 141 | 142 | Check means currently completed: 143 | 144 | - [x] CreateTable - Done in json_blob 145 | - [x] DeleteItem - Done in json_blob 146 | - [x] DeleteTable - Done in json_blob 147 | - [x] GetItem - Done in json_blob 148 | - [x] PutItem - Done in json_blob 149 | - [x] Query - key condition expression (json_blob) 150 | - [x] Query - key conditions (json_blob) 151 | - [x] Query - filter expressions (json_blob) 152 | - [ ] Query - projection expressions (json_blob) 153 | - [ ] Query - limit (json_blob) 154 | - [ ] Query - withSelect (json_blob) 155 | - [ ] Scan 156 | - [x] Hybrid functionality - DDB to Cassandra 157 | - [ ] Hybrid functionality - Cassandra to DDB 158 | 159 | **Other features** not yet implemented: 160 | 161 | - UpdateItem 162 | - BatchGetItem 163 | - BatchWriteItem 164 | - DescribeStream 165 | - DescribeTable 166 | - DescribeLimits 167 | - DescribeTimeToLive 168 | - GetRecords 169 | - GetShardIterator 170 | - ListStreams 171 | - ListTables 172 | - ListTagsOfResource 173 | - TagResource 174 | - UntagResource 175 | - UpdateTable 176 | - UpdateTimeToLive 177 | - ConsistentRead 178 | 179 | ## License 180 | This project is licensed under the Apache Public License 2.0 181 | -------------------------------------------------------------------------------- /common/cassandra/conf-dir/resources/cassandra/conf/cassandra-rackdc.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # These properties are used with GossipingPropertyFileSnitch and will 18 | # indicate the rack and dc for this node 19 | dc=dc1 20 | rack=rack1 21 | 22 | # Add a suffix to a datacenter name. Used by the Ec2Snitch and Ec2MultiRegionSnitch 23 | # to append a string to the EC2 region name. 24 | #dc_suffix= 25 | 26 | # Uncomment the following line to make this snitch prefer the internal ip when possible, as the Ec2MultiRegionSnitch does. 27 | # prefer_local=true 28 | -------------------------------------------------------------------------------- /conf/dynamo-cassandra-proxy.yaml.template: -------------------------------------------------------------------------------- 1 | dynamoRegion: us-east-2 2 | dynamoAccessKey: 3 | dynamoSecretKey: 4 | awsDynamodbEndpoint: apigateway.us-east-2.amazonaws.com 5 | streamsEnabled: false 6 | contactPoints: localhost 7 | dockerCassandra: false 8 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | ddac: 4 | image: "datastax/ddac" 5 | container_name: "ddac" 6 | environment: 7 | - DS_LICENSE=accept 8 | ports: 9 | - "9042:9042" 10 | ddb-cassandra-proxy: 11 | build: ./ 12 | container_name: "ddb-cas-proxy" 13 | command: | 14 | java -Ddw.contactPoints="ddac" -jar /opt/dynamo-cassandra-proxy/dynamodb-cassandra-proxy-0.1.0.jar server /opt/dynamo-cassandra-proxy/dynamo-cassandra-proxy.yaml 15 | ports: 16 | - "8080:8080" 17 | depends_on: 18 | - ddac 19 | restart: unless-stopped 20 | -------------------------------------------------------------------------------- /docs/Summary.md: -------------------------------------------------------------------------------- 1 | ## Running existing DynamoDB applications on a Cassandra cluster 2 | 3 | Many developers build on premises and then deploy to the cloud for production. The proxy enables customers to run their existing DynamoDB applications using Cassandra clusters on-prem. 4 | 5 | ![1](images/1.png) 6 | 7 | ## Live replication of DynamoDB applications to Cassandra 8 | 9 | DynamoDB Streams can be used to enable hybrid workload management and transfers from DynamoDB cloud deployments to on-prem Cassandra-proxied deployments. This is supported in the current implementation and, like DynamoDB Global Tables, it uses DynamoDB Streams to move the data. For hybrid transfer to DynamoDB, check out the Cassandra CDC improvements which could be leveraged and stay tuned to the DataStax blog for updates on our Change Data Capture (CDC) capabilities. 10 | 11 | ![2](images/2.png) 12 | 13 | 14 | ## What's in the Proxy? 15 | The proxy is designed to enable users to back their DynamoDB applications with Cassandra. We determined that the best way to help users leverage this new tool and to help it flourish was to make it an open source Apache 2 licensed project. 16 | The code consists of a scalable proxy layer that sits between your app and the database. It provides compatibility with the DynamoDB SDK which allows existing DynamoDB applications to read and write data to Cassandra without application changes. 17 | 18 | 19 | ## How It Works 20 | 21 | A few design decisions were made when designing the proxy. As always, these are in line with the design principles that we use to guide development for both Cassandra and our DataStax Enterprise product. 22 | 23 | ## Why a Separate Process? 24 | 25 | We could have built this as a Cassandra plugin that would execute as part of the core process but we decided to build it as a separate process for the following reasons: 26 | - Ability to scale the proxy independently of Cassandra 27 | - Ability to leverage k8s / cloud-native tooling 28 | - Developer agility and to attract contributors—developers can work on the proxy with limited knowledge of Cassandra internals 29 | - Independent release cadence, not tied to the Apache Cassandra project 30 | - Better AWS integration story for stateless apps (i.e., leverage CloudWatch alarm, autoscaling, etc.) 31 | 32 | ## Why Pluggable Persistence? 33 | 34 | On quick inspection, DynamoDB's data model is quite simple. It consists of a hash key, a sort key, and a JSON structure which is referred to as an item. Depending on your goals, the DynamoDB data model can be persisted in Cassandra Query Language (CQL) in different ways. To allow for experimentation and pluggability, we have built the translation layer in a pluggable way that allows for different translators. We continue to build on this scaffolding to test out multiple data models and determine which are best suited for: 35 | 36 | - Different workloads 37 | - Different support for consistency / linearization requirements 38 | - Different performance tradeoffs based on SLAs 39 | -------------------------------------------------------------------------------- /docs/Translators-JP.md: -------------------------------------------------------------------------------- 1 | ## Translator 2 | 3 | `dynamo-db-proxy` は、dynamodb のリクエストから CQL のデータモデルへの変換のために Translator を使用します。 4 | 5 | dynamodb は、単独のパーティションキーと、単独のソートキーを使用します。その他はすべて JSON です。クエリー API は、クエリー時に JSON ペイロード内の個別フィールドの選択を可能にしますが、グローバルインデックスを除き、その選択範囲は、パーティション内に限られます。 6 | DDB supports a single partition key and a single sort key. Everything else is JSON. The query API allows users to select individual fields within the JSON payload at query time but only within a partition with the exception of global indexes. 7 | 8 | Translator は、DynamoDB の機能を部分的にサポートしたり、それ以外のみ実装の機能については UnimplementedExceptions を throw することができます。 9 | 10 | Translator クラスは、`DynamoDSETranslator` を extend しなければなりません。 11 | 12 | ## Cassandra / DSE のデータモデル 13 | 14 | ### json_blob 15 | 16 | json_blob - 現在の実装 17 | 18 | ``` 19 | CREATE TABLE dynamoks.test ( 20 | hash_key double, 21 | sort_key double, 22 | json_blob text, 23 | PRIMARY KEY (hash_key, sort_key) 24 | ) WITH CLUSTERING ORDER BY (sort_key ASC) 25 | ``` 26 | 27 | 最も単純な実装となっています。制限としては、DynamoDB の UpdateItem API は、クエリーのたびに write の前に read が必要となるため、対応が容易ではないということです。そのため、更新クエリーは、サポートされていない旨の例外を返します。 28 | ``` 29 | cqlsh> select * from dynamoks.test ; 30 | 31 | hash_key | sort_key | json_blob 32 | ----------+----------+---------------------------------------------------------------------------------------------------------------------------------- 33 | 1 | 2 | {"favorites":{"SS":["puppies","kittens","other cute animals"]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 34 | 35 | ``` 36 | ### マルチバージョン 37 | 38 | LSM の考えを取り入れたデータモデルであれば、更新をバージョニングし、read 時に LWW のセマンティクスをクライアント側で実装して使用し、（コンパクションのような）バックグラウドで動作するクリーンアップジョブを用意すれば、DynamoDB の UpdateItem API をサポートできるかもしれません。 39 | 40 | ``` 41 | CREATE TABLE dynamoks.test ( 42 | hash_key double, 43 | sort_key double, 44 | version timeuuid, 45 | json_blob text, 46 | PRIMARY KEY (hash_key, sort_key, version) 47 | ) WITH CLUSTERING ORDER BY (sort_key ASC) 48 | ``` 49 | 50 | データ例: 51 | ``` 52 | cqlsh> select * from dynamoks.test ; 53 | 54 | hash_key | sort_key | version |json_blob 55 | ----------+----------+--------------------- 56 | 1 | 2 | 1 |{"favorites":{"SS":["puppies","kittens","other cute animals"]},"hash_key":{"S":"1.0"},"sort_key":{"S":"2.0"}} 57 | 1 | 2 | 2 |{"favorites":{"SS":["kittens","other cute animals"]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 58 | ``` 59 | 60 | クエリー時には、プロキシがパーティション全体を読み取り、マージを行い、マージ結果を返すという方法が考えられます。 61 | 62 | ``` 63 | {"favorites":{"SS":["puppies","kittens","other cute animals"]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 64 | ``` 65 | 66 | 67 | 68 | ### 完全非正規化 69 | 70 | より良いマルチモデルと CQL への対応を考えた場合、以下のようなデータモデルならば、dynamodb 項目を非正規化して複数のレコードにできます。 71 | 72 | ``` 73 | CREATE TABLE dynamoks.test ( 74 | hash_key double, 75 | sort_key double, 76 | column_name text, 77 | value_numeric double, 78 | value_string text, 79 | value_date date, 80 | PRIMARY KEY (hash_key, sort_key, column_name) 81 | ) WITH CLUSTERING ORDER BY (sort_key ASC) 82 | ``` 83 | 84 | データ例: 85 | ``` 86 | cqlsh> select * from dynamoks.test ; 87 | 88 | hash_key | sort_key | column_name | value_numeric | value_string | value_date 89 | ----------+----------+---------------------------------------------------- 90 | 1 | 2 | 'puppies' | 3 | | 91 | 1 | 2 | 'City' | | 'NYC' | 92 | ``` 93 | 94 | 95 | クエリー時には、プロキシがパーティション全体を読み取り、マージを行い、マージ結果を返すということになります。 96 | 97 | ``` 98 | {"favorites":{"N":["puppies",3]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 99 | ``` 100 | 101 | 上記の混合も考えられます（完全非正規化とマルチバージョンの混合） -------------------------------------------------------------------------------- /docs/Translators.md: -------------------------------------------------------------------------------- 1 | ## Translators 2 | 3 | The `dynamo-db-proxy` uses Translators to go from dynamodb requests to CQL data models. 4 | 5 | DDB supports a single partition key and a single sort key. Everything else is JSON. The query API allows users to select individual fields within the JSON payload at query time but only within a partition with the exception of global indexes. 6 | 7 | Some Translators can support partial DynamoDB functionality and throw UnimplementedExceptions for other functionality. 8 | 9 | A translator class must extend `DynamoDSETranslator`. 10 | 11 | ## DSE Data Models: 12 | 13 | ### json_blob 14 | 15 | json_blob - currently implemented 16 | 17 | ``` 18 | CREATE TABLE dynamoks.test ( 19 | hash_key double, 20 | sort_key double, 21 | json_blob text, 22 | PRIMARY KEY (hash_key, sort_key) 23 | ) WITH CLUSTERING ORDER BY (sort_key ASC) 24 | ``` 25 | 26 | This is the simplest implementation. The downside is that it cannot feasibly support the DDB UpdateItem API because it would require a read before write on every query. Update queries will return a not supported exception. 27 | ``` 28 | cqlsh> select * from dynamoks.test ; 29 | 30 | hash_key | sort_key | json_blob 31 | ----------+----------+---------------------------------------------------------------------------------------------------------------------------------- 32 | 1 | 2 | {"favorites":{"SS":["puppies","kittens","other cute animals"]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 33 | 34 | ``` 35 | ### multi-version 36 | An LSM inspired data model could support the DDB UpdateItem API by versioning updates and using client implemented LWW semantics on read and potentially a cleanup (compaction-like) background job. 37 | ``` 38 | CREATE TABLE dynamoks.test ( 39 | hash_key double, 40 | sort_key double, 41 | version timeuuid, 42 | json_blob text, 43 | PRIMARY KEY (hash_key, sort_key, version) 44 | ) WITH CLUSTERING ORDER BY (sort_key ASC) 45 | ``` 46 | 47 | Example data: 48 | ``` 49 | cqlsh> select * from dynamoks.test ; 50 | 51 | hash_key | sort_key | version |json_blob 52 | ----------+----------+--------------------- 53 | 1 | 2 | 1 |{"favorites":{"SS":["puppies","kittens","other cute animals"]},"hash_key":{"S":"1.0"},"sort_key":{"S":"2.0"}} 54 | 1 | 2 | 2 |{"favorites":{"SS":["kittens","other cute animals"]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 55 | ``` 56 | 57 | For queries, the proxy would read the whole partition, merge and return a merged response: 58 | ``` 59 | {"favorites":{"SS":["puppies","kittens","other cute animals"]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 60 | ``` 61 | 62 | 63 | 64 | ### fully-denormalized 65 | For better multi-model and cql support, the following data model would denormalize the dynamodb item into multiple records. 66 | ``` 67 | CREATE TABLE dynamoks.test ( 68 | hash_key double, 69 | sort_key double, 70 | column_name text, 71 | value_numeric double, 72 | value_string text, 73 | value_date date, 74 | PRIMARY KEY (hash_key, sort_key, column_name) 75 | ) WITH CLUSTERING ORDER BY (sort_key ASC) 76 | ``` 77 | 78 | Example data: 79 | ``` 80 | cqlsh> select * from dynamoks.test ; 81 | 82 | hash_key | sort_key | column_name | value_numeric | value_string | value_date 83 | ----------+----------+---------------------------------------------------- 84 | 1 | 2 | 'puppies' | 3 | | 85 | 1 | 2 | 'City' | | 'NYC' | 86 | ``` 87 | 88 | 89 | For queries, the proxy would read the whole partition, merge and return a merged response: 90 | ``` 91 | {"favorites":{"N":["puppies",3]},"hash_key":{"S":"1.0"},"city":{"S":"NYC"},"sort_key":{"S":"2.0"}} 92 | ``` 93 | 94 | A mix is also possible (Fully denormalized / multi-version) 95 | -------------------------------------------------------------------------------- /docs/images/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dynamo-cassandra-proxy/f85114313cfd4767d6072113f552d89549970d9a/docs/images/1.png -------------------------------------------------------------------------------- /docs/images/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dynamo-cassandra-proxy/f85114313cfd4767d6072113f552d89549970d9a/docs/images/2.png -------------------------------------------------------------------------------- /docs/images/gke.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dynamo-cassandra-proxy/f85114313cfd4767d6072113f552d89549970d9a/docs/images/gke.png -------------------------------------------------------------------------------- /gke/proxy-suite.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: fast 5 | provisioner: kubernetes.io/gce-pd 6 | parameters: 7 | type: pd-ssd 8 | --- 9 | apiVersion: v1 10 | kind: Service 11 | metadata: 12 | name: cassandra-ext-lb 13 | labels: 14 | app: cassandra 15 | spec: 16 | type: LoadBalancer 17 | ports: 18 | - port: 9042 19 | name: cql-port 20 | selector: 21 | app: cassandra 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | name: cassandra 27 | labels: 28 | app: cassandra 29 | spec: 30 | ports: 31 | - port: 9042 32 | name: cql-port 33 | clusterIP: None 34 | selector: 35 | app: cassandra 36 | --- 37 | apiVersion: apps/v1 38 | kind: StatefulSet 39 | metadata: 40 | name: cassandra 41 | spec: 42 | selector: 43 | matchLabels: 44 | app: cassandra 45 | serviceName: "cassandra" 46 | replicas: 3 47 | podManagementPolicy: OrderedReady 48 | updateStrategy: 49 | type: RollingUpdate 50 | template: 51 | metadata: 52 | labels: 53 | app: cassandra 54 | spec: 55 | securityContext: 56 | fsGroup: 999 57 | affinity: 58 | podAntiAffinity: 59 | requiredDuringSchedulingIgnoredDuringExecution: 60 | - labelSelector: 61 | matchExpressions: 62 | - key: app 63 | operator: In 64 | values: 65 | - cassandra 66 | topologyKey: kubernetes.io/hostname 67 | terminationGracePeriodSeconds: 12000 68 | containers: 69 | - name: cassandra 70 | image: datastax/ddac:latest 71 | imagePullPolicy: IfNotPresent 72 | resources: 73 | requests: 74 | cpu: "2" 75 | memory: "4000Mi" 76 | env: 77 | - name: DS_LICENSE 78 | value: accept 79 | - name: SEEDS 80 | value: cassandra-0.cassandra.default.svc.cluster.local,cassandra-1.cassandra.default.svc.cluster.local,cassandra-2.cassandra.default.svc.cluster.local 81 | - name: CLUSTER_NAME 82 | value: "Test_Cluster" 83 | - name: NUM_TOKENS 84 | value: "64" 85 | - name: DC 86 | value: "DC-1" 87 | - name: RACK 88 | value: "rack-1" 89 | - name: SNITCH 90 | value: GossipingPropertyFileSnitch 91 | ports: 92 | - containerPort: 7000 93 | name: intra-node-port 94 | - containerPort: 7001 95 | name: tls-intra-node 96 | - containerPort: 7199 97 | name: jmx-port 98 | - containerPort: 8609 99 | name: inter-node-msg 100 | - containerPort: 9042 101 | name: cql-port 102 | - containerPort: 9160 103 | name: thrift-port 104 | - containerPort: 9103 105 | name: ds-agent-port 106 | volumeMounts: 107 | - name: cassandra-data 108 | mountPath: /var/lib/cassandra 109 | - name: config-volume 110 | mountPath: /config 111 | livenessProbe: 112 | tcpSocket: 113 | port: 9042 114 | initialDelaySeconds: 900 115 | timeoutSeconds: 1 116 | periodSeconds: 30 117 | failureThreshold: 10 118 | lifecycle: 119 | postStart: 120 | exec: 121 | command: ['/bin/sh', '-c', 'until cqlsh -e "describe cluster"; do echo waiting for DSE Startup Complete; sleep 10; done;'] 122 | preStop: 123 | exec: 124 | command: ["/bin/sh", "-c", "exec nodetool decommission"] 125 | volumes: 126 | - name: config-volume 127 | configMap: 128 | name: cassandra-config 129 | volumeClaimTemplates: 130 | - metadata: 131 | name: cassandra-data 132 | spec: 133 | accessModes: [ "ReadWriteOnce" ] 134 | storageClassName: fast 135 | resources: 136 | requests: 137 | storage: 40Gi 138 | --- 139 | apiVersion: v1 140 | kind: Service 141 | metadata: 142 | name: dynamo-cass-proxy-ext-lb 143 | labels: 144 | app: dynamo-cass-proxy 145 | spec: 146 | type: LoadBalancer 147 | ports: 148 | - port: 8080 149 | name: port-8080 150 | selector: 151 | app: dynamo-cass-poxy 152 | --- 153 | apiVersion: v1 154 | kind: Service 155 | metadata: 156 | name: dynamo-cass-proxy 157 | labels: 158 | app: proxy 159 | spec: 160 | ports: 161 | - port: 8080 162 | name: port-8080 163 | clusterIP: None 164 | selector: 165 | app: dynamo-cass-proxy 166 | --- 167 | apiVersion: apps/v1 168 | kind: StatefulSet 169 | metadata: 170 | name: dynamo-cass-proxy 171 | spec: 172 | selector: 173 | matchLabels: 174 | app: dynamo-cass-proxy 175 | serviceName: "dynamo-cass-proxy" 176 | replicas: 1 177 | podManagementPolicy: OrderedReady 178 | updateStrategy: 179 | type: RollingUpdate 180 | template: 181 | metadata: 182 | labels: 183 | app: dynamo-cass-proxy 184 | spec: 185 | securityContext: 186 | fsGroup: 472 187 | containers: 188 | - name: dynamo-cass-proxy 189 | image: phact/dynamo-cassandra-proxy 190 | command: ['/bin/sh', '-c', 'java -Ddw.contactPoints="cassandra" -cp /opt/dynamo-cassandra-proxy/dynamodb-cassandra-proxy-0.1.0.jar com.datastax.powertools.dcp.DCProxyApplication server /opt/dynamo-cassandra-proxy/dynamo-cassandra-proxy.yaml'] 191 | imagePullPolicy: IfNotPresent 192 | resources: 193 | requests: 194 | cpu: "2" 195 | memory: "4000Mi" 196 | ports: 197 | - containerPort: 8080 198 | name: port-8080 199 | -------------------------------------------------------------------------------- /k8s-local/proxy-suite.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: cassandra-ext-lb 5 | labels: 6 | app: cassandra 7 | spec: 8 | type: LoadBalancer 9 | ports: 10 | - port: 9042 11 | name: cql-port 12 | selector: 13 | app: cassandra 14 | --- 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: cassandra 19 | labels: 20 | app: cassandra 21 | spec: 22 | ports: 23 | - port: 9042 24 | name: cql-port 25 | - port: 9103 26 | name: prom-port 27 | clusterIP: None 28 | selector: 29 | app: cassandra 30 | --- 31 | apiVersion: apps/v1 32 | kind: StatefulSet 33 | metadata: 34 | name: cassandra 35 | spec: 36 | selector: 37 | matchLabels: 38 | app: cassandra 39 | serviceName: "cassandra" 40 | replicas: 3 41 | podManagementPolicy: OrderedReady 42 | updateStrategy: 43 | type: RollingUpdate 44 | template: 45 | metadata: 46 | labels: 47 | app: cassandra 48 | spec: 49 | securityContext: 50 | fsGroup: 999 51 | affinity: 52 | podAntiAffinity: 53 | requiredDuringSchedulingIgnoredDuringExecution: 54 | - labelSelector: 55 | matchExpressions: 56 | - key: app 57 | operator: In 58 | values: 59 | - cassandra 60 | topologyKey: kubernetes.io/hostname 61 | terminationGracePeriodSeconds: 12000 62 | containers: 63 | - name: cassandra 64 | image: datastax/ddac:latest 65 | imagePullPolicy: IfNotPresent 66 | resources: 67 | requests: 68 | cpu: "2" 69 | memory: "4000Mi" 70 | env: 71 | - name: DS_LICENSE 72 | value: accept 73 | - name: SEEDS 74 | value: cassandra-0.cassandra.default.svc.cluster.local,cassandra-1.cassandra.default.svc.cluster.local,cassandra-2.cassandra.default.svc.cluster.local 75 | - name: CLUSTER_NAME 76 | value: "Test_Cluster" 77 | - name: NUM_TOKENS 78 | value: "64" 79 | - name: DC 80 | value: "DC-1" 81 | - name: RACK 82 | value: "rack-1" 83 | - name: SNITCH 84 | value: GossipingPropertyFileSnitch 85 | ports: 86 | - containerPort: 7000 87 | name: intra-node-port 88 | - containerPort: 7001 89 | name: tls-intra-node 90 | - containerPort: 7199 91 | name: jmx-port 92 | - containerPort: 8609 93 | name: inter-node-msg 94 | - containerPort: 9042 95 | name: cql-port 96 | - containerPort: 9160 97 | name: thrift-port 98 | - containerPort: 9103 99 | name: prom-port 100 | - containerPort: 61621 101 | name: ds-agent-port 102 | volumeMounts: 103 | - name: cassandra-data 104 | mountPath: /var/lib/cassandra 105 | - name: config-volume 106 | mountPath: /config 107 | - name: prometheus-volume 108 | mountPath: /opt/cassandra/resources/cassandra/collectd/etc/collectd 109 | livenessProbe: 110 | tcpSocket: 111 | port: 9042 112 | initialDelaySeconds: 900 113 | timeoutSeconds: 1 114 | periodSeconds: 30 115 | failureThreshold: 10 116 | lifecycle: 117 | postStart: 118 | exec: 119 | command: ['/bin/sh', '-c', 'until cqlsh -e "describe cluster"; do echo waiting for Cassandra Startup Complete; sleep 10; done;'] 120 | preStop: 121 | exec: 122 | command: ["/bin/sh", "-c", "exec nodetool decommission"] 123 | volumes: 124 | - name: config-volume 125 | configMap: 126 | name: cassandra-config 127 | - name: prometheus-volume 128 | - name: cassandra-data 129 | emptyDir: {} 130 | --- 131 | apiVersion: v1 132 | kind: Service 133 | metadata: 134 | name: dynamo-cass-proxy-ext-lb 135 | labels: 136 | app: dynamo-cass-proxy 137 | spec: 138 | type: LoadBalancer 139 | ports: 140 | - port: 8080 141 | name: port-8080 142 | selector: 143 | app: dynamo-cass-proxy 144 | --- 145 | apiVersion: v1 146 | kind: Service 147 | metadata: 148 | name: dynamo-cass-proxy 149 | labels: 150 | app: dynamo-cass-proxy 151 | spec: 152 | ports: 153 | - port: 8080 154 | name: port-8080 155 | clusterIP: None 156 | selector: 157 | app: dynamo-cass-proxy 158 | --- 159 | apiVersion: apps/v1 160 | kind: StatefulSet 161 | metadata: 162 | name: dynamo-cass-proxy 163 | spec: 164 | selector: 165 | matchLabels: 166 | app: dynamo-cass-proxy 167 | serviceName: "dynamo-cass-proxy" 168 | replicas: 1 169 | podManagementPolicy: OrderedReady 170 | updateStrategy: 171 | type: RollingUpdate 172 | template: 173 | metadata: 174 | labels: 175 | app: dynamo-cass-proxy 176 | spec: 177 | containers: 178 | - name: dynamo-cass-proxy 179 | image: phact/dynamo-cassandra-proxy 180 | command: ['/bin/sh', '-c', 'java -Ddw.contactPoints="cassandra" -cp /opt/dynamo-cassandra-proxy/dynamodb-cassandra-proxy-0.1.0.jar com.datastax.powertools.dcp.DCProxyApplication server /opt/dynamo-cassandra-proxy/dynamo-cassandra-proxy.yaml'] 181 | imagePullPolicy: IfNotPresent 182 | resources: 183 | # requests: 184 | # cpu: "2" 185 | # memory: "4000Mi" 186 | ports: 187 | - containerPort: 8080 188 | name: port-8080 189 | # configMap: 190 | # name: dynamo-cass-proxy-config 191 | -------------------------------------------------------------------------------- /migrate/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /migrate/.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /migrate/.idea/hydra.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | -------------------------------------------------------------------------------- /migrate/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /migrate/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /migrate/.idea/sbt.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /migrate/.idea/scala_compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /migrate/.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 35 | 36 | 37 | 42 | 43 | 44 | 49 | 50 | 51 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 |