├── .gitignore ├── CONTRIBUTING.md ├── Jenkinsfile ├── LICENSE ├── Makefile ├── OWNERS ├── README-QWICKLABS.md ├── README.md ├── common.sh ├── container ├── Dockerfile ├── Makefile ├── README.md ├── cloudbuild.yaml └── files │ ├── cassandra.yaml │ ├── jvm.options │ ├── logback-files.xml │ ├── logback-json-files.xml │ ├── logback-json-stdout.xml │ ├── logback-stdout.xml │ ├── prometheus.yaml │ ├── ready-probe.sh │ └── run.sh ├── create.sh ├── delete.sh ├── images └── gke-cassandra.png ├── manifests ├── cassandra-pdb.yaml ├── cassandra-service.yaml ├── cassandra-statefulset.yaml └── cassandra-storageclass.yaml ├── renovate.json ├── test ├── boilerplate │ ├── boilerplate.BUILD.txt │ ├── boilerplate.Dockerfile.txt │ ├── boilerplate.Makefile.txt │ ├── boilerplate.WORKSPACE.txt │ ├── boilerplate.bazel.txt │ ├── boilerplate.bzl.txt │ ├── boilerplate.css.txt │ ├── boilerplate.go.preamble │ ├── boilerplate.go.txt │ ├── boilerplate.html.preamble │ ├── boilerplate.html.txt │ ├── boilerplate.java.txt │ ├── boilerplate.js.txt │ ├── boilerplate.py.preamble │ ├── boilerplate.py.txt │ ├── boilerplate.scss.txt │ ├── boilerplate.sh.preamble │ ├── boilerplate.sh.txt │ ├── boilerplate.tf.txt │ ├── boilerplate.ts.txt │ ├── boilerplate.xml.preamble │ ├── boilerplate.xml.txt │ └── boilerplate.yaml.txt ├── make.sh └── verify_boilerplate.py ├── update_image_tag.sh └── validate.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # OSX leaves these everywhere on SMB shares 2 | ._* 3 | 4 | # OSX trash 5 | .DS_Store 6 | 7 | # Emacs save files 8 | *~ 9 | \#*\# 10 | .\#* 11 | 12 | # Vim-related files 13 | [._]*.s[a-w][a-z] 14 | [._]s[a-w][a-z] 15 | *.un~ 16 | Session.vim 17 | .netrwhist 18 | 19 | test/__pycache__ 20 | 21 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | Contributions to this project must be accompanied by a Contributor License 8 | Agreement. You (or your employer) retain the copyright to your contribution; 9 | this simply gives us permission to use and redistribute your contributions as 10 | part of the project. Head over to https://cla.developers.google.com/ to see your 11 | current agreements on file or to sign a new one. 12 | 13 | You generally only need to submit a CLA once, so if you've already submitted one 14 | (even if it was for a different project), you probably don't need to do it again. 15 | 16 | ## Code reviews 17 | All submissions, including submissions by project members, require review. We 18 | use GitHub pull requests for this purpose. Consult GitHub Help for more 19 | information on using pull requests. 20 | 21 | ## Community Guidelines 22 | This project follows 23 | [Google's Open Source Community Guidelines](CODE-OF-CONDUCT.md). 24 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env groovy 2 | 3 | /* 4 | Copyright 2018 Google LLC 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | https://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | */ 19 | 20 | // The declarative agent is defined in yaml. It was previously possible to 21 | // define containerTemplate but that has been deprecated in favor of the yaml 22 | // format 23 | // Reference: https://github.com/jenkinsci/kubernetes-plugin 24 | pipeline { 25 | agent { 26 | kubernetes { 27 | defaultContainer 'jnlp' 28 | yaml """ 29 | apiVersion: v1 30 | kind: Pod 31 | metadata: 32 | labels: 33 | jenkins: build-node 34 | spec: 35 | containers: 36 | - name: stateful-applications 37 | image: gcr.io/pso-helmsman-cicd/jenkins-k8s-node:${env.JENKINS_CONTAINER_VERSION} 38 | command: 39 | - cat 40 | tty: true 41 | volumeMounts: 42 | # Mount the dev service account key 43 | - name: dev-key 44 | mountPath: /home/jenkins/dev 45 | volumes: 46 | # Create a volume that contains the dev json key that was saved as a secret 47 | - name: dev-key 48 | secret: 49 | secretName: jenkins-deploy-dev-infra 50 | """ 51 | } 52 | } 53 | 54 | environment { 55 | CASSANDRA_VERSION = "3.11.4" 56 | REV = "v${env.BUILD_NUMBER}" 57 | IMAGE_TAG = "${CASSANDRA_VERSION}-${REV}" 58 | APP_NAME='cassandra' 59 | MANIFEST_FILE='manifests/cassandra-statefulset.yaml' 60 | GOOGLE_APPLICATION_CREDENTIALS = '/home/jenkins/dev/jenkins-deploy-dev-infra.json' 61 | } 62 | 63 | stages { 64 | 65 | // Run our various linters against the project 66 | stage('Lint') { 67 | steps { 68 | container('stateful-applications') { 69 | sh "make all" 70 | } 71 | } 72 | } 73 | 74 | // Setup the GCE access for Jenkins test run 75 | stage('Setup') { 76 | steps { 77 | container('stateful-applications') { 78 | script { 79 | // env.CLUSTER_ZONE will need to be updated to match the 80 | // ZONE in the jenkins.propeties file 81 | env.CLUSTER_ZONE = "${CLUSTER_ZONE}" 82 | // env.PROJECT_ID will need to be updated to match your GCP 83 | // development project id 84 | env.PROJECT_ID = "${PROJECT_ID}" 85 | env.REGION = "${REGION}" 86 | def shortCommit = sh ( returnStdout: true, script: 'git rev-parse HEAD | cut -c 1-6').trim() 87 | env.CLUSTER_NAME = "mycsharp-${shortCommit}" 88 | env.KEYFILE = GOOGLE_APPLICATION_CREDENTIALS 89 | } 90 | // Setup gcloud service account access 91 | sh "gcloud auth activate-service-account --key-file=${env.KEYFILE}" 92 | sh "gcloud config set compute/zone ${env.CLUSTER_ZONE}" 93 | sh "gcloud config set core/project ${env.PROJECT_ID}" 94 | sh "gcloud config set container/cluster ${env.CLUSTER_NAME}" 95 | sh "gcloud config set compute/region ${env.REGION}" 96 | } 97 | } 98 | } 99 | 100 | // Use Cloud Build to build our two containers 101 | stage('Build Containers') { 102 | steps { 103 | container('stateful-applications') { 104 | dir ('container') { 105 | sh 'gcloud builds submit --config=cloudbuild.yaml --substitutions=_CASSANDRA_VERSION=${CASSANDRA_VERSION},_REV_=${REV} .' 106 | } 107 | } 108 | } 109 | } 110 | 111 | // Update the manifest and build the Cassandra Cluster 112 | stage('Create') { 113 | steps { 114 | container('stateful-applications') { 115 | timeout(time: 30, unit: 'MINUTES') { 116 | // update the cassandra image tag 117 | sh "./update_image_tag.sh ${PROJECT_ID} ${APP_NAME} ${IMAGE_TAG} ${MANIFEST_FILE}" 118 | sh "make create CLUSTER_NAME=${env.CLUSTER_NAME}" 119 | sh "sleep 360" 120 | } 121 | } 122 | } 123 | } 124 | 125 | // Validate the Cassandra Cluster 126 | stage('Validate') { 127 | steps { 128 | container('stateful-applications') { 129 | script { 130 | for (int i = 0; i < 3; i++) { 131 | sh "make validate CLUSTER_NAME=${env.CLUSTER_NAME}" 132 | } 133 | } 134 | } 135 | } 136 | } 137 | } 138 | 139 | // Tear down everything 140 | post { 141 | always { 142 | container('stateful-applications') { 143 | sh "make delete CLUSTER_NAME=${env.CLUSTER_NAME}" 144 | } 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Make will use bash instead of sh 16 | SHELL := /usr/bin/env bash 17 | 18 | # add for CICD 19 | ROOT := ${CURDIR} 20 | CLUSTER_NAME ?= foo-csharp-1 21 | 22 | .PHONY: create 23 | create: 24 | $(ROOT)/create.sh -c $(CLUSTER_NAME) 25 | 26 | .PHONY: delete 27 | delete: 28 | $(ROOT)/delete.sh -c $(CLUSTER_NAME) 29 | 30 | .PHONY: validate 31 | validate: 32 | $(ROOT)/validate.sh -c $(CLUSTER_NAME) 33 | # add for CICD 34 | # All is the first target in the file so it will get picked up when you just run 'make' on its own 35 | all: check_shell check_python check_golang check_terraform check_docker check_base_files check_headers check_trailing_whitespace 36 | 37 | # The .PHONY directive tells make that this isn't a real target and so 38 | # the presence of a file named 'check_shell' won't cause this target to stop 39 | # working 40 | .PHONY: check_shell 41 | check_shell: 42 | @source test/make.sh && check_shell 43 | 44 | .PHONY: check_python 45 | check_python: 46 | @source test/make.sh && check_python 47 | 48 | .PHONY: check_golang 49 | check_golang: 50 | @source test/make.sh && golang 51 | 52 | .PHONY: check_terraform 53 | check_terraform: 54 | @source test/make.sh && check_terraform 55 | 56 | .PHONY: check_docker 57 | check_docker: 58 | @source test/make.sh && docker 59 | 60 | .PHONY: check_base_files 61 | check_base_files: 62 | @source test/make.sh && basefiles 63 | 64 | .PHONY: check_shebangs 65 | check_shebangs: 66 | @source test/make.sh && check_bash 67 | 68 | .PHONY: check_trailing_whitespace 69 | check_trailing_whitespace: 70 | @source test/make.sh && check_trailing_whitespace 71 | 72 | .PHONY: check_headers 73 | check_headers: 74 | @echo "Checking file headers" 75 | @python3.7 test/verify_boilerplate.py 76 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - chrislovecnm 3 | - robinpercy 4 | - geojaz 5 | - techgnosis 6 | - erkolson 7 | labels: 8 | - gke-helmsman 9 | -------------------------------------------------------------------------------- /README-QWICKLABS.md: -------------------------------------------------------------------------------- 1 | 2 | # Google Kubernetes Engine Stateful Application Demo 3 | 4 | ## Table of Contents 5 | 6 | 7 | * [Google Kubernetes Engine Stateful Application Demo](#google-kubernetes-engine-stateful-application-demo) 8 | * [Introduction](#introduction) 9 | * [Architecture](#architecture) 10 | * [Architecture Diagram](#architecture-diagram) 11 | * [Kubernetes Manifests and Controllers](#kubernetes-manifests-and-controllers) 12 | * [StatefulSets](#statefulsets) 13 | * [Persistent Storage with Persistent Volumes](#persistent-storage-with-persistent-volumes) 14 | * [Pod Disruptions Budgets](#pod-disruptions-budgets) 15 | * [Pod Scheduling](#pod-scheduling) 16 | * [DNS and Headless Services](#dns-and-headless-services) 17 | * [Cassandra Container](#cassandra-container) 18 | * [Deployment Steps](#deployment-steps) 19 | * [Validation](#validation) 20 | * [Using Cassandara](#using-cassandara) 21 | * [Launch a Cassandra container](#launch-a-cassandra-container) 22 | * [Connect to the ring with cqlsh](#connect-to-the-ring-with-cqlsh) 23 | * [cqlsh command](#cqlsh-command) 24 | * [Create a keyspace](#create-a-keyspace) 25 | * [Describe the keyspace](#describe-the-keyspace) 26 | * [Use the new keyspace](#use-the-new-keyspace) 27 | * [Create a table](#create-a-table) 28 | * [See the newly created table](#see-the-newly-created-table) 29 | * [Insert data into the table](#insert-data-into-the-table) 30 | * [Select the data from the table](#select-the-data-from-the-table) 31 | * [Exit](#exit) 32 | * [Delete the deployment](#delete-the-deployment) 33 | * [Tear Down](#tear-down) 34 | * [Troubleshooting](#troubleshooting) 35 | * [Relevant Materials](#relevant-materials) 36 | 37 | 38 | ## Introduction 39 | 40 | This proof of concept deploys a 41 | [Kubernetes Engine](https://cloud.google.com/kubernetes-engine/) 42 | Cluster and then installs an [Apache Cassandra](http://cassandra.apache.org/) 43 | database running on that cluster. 44 | 45 | Various scripts are contained within this project that provide push button 46 | creation, validation, and deletion of the Cassandra(C*) database and Kubernetes 47 | Engine cluster. 48 | 49 | Apache Cassandra was chosen for various reasons. These reasons include that 50 | out of the box Cassandra functions well as a cloud native database. Moreover, 51 | this POC continues the work started with the Kubernetes example, and the blog 52 | post: [Thousand Instances of Cassandra using Kubernetes Pet Set][1]. 53 | 54 | When running a database using Kubernetes Engine, as an operator you need to be experienced with Kubernetes Engine 55 | and the datasource that you are running. If you are able to use a hosted 56 | solution then it is recommended that you use the hosted solution. On the other 57 | hand, if you are not able to run a datasource in a hosted solution then hosting 58 | it is Kubernetes Engine is doable. The challenge is not Kubernetes or Kubernetes Engine, but the challenge 59 | is migrating and operating a database in a container based system that can do 60 | things like moving pods/containers between nodes. 61 | 62 | ## Architecture 63 | 64 | The intricacy of the running stateful datastores via Kubernetes Engine or K8s lies with 65 | the various Kubernetes manifests and pod configurations used. A container that 66 | has been customized to run as a pod is used as well. This demo includes multiple 67 | configured manifests and a specialized container for Cassandra. 68 | 69 | Many people, including Kelsey Hightower, have talked about how running stateful 70 | datestores inside of K8s is not trivial, and frankly quite complicated. If you 71 | can run a database on a managed service, do it. Let someone else wake up at 2 72 | am to fix your database. 73 | 74 | There are two possibilities when you run a stateful datastore on K8s. 75 | 76 | 1. You are very experienced K8s user and know exactly what is around the corner. 77 | 1. You have no idea what is around the corner, and you are going to learn very 78 | fast. 79 | 80 | ### Architecture Diagram 81 | 82 | The following diagram represents a Cassandra cluster deployed on Kubernetes Engine. 83 | 84 | ![Cassandra Running on Kubernetes Engine](images/gke-cassandra.png) 85 | 86 | ### Kubernetes Manifests and Controllers 87 | 88 | Various manifests and controllers are utilized to install Cassandra. The 89 | following sections outline the different types used. 90 | 91 | #### StatefulSets 92 | 93 | StatefulSets is the controller type that is used to install a Cassandra Ring on 94 | Kubernetes Engine. This controller manages the installation and scaling of a set of Pods and 95 | provides various features: 96 | 97 | 1. Stable, unique network identifiers. 98 | 1. Stable, persistent storage. 99 | 1. Ordered, graceful deployment and scaling. 100 | 1. Ordered, graceful deletion and termination. 101 | 1. Ordered, automated rolling updates. 102 | 1. Automated creation of storage volumes. 103 | 1. Stable guaranteed storage. 104 | 105 | Like a Deployment, a StatefulSet manages Pods that are based on an identical 106 | container spec. Unlike a Deployment, a StatefulSet maintains a sticky identity 107 | for each of their Pods. These pods are created from the same spec, but are not 108 | interchangeable: each has a persistent identifier that it maintains across any 109 | rescheduling. 110 | 111 | Cassandra utilizes all of these features in order to run on Kubernetes Engine. 112 | 113 | Find more information [here][2]. 114 | 115 | #### Persistent Storage with Persistent Volumes 116 | 117 | With a stateful datasource storage is required. 118 | 119 | A PersistentVolume (PV) is a piece of storage in the cluster that has been 120 | provisioned by an administrator, or automatically by a Stateful Set. It is a 121 | resource in the cluster just like a node is a cluster resource. PVs are volume 122 | plugins like Volumes, but have a lifecycle independent of any individual pod 123 | that uses the PV. 124 | 125 | A PersistentVolumeClaim (PVC) is a request for storage by a user. It is similar 126 | to a pod. Pods consume node resources and PVCs consume PV resources. Pods can 127 | request specific levels of resources (CPU and Memory). Claims can request 128 | specific size and access modes (e.g., can be mounted once read/write or many 129 | times read-only). 130 | 131 | Find more information [here][3]. 132 | 133 | #### Pod Disruptions Budgets 134 | 135 | An Application Owner can create a PodDisruptionBudget object (PDB) for each 136 | application. A PDB limits the number pods of a replicated application that are 137 | down simultaneously from voluntary disruptions. For example, with Cassandra 138 | we need to ensure that the number of replicas running is never brought below 139 | the number needed for a quorum. 140 | 141 | Find more information visit [here][4]. 142 | 143 | #### Pod Scheduling 144 | 145 | This proof of concept utilizes advanced scheduling for pod placement. Both 146 | scheduling anti-affinity rules and taints are used to instruct the K8s scheduler 147 | where to launch C* pods. 148 | 149 | Anti-Affinity rules instruct the scheduler to use preferred or required pod 150 | placement. Preferred rules provide a weight value that impacts the scheduling 151 | algorithm to not schedule C* pods on the same node. While required rules force 152 | the scheduler to not schedule C* pods on the same node. Not having multiple 153 | Cassandra pods on the same node increases fault tolerance, as when you lose a 154 | node you only lose one pod. But then you need to have an excess of nodes because 155 | a pod will not reschedule. Preferred does not provide the same level of high 156 | availability, but will allow pods to reschedule on existing nodes if headroom 157 | exists. 158 | 159 | Find more information visit [here][5]. 160 | 161 | Taints prevent pods to be scheduled on a node, unless the pod's manifest 162 | contains the required toleration. The typical use case for this is to target 163 | a specific node pool for say Cassandra pods. For instance, often larger machine 164 | types are required for C*, and adding taints to those nodes ensure that 165 | Cassandra pods will only be scheduled on the nodes. 166 | 167 | Find more information visit [here][6]. 168 | 169 | #### DNS and Headless Services 170 | 171 | The Cassandra ring that is installed as part of this demo is named "cassandra". 172 | The name of the service "cassandra" creates the DNS name, and the tld is 173 | "default.svc.cluster.local", where "default" is the name of the namespace. With 174 | an application you would connect to at least three of the nodes to provide an HA 175 | connection with a client. The name of the nodes, for example would be: 176 | 177 | ``` 178 | cassandra-0.cassandra.default.svc.cluster.local 179 | cassandra-1.cassandra.default.svc.cluster.local 180 | cassandra-1.cassandra.default.svc.cluster.local 181 | ``` 182 | 183 | For more information visit [here][7]. 184 | 185 | ### Cassandra Container 186 | 187 | Within this demo we have included a container that is specifically built for 188 | running on Kubernetes. Features: 189 | 190 | 1. OSS maintained base container that is used by K8s OSS project 191 | 1. Cassandra is installed from a tarball 192 | 1. Container size has been reduced 193 | 1. A readiness and liveness probes are built-in 194 | 1. Prometheus is installed, but the port is not exposed in the manifest by 195 | default 196 | 1. dumb-init is used to handle signal processing 197 | 1. A logging encoder is provided for JSON logging 198 | 1. Datacenter configuration 199 | 1. JMX configuration 200 | 1. Multiple ENV variables are exposed for manifest based configuration 201 | 202 | For more information about the ENV variables that have been exposed see 203 | [container/README.md](contain/README.md), 204 | [container/files/run.sh](container/files/run.sh) and also the 205 | [manifest](manifests/cassandra-statefulset.yaml). 206 | 207 | This container is hosted by Google Professional Services. 208 | 209 | ## Deployment Steps 210 | 211 | To deploy the demo execute the following commands. 212 | 213 | ```console 214 | git clone https://github.com/GoogleCloudPlatform/gke-stateful-applications-demo 215 | cd gke-stateful-applications-demo 216 | ./create.sh -c my-cluster-1 217 | ``` 218 | 219 | Replace the text 'my-cluster-1' the name of the cluster that you would like to 220 | create. 221 | 222 | The create script will output the following message when complete. 223 | 224 | ```console 225 | NAME MACHINE_TYPE DISK_SIZE_GB NODE_VERSION 226 | nodepool-cassdemo-2 n1-standard-1 100 1.10.2-gke.3 227 | ``` 228 | 229 | The script will: 230 | 231 | 1. create a new Kubernetes Engine cluster in your default ZONE, VPC and network. 232 | 1. install multiple Kubernetes manifests that can be found in the 233 | [manifests](./manifests) directory. 234 | 235 | Because we are creating a cluster of 6 nodes, the cluster may go into a 236 | 'RECONCILING' status as the control plane's instance size may be increased. 237 | 238 | Use the following command to view the current status of the cluster. 239 | 240 | ```console 241 | gcloud container clusters list 242 | ``` 243 | 244 | An example of the output while the cluster is reconciling. 245 | 246 | ```console 247 | NAME LOCATION MASTER_VERSION MASTER_IP MACHINE_TYPE NODE_VERSION NUM_NODES STATUS 248 | my-cluster-1 us-central1-a 1.10.2-gke.3 35.184.70.165 n1-standard-4 1.10.2-gke.3 6 RECONCILING 249 | ``` 250 | 251 | The status will change to RUNNING once the masters have been updated. 252 | 253 | ## Validation 254 | 255 | The following script will validate that the demo is deployed correctly: 256 | 257 | ```console 258 | ./validate.sh -c my-cluster-1 259 | ``` 260 | Replace the text 'my-cluster-1' the name of the cluster that you would like to 261 | validate. 262 | 263 | The validation script uses `kubectl rollout status` to test if the rollout is 264 | complete. If the cluster is in 'RECONCILING' state this script will fail as 265 | well. 266 | 267 | If the script fails it will output: 268 | 269 | ```console 270 | Validation Failed: Statefulset has not been deployed 271 | ``` 272 | 273 | If the script passes if will output: 274 | 275 | ```console 276 | Validation Passed: the Statefulset has been deployed 277 | ``` 278 | 279 | ### Using Cassandara 280 | 281 | These commands exercise the Cassandra cluster. 282 | 283 | ### Launch a Cassandra container 284 | 285 | These `kubectl` commands launch a K8s deployment, wait for the deployment, 286 | and exec into the shell. 287 | 288 | ```console 289 | kubectl run cass-dev --image gcr.io/pso-examples/cassandra:3.11.4-cqlsh-v22 --command -- /bin/bash -c "tail -f /dev/null" 290 | kubectl rollout status deployment cass-dev 291 | kubectl exec $(kubectl get po --no-headers | grep cass-dev | awk '{print $1}') -it -- /bin/bash 292 | ``` 293 | 294 | This will launch a bash prompt for the next steps. 295 | 296 | ### Connect to the ring with cqlsh 297 | 298 | #### cqlsh command 299 | 300 | ```console 301 | /usr/local/apache-cassandra/bin/cqlsh cassandra-0.cassandra.default.svc.cluster.local 302 | ``` 303 | 304 | You will now be using the interactive `cqlsh` shell. 305 | 306 | The output of the command: 307 | 308 | ```console 309 | Connected to K8Demo at cassandra-0.cassandra.default.svc.cluster.local:9042. 310 | [cqlsh 5.0.1 | Cassandra 3.11.4 | CQL spec 3.4.4 | Native protocol v4] 311 | Use HELP for help. 312 | cqlsh> 313 | ``` 314 | 315 | ### Create a keyspace 316 | 317 | ```console 318 | CREATE KEYSPACE greek_monsters WITH REPLICATION = { 'class' : 'SimpleStrategy' , 'replication_factor' : 3 }; 319 | ``` 320 | 321 | The user prompt is shown if the command is successful. 322 | 323 | ### Describe the keyspace 324 | 325 | ```console 326 | DESC greek_monsters; 327 | ``` 328 | 329 | Example output: 330 | 331 | ```console 332 | CREATE KEYSPACE greek_monsters WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} AND durable_writes = true; 333 | 334 | cqlsh> 335 | ``` 336 | 337 | ### Use the new keyspace 338 | 339 | ```console 340 | USE greek_monsters; 341 | ``` 342 | 343 | The prompt will now include the keyspace that is selected. 344 | 345 | ```console 346 | cqlsh:greek_monsters> 347 | ``` 348 | 349 | ### Create a table 350 | 351 | ```console 352 | CREATE TABLE monsters (pet_id timeuuid, name text, description text, PRIMARY KEY (pet_id)); 353 | ``` 354 | The user prompt is shown if the command is successful. 355 | 356 | ### See the newly created table 357 | 358 | ```console 359 | DESCRIBE TABLES; 360 | ``` 361 | 362 | This command will output: 363 | 364 | ```console 365 | monsters 366 | 367 | cqlsh:greek_monsters> 368 | ``` 369 | 370 | ### Insert data into the table 371 | 372 | ```console 373 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Cerberus (Hellhound)','The three-headed giant hound, that guarded the gates of the Underworld.'); 374 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Orthrus','A two-headed dog, brother of Cerberus, slain by Heracles.'); 375 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Argos','Odysseus faithful dog, known for his speed, strength and his superior tracking skills.'); 376 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Golden Dog','A dog which guarded the infant god Zeus.'); 377 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Guard Dog of Hephaestus Temple','The temple of Hephaestus at Mount Etna was guarded by a pack of sacred dogs.'); 378 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Laelaps',' female dog destined always to catch its prey.'); 379 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Maera','The hound of Erigone, daughter of Icarius of Athens.'); 380 | ``` 381 | 382 | The user prompt is shown if the command is successful. 383 | 384 | ### Select the data from the table 385 | 386 | ```console 387 | SELECT * from monsters ; 388 | ``` 389 | 390 | Example output: 391 | 392 | ```console 393 | pet_id | description | name 394 | --------------------------------------+----------------------------------------------------------------------------------------+-------------------------------- 395 | ca3d9f20-6a89-11e8-a0dd-114bb9e30b07 | Odysseus faithful dog, known for his speed, strength and his superior tracking skills. | Argos 396 | ca3c3f90-6a89-11e8-a0dd-114bb9e30b07 | A two-headed dog, brother of Cerberus, slain by Heracles. | Orthrus 397 | ca42f650-6a89-11e8-a0dd-114bb9e30b07 | The hound of Erigone, daughter of Icarius of Athens. | Maera 398 | ca3f4cd0-6a89-11e8-a0dd-114bb9e30b07 | A dog which guarded the infant god Zeus. | Golden Dog 399 | ca41bdd0-6a89-11e8-a0dd-114bb9e30b07 | female dog destined always to catch its prey. | Laelaps 400 | ca40ac60-6a89-11e8-a0dd-114bb9e30b07 | The temple of Hephaestus at Mount Etna was guarded by a pack of sacred dogs. | Guard Dog of Hephaestus Temple 401 | ca3a6ad0-6a89-11e8-a0dd-114bb9e30b07 | The three-headed giant hound, that guarded the gates of the Underworld. | Cerberus (Hellhound) 402 | ``` 403 | 404 | ### Exit 405 | 406 | Exit the pod with the following commands: 407 | 408 | ```console 409 | exit 410 | exit 411 | ``` 412 | 413 | This will return you to your command line. 414 | 415 | #### Delete the deployment 416 | 417 | Execute the following command to remove the deployment. 418 | 419 | ```console 420 | kubectl delete deployment cass-dev 421 | ``` 422 | 423 | The following message is displayed: 424 | 425 | ```console 426 | deployment "cass-dev" deleted 427 | ``` 428 | 429 | ## Tear Down 430 | 431 | The following script will tear down the Cassandra cluster and remove the Kubernetes Engine 432 | cluster. 433 | 434 | ```console 435 | ./delete.sh -c my-cluster-1 436 | ``` 437 | 438 | This output will change depending on the cluster name. In this example the name 439 | "my-cluster-1" was used. 440 | 441 | Replace the text 'my-cluster-1' the name of the cluster that you would like to 442 | validate. The last lines of the output will be: 443 | 444 | ```console 445 | persistentvolumeclaim "cassandra-data-cassandra-0" deleted 446 | persistentvolumeclaim "cassandra-data-cassandra-1" deleted 447 | persistentvolumeclaim "cassandra-data-cassandra-2" deleted 448 | Deleting cluster 449 | ``` 450 | 451 | The tear down script removes all of the components in the manifests directory, 452 | and it also destroys the cluster. This script also waits 60 seconds before 453 | it removes the PVC storage components. 454 | 455 | ## Troubleshooting 456 | 457 | 1. Since we are creating two nodepools in this demo, the cluster may upgrade 458 | the control plane and it may go to a "RECONCILING" state. Give the cluster 459 | some time and it will scale. The validate script will fail during this time. 460 | 2. Run "gcloud container clusters list" command to check the cluster status. 461 | Cluster 462 | 3. If you get errors about quotas, please increase your quota in the project. 463 | See [here][8] for more details. 464 | 4. A great diagnostic command to run is simply `kubectl get pods`. It will show 465 | the running pods. 466 | 467 | Initially, the cluster may show as "RUNNING" but then go into a "RECONCILING" state 468 | The symptom will be timeouts when running `kubectl get pods` 469 | Use the "gcloud container clusters list" command to check the latest state, and wait until it changes back to "RUNNING" 470 | 471 | 472 | ## Relevant Materials 473 | 474 | 1. [Thousand Instances of Cassandra][1] 475 | 2. [Stateful Sets][2] 476 | 3. [Persistent Volumes][3] 477 | 4. [Pod Disruption Budgets][4] 478 | 5. [Pod Scheduling][5] 479 | 6. [Taints and Tolerations][6] 480 | 7. [Headless Services][7] 481 | 8. [Google Cloud Quotas][8] 482 | 9. [Signup for Google Cloud][9] 483 | 10. [Google Cloud Shell][10] 484 | 485 | [1]: https://kubernetes.io/blog/2016/07/thousand-instances-of-cassandra-using-kubernetes-pet-set 486 | [2]: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/ 487 | [3]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ 488 | [4]: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ 489 | [5]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature 490 | [6]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ 491 | [7]: https://kubernetes.io/docs/concepts/services-networking/service/#headless-services 492 | [8]: https://cloud.google.com/compute/quotas 493 | [9]: https://cloud.google.com 494 | [10]: https://cloud.google.com/shell/docs/ 495 | 496 | **This is not an officially supported Google product** 497 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Google Kubernetes Engine Stateful Application Demo 2 | 3 | ## Table of Contents 4 | 5 | 6 | * [Google Kubernetes Engine Stateful Application Demo](#google-kubernetes-engine-stateful-application-demo) 7 | * [Introduction](#introduction) 8 | * [Architecture](#architecture) 9 | * [Architecture Diagram](#architecture-diagram) 10 | * [Kubernetes Manifests and Controllers](#kubernetes-manifests-and-controllers) 11 | * [StatefulSets](#statefulsets) 12 | * [Persistent Storage with Persistent Volumes](#persistent-storage-with-persistent-volumes) 13 | * [Pod Disruptions Budgets](#pod-disruptions-budgets) 14 | * [Pod Scheduling](#pod-scheduling) 15 | * [DNS and Headless Services](#dns-and-headless-services) 16 | * [Cassandra Container](#cassandra-container) 17 | * [Prerequisites](#prerequisites) 18 | * [Run Demo in a Google Cloud Shell](#run-demo-in-a-google-cloud-shell) 19 | * [Supported Operating Systems](#supported-operating-systems) 20 | * [Tools](#tools) 21 | * [Versions](#versions) 22 | * [Deployment Steps](#deployment-steps) 23 | * [Validation](#validation) 24 | * [Using Cassandara](#using-cassandara) 25 | * [Launch a Cassandra container](#launch-a-cassandra-container) 26 | * [Connect to the ring with cqlsh](#connect-to-the-ring-with-cqlsh) 27 | * [cqlsh command](#cqlsh-command) 28 | * [Create a keyspace](#create-a-keyspace) 29 | * [Describe the keyspace](#describe-the-keyspace) 30 | * [Use the new keyspace](#use-the-new-keyspace) 31 | * [Create a table](#create-a-table) 32 | * [See the newly created table](#see-the-newly-created-table) 33 | * [Insert data into the table](#insert-data-into-the-table) 34 | * [Select the data from the table](#select-the-data-from-the-table) 35 | * [Exit](#exit) 36 | * [Delete the deployment](#delete-the-deployment) 37 | * [Tear Down](#tear-down) 38 | * [Troubleshooting](#troubleshooting) 39 | * [Relevant Materials](#relevant-materials) 40 | 41 | 42 | ## Introduction 43 | 44 | This proof of concept deploys a 45 | [Kubernetes Engine](https://cloud.google.com/kubernetes-engine/) 46 | Cluster and then installs an [Apache Cassandra](http://cassandra.apache.org/) 47 | database running on that cluster. 48 | 49 | Various scripts are contained within this project that provide push button 50 | creation, validation, and deletion of the Cassandra(C*) database and Kubernetes 51 | Engine cluster. 52 | 53 | Apache Cassandra was chosen for various reasons. These reasons include that 54 | out of the box Cassandra functions well as a cloud native database. Moreover, 55 | this POC continues the work started with the Kubernetes example, and the blog 56 | post: [Thousand Instances of Cassandra using Kubernetes Pet Set][1]. 57 | 58 | When running a database using Kubernetes Engine, as an operator you need to be experienced with Kubernetes Engine 59 | and the datasource that you are running. If you are able to use a hosted 60 | solution then it is recommended that you use the hosted solution. On the other 61 | hand, if you are not able to run a datasource in a hosted solution then hosting 62 | it is Kubernetes Engine is doable. The challenge is not Kubernetes or Kubernetes Engine, but the challenge 63 | is migrating and operating a database in a container based system that can do 64 | things like moving pods/containers between nodes. 65 | 66 | ## Architecture 67 | 68 | The intricacy of the running stateful datastores via Kubernetes Engine or K8s lies with 69 | the various Kubernetes manifests and pod configurations used. A container that 70 | has been customized to run as a pod is used as well. This demo includes multiple 71 | configured manifests and a specialized container for Cassandra. 72 | 73 | Many people, including Kelsey Hightower, have talked about how running stateful 74 | datestores inside of K8s is not trivial, and frankly quite complicated. If you 75 | can run a database on a managed service, do it. Let someone else wake up at 2 76 | am to fix your database. 77 | 78 | There are two possibilities when you run a stateful datastore on K8s. 79 | 80 | 1. You are very experienced K8s user and know exactly what is around the corner. 81 | 1. You have no idea what is around the corner, and you are going to learn very 82 | fast. 83 | 84 | ### Architecture Diagram 85 | 86 | The following diagram represents a Cassandra cluster deployed on Kubernetes Engine. 87 | 88 | ![Cassandra Running on Kubernetes Engine](images/gke-cassandra.png) 89 | 90 | ### Kubernetes Manifests and Controllers 91 | 92 | Various manifests and controllers are utilized to install Cassandra. The 93 | following sections outline the different types used. 94 | 95 | #### StatefulSets 96 | 97 | StatefulSets is the controller type that is used to install a Cassandra Ring on 98 | Kubernetes Engine. This controller manages the installation and scaling of a set of Pods and 99 | provides various features: 100 | 101 | 1. Stable, unique network identifiers. 102 | 1. Stable, persistent storage. 103 | 1. Ordered, graceful deployment and scaling. 104 | 1. Ordered, graceful deletion and termination. 105 | 1. Ordered, automated rolling updates. 106 | 1. Automated creation of storage volumes. 107 | 1. Stable guaranteed storage. 108 | 109 | Like a Deployment, a StatefulSet manages Pods that are based on an identical 110 | container spec. Unlike a Deployment, a StatefulSet maintains a sticky identity 111 | for each of their Pods. These pods are created from the same spec, but are not 112 | interchangeable: each has a persistent identifier that it maintains across any 113 | rescheduling. 114 | 115 | Cassandra utilizes all of these features in order to run on Kubernetes Engine. 116 | 117 | Find more information [here][2]. 118 | 119 | #### Persistent Storage with Persistent Volumes 120 | 121 | With a stateful datasource storage is required. 122 | 123 | A PersistentVolume (PV) is a piece of storage in the cluster that has been 124 | provisioned by an administrator, or automatically by a Stateful Set. It is a 125 | resource in the cluster just like a node is a cluster resource. PVs are volume 126 | plugins like Volumes, but have a lifecycle independent of any individual pod 127 | that uses the PV. 128 | 129 | A PersistentVolumeClaim (PVC) is a request for storage by a user. It is similar 130 | to a pod. Pods consume node resources and PVCs consume PV resources. Pods can 131 | request specific levels of resources (CPU and Memory). Claims can request 132 | specific size and access modes (e.g., can be mounted once read/write or many 133 | times read-only). 134 | 135 | Find more information [here][3]. 136 | 137 | #### Pod Disruptions Budgets 138 | 139 | An Application Owner can create a PodDisruptionBudget object (PDB) for each 140 | application. A PDB limits the number pods of a replicated application that are 141 | down simultaneously from voluntary disruptions. For example, with Cassandra 142 | we need to ensure that the number of replicas running is never brought below 143 | the number needed for a quorum. 144 | 145 | Find more information visit [here][4]. 146 | 147 | #### Pod Scheduling 148 | 149 | This proof of concept utilizes advanced scheduling for pod placement. Both 150 | scheduling anti-affinity rules and taints are used to instruct the K8s scheduler 151 | where to launch C* pods. 152 | 153 | Anti-Affinity rules instruct the scheduler to use preferred or required pod 154 | placement. Preferred rules provide a weight value that impacts the scheduling 155 | algorithm to not schedule C* pods on the same node. While required rules force 156 | the scheduler to not schedule C* pods on the same node. Not having multiple 157 | Cassandra pods on the same node increases fault tolerance, as when you lose a 158 | node you only lose one pod. But then you need to have an excess of nodes because 159 | a pod will not reschedule. Preferred does not provide the same level of high 160 | availability, but will allow pods to reschedule on existing nodes if headroom 161 | exists. 162 | 163 | Find more information visit [here][5]. 164 | 165 | Taints prevent pods to be scheduled on a node, unless the pod's manifest 166 | contains the required toleration. The typical use case for this is to target 167 | a specific node pool for say Cassandra pods. For instance, often larger machine 168 | types are required for C*, and adding taints to those nodes ensure that 169 | Cassandra pods will only be scheduled on the nodes. 170 | 171 | Find more information visit [here][6]. 172 | 173 | #### DNS and Headless Services 174 | 175 | The Cassandra ring that is installed as part of this demo is named "cassandra". 176 | The name of the service "cassandra" creates the DNS name, and the tld is 177 | "default.svc.cluster.local", where "default" is the name of the namespace. With 178 | an application you would connect to at least three of the nodes to provide an HA 179 | connection with a client. The name of the nodes, for example would be: 180 | 181 | ``` 182 | cassandra-0.cassandra.default.svc.cluster.local 183 | cassandra-1.cassandra.default.svc.cluster.local 184 | cassandra-1.cassandra.default.svc.cluster.local 185 | ``` 186 | 187 | For more information visit [here][7]. 188 | 189 | ### Cassandra Container 190 | 191 | Within this demo we have included a container that is specifically built for 192 | running on Kubernetes. Features: 193 | 194 | 1. OSS maintained base container that is used by K8s OSS project 195 | 1. Cassandra is installed from a tarball 196 | 1. Container size has been reduced 197 | 1. A readiness and liveness probes are built-in 198 | 1. Prometheus is installed, but the port is not exposed in the manifest by 199 | default 200 | 1. dumb-init is used to handle signal processing 201 | 1. A logging encoder is provided for JSON logging 202 | 1. Datacenter configuration 203 | 1. JMX configuration 204 | 1. Multiple ENV variables are exposed for manifest based configuration 205 | 206 | For more information about the ENV variables that have been exposed see 207 | [container/README.md](contain/README.md), 208 | [container/files/run.sh](container/files/run.sh) and also the 209 | [manifest](manifests/cassandra-statefulset.yaml). 210 | 211 | This container is hosted by Google Professional Services. 212 | 213 | ## Prerequisites 214 | 215 | A Google Cloud account and project is required for this demo. The project must 216 | have the proper quota to run a Kubernetes Engine cluster with at least 217 | 3 n1-standard-4 and 3 n1-standard-1 nodes. Additionally, the project must have 218 | at least 30 Compute Engine API CPUs and 12 Compute Engine API In-use IP Addresses. 219 | 220 | Access to an existing Google Cloud project with the Kubernetes Engine service enabled 221 | If you do not have a Google Cloud account please signup for a free trial 222 | [here][9]. 223 | 224 | How to check your account's quota is documented here: [quotas][8]. 225 | 226 | ### Run Demo in a Google Cloud Shell 227 | 228 | Click the button below to run the demo in a [Google Cloud Shell][10]. 229 | 230 | [![Open in Cloud Shell](http://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/open?git_repo=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fgke-stateful-applications-demo&page=editor&tutorial=README.md) 231 | 232 | All the tools for the demo are installed. When using Cloud Shell execute the following 233 | command in order to setup gcloud cli. When executing this command please setup your region 234 | and zone. 235 | 236 | ```console 237 | gcloud init 238 | ``` 239 | 240 | ### Supported Operating Systems 241 | 242 | This project will run on macOS, or in a [Google Cloud Shell][10]. 243 | 244 | ### Tools 245 | 246 | When not using Cloud Shell, the following tools are required. 247 | 248 | 1. [Google Cloud SDK version >= 204.0.0](https://cloud.google.com/sdk/docs/downloads-versioned-archives) 249 | 2. [gcloud cli](https://cloud.google.com/sdk/gcloud/) 250 | 3. [kubectl matching the latest GKE version](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 251 | 4. docker (used to build container, you can use hosted container) 252 | 253 | More recent versions of all the tools may function, please feel free to file an 254 | issue if you encounter problems with newer versions. 255 | 256 | #### Install Cloud SDK 257 | The Google Cloud SDK is used to interact with your GCP resources. 258 | [Installation instructions](https://cloud.google.com/sdk/downloads) for multiple platforms are available online. 259 | 260 | #### Install kubectl CLI 261 | 262 | The kubectl CLI is used to interteract with both Kubernetes Engine and kubernetes in general. 263 | [Installation instructions](https://cloud.google.com/kubernetes-engine/docs/quickstart) 264 | for multiple platforms are available online. 265 | 266 | ### Configure Authentication 267 | 268 | The script will execute against your GCP environment and it will use your personal account to build out these resources. To setup the default account the script will use, run the following command to select the appropriate account: 269 | 270 | `gcloud auth login` 271 | 272 | ## Deployment Steps 273 | 274 | To deploy the demo execute the following commands. 275 | 276 | ```console 277 | git clone https://github.com/GoogleCloudPlatform/gke-stateful-applications-demo 278 | cd gke-stateful-applications-demo 279 | ./create.sh -c my-cluster-1 280 | ``` 281 | 282 | Replace the text 'my-cluster-1' the name of the cluster that you would like to 283 | create. 284 | 285 | The create script will output the following message when complete. 286 | 287 | ```console 288 | NAME MACHINE_TYPE DISK_SIZE_GB NODE_VERSION 289 | nodepool-cassdemo-2 n1-standard-1 100 1.10.2-gke.3 290 | ``` 291 | 292 | The script will: 293 | 294 | 1. create a new Kubernetes Engine cluster in your default ZONE, VPC and network. 295 | 1. install multiple Kubernetes manifests that can be found in the 296 | [manifests](./manifests) directory. 297 | 298 | Because we are creating a cluster of 6 nodes, the cluster may go into a 299 | 'RECONCILING' status as the control plane's instance size may be increased. 300 | 301 | Use the following command to view the current status of the cluster. 302 | 303 | ```console 304 | gcloud container clusters list 305 | ``` 306 | 307 | An example of the output while the cluster is reconciling. 308 | 309 | ```console 310 | NAME LOCATION MASTER_VERSION MASTER_IP MACHINE_TYPE NODE_VERSION NUM_NODES STATUS 311 | my-cluster-1 us-central1-a 1.10.2-gke.3 35.184.70.165 n1-standard-4 1.10.2-gke.3 6 RECONCILING 312 | ``` 313 | 314 | The status will change to RUNNING once the masters have been updated. 315 | 316 | ## Validation 317 | 318 | The following script will validate that the demo is deployed correctly: 319 | 320 | ```console 321 | ./validate.sh -c my-cluster-1 322 | ``` 323 | Replace the text 'my-cluster-1' the name of the cluster that you would like to 324 | validate. 325 | 326 | The validation script uses `kubectl rollout status` to test if the rollout is 327 | complete. If the cluster is in 'RECONCILING' state this script will fail as 328 | well. 329 | 330 | If the script fails it will output: 331 | 332 | ```console 333 | Validation Failed: Statefulset has not been deployed 334 | ``` 335 | 336 | If the script passes if will output: 337 | 338 | ```console 339 | Validation Passed: the Statefulset has been deployed 340 | ``` 341 | 342 | ### Using Cassandara 343 | 344 | These commands exercise the Cassandra cluster. 345 | 346 | ### Launch a Cassandra container 347 | 348 | These `kubectl` commands launch a K8s deployment, wait for the deployment, 349 | and exec into the shell. 350 | 351 | ```console 352 | kubectl run cass-dev --image gcr.io/pso-examples/cassandra:3.11.4-cqlsh-v22 --command -- /bin/bash -c "tail -f /dev/null" 353 | kubectl rollout status deployment cass-dev 354 | kubectl exec $(kubectl get po --no-headers | grep cass-dev | awk '{print $1}') -it -- /bin/bash 355 | ``` 356 | 357 | This will launch a bash prompt for the next steps. 358 | 359 | ### Connect to the ring with cqlsh 360 | 361 | #### cqlsh command 362 | 363 | ```console 364 | /usr/local/apache-cassandra/bin/cqlsh cassandra-0.cassandra.default.svc.cluster.local 365 | ``` 366 | 367 | You will now be using the interactive `cqlsh` shell. 368 | 369 | The output of the command: 370 | 371 | ```console 372 | Connected to K8Demo at cassandra-0.cassandra.default.svc.cluster.local:9042. 373 | [cqlsh 5.0.1 | Cassandra 3.11.4 | CQL spec 3.4.4 | Native protocol v4] 374 | Use HELP for help. 375 | cqlsh> 376 | ``` 377 | 378 | ### Create a keyspace 379 | 380 | ```console 381 | CREATE KEYSPACE greek_monsters WITH REPLICATION = { 'class' : 'SimpleStrategy' , 'replication_factor' : 3 }; 382 | ``` 383 | 384 | The user prompt is shown if the command is successful. 385 | 386 | ### Describe the keyspace 387 | 388 | ```console 389 | DESC greek_monsters; 390 | ``` 391 | 392 | Example output: 393 | 394 | ```console 395 | CREATE KEYSPACE greek_monsters WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} AND durable_writes = true; 396 | 397 | cqlsh> 398 | ``` 399 | 400 | ### Use the new keyspace 401 | 402 | ```console 403 | USE greek_monsters; 404 | ``` 405 | 406 | The prompt will now include the keyspace that is selected. 407 | 408 | ```console 409 | cqlsh:greek_monsters> 410 | ``` 411 | 412 | ### Create a table 413 | 414 | ```console 415 | CREATE TABLE monsters (pet_id timeuuid, name text, description text, PRIMARY KEY (pet_id)); 416 | ``` 417 | The user prompt is shown if the command is successful. 418 | 419 | ### See the newly created table 420 | 421 | ```console 422 | DESCRIBE TABLES; 423 | ``` 424 | 425 | This command will output: 426 | 427 | ```console 428 | monsters 429 | 430 | cqlsh:greek_monsters> 431 | ``` 432 | 433 | ### Insert data into the table 434 | 435 | ```console 436 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Cerberus (Hellhound)','The three-headed giant hound, that guarded the gates of the Underworld.'); 437 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Orthrus','A two-headed dog, brother of Cerberus, slain by Heracles.'); 438 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Argos','Odysseus faithful dog, known for his speed, strength and his superior tracking skills.'); 439 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Golden Dog','A dog which guarded the infant god Zeus.'); 440 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Guard Dog of Hephaestus Temple','The temple of Hephaestus at Mount Etna was guarded by a pack of sacred dogs.'); 441 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Laelaps',' female dog destined always to catch its prey.'); 442 | INSERT INTO monsters (pet_id,name,description) VALUES (now(), 'Maera','The hound of Erigone, daughter of Icarius of Athens.'); 443 | ``` 444 | 445 | The user prompt is shown if the command is successful. 446 | 447 | ### Select the data from the table 448 | 449 | ```console 450 | SELECT * from monsters ; 451 | ``` 452 | 453 | Example output: 454 | 455 | ```console 456 | pet_id | description | name 457 | --------------------------------------+----------------------------------------------------------------------------------------+-------------------------------- 458 | ca3d9f20-6a89-11e8-a0dd-114bb9e30b07 | Odysseus faithful dog, known for his speed, strength and his superior tracking skills. | Argos 459 | ca3c3f90-6a89-11e8-a0dd-114bb9e30b07 | A two-headed dog, brother of Cerberus, slain by Heracles. | Orthrus 460 | ca42f650-6a89-11e8-a0dd-114bb9e30b07 | The hound of Erigone, daughter of Icarius of Athens. | Maera 461 | ca3f4cd0-6a89-11e8-a0dd-114bb9e30b07 | A dog which guarded the infant god Zeus. | Golden Dog 462 | ca41bdd0-6a89-11e8-a0dd-114bb9e30b07 | female dog destined always to catch its prey. | Laelaps 463 | ca40ac60-6a89-11e8-a0dd-114bb9e30b07 | The temple of Hephaestus at Mount Etna was guarded by a pack of sacred dogs. | Guard Dog of Hephaestus Temple 464 | ca3a6ad0-6a89-11e8-a0dd-114bb9e30b07 | The three-headed giant hound, that guarded the gates of the Underworld. | Cerberus (Hellhound) 465 | ``` 466 | 467 | ### Exit 468 | 469 | Exit the pod with the following commands: 470 | 471 | ```console 472 | exit 473 | exit 474 | ``` 475 | 476 | This will return you to your command line. 477 | 478 | #### Delete the deployment 479 | 480 | Execute the following command to remove the deployment. 481 | 482 | ```console 483 | kubectl delete deployment cass-dev 484 | ``` 485 | 486 | The following message is displayed: 487 | 488 | ```console 489 | deployment "cass-dev" deleted 490 | ``` 491 | 492 | ## Tear Down 493 | 494 | The following script will tear down the Cassandra cluster and remove the Kubernetes Engine 495 | cluster. 496 | 497 | ```console 498 | ./delete.sh -c my-cluster-1 499 | ``` 500 | 501 | This output will change depending on the cluster name. In this example the name 502 | "my-cluster-1" was used. 503 | 504 | Replace the text 'my-cluster-1' the name of the cluster that you would like to 505 | validate. The last lines of the output will be: 506 | 507 | ```console 508 | persistentvolumeclaim "cassandra-data-cassandra-0" deleted 509 | persistentvolumeclaim "cassandra-data-cassandra-1" deleted 510 | persistentvolumeclaim "cassandra-data-cassandra-2" deleted 511 | Deleting cluster 512 | ``` 513 | 514 | The tear down script removes all of the components in the manifests directory, 515 | and it also destroys the cluster. This script also waits 60 seconds before 516 | it removes the PVC storage components. 517 | 518 | ## Troubleshooting 519 | 520 | 1. Since we are creating two nodepools in this demo, the cluster may upgrade 521 | the control plane and it may go to a "RECONCILING" state. Give the cluster 522 | some time and it will scale. The validate script will fail during this time. 523 | 2. Run "gcloud container clusters list" command to check the cluster status. 524 | Cluster 525 | 3. If you get errors about quotas, please increase your quota in the project. 526 | See [here][8] for more details. 527 | 4. A great diagnostic command to run is simply `kubectl get pods`. It will show 528 | the running pods. 529 | 530 | Initially, the cluster may show as "RUNNING" but then go into a "RECONCILING" state 531 | The symptom will be timeouts when running `kubectl get pods` 532 | Use the "gcloud container clusters list" command to check the latest state, and wait until it changes back to "RUNNING" 533 | 534 | 535 | ## Relevant Materials 536 | 537 | 1. [Thousand Instances of Cassandra][1] 538 | 2. [Stateful Sets][2] 539 | 3. [Persistent Volumes][3] 540 | 4. [Pod Disruption Budgets][4] 541 | 5. [Pod Scheduling][5] 542 | 6. [Taints and Tolerations][6] 543 | 7. [Headless Services][7] 544 | 8. [Google Cloud Quotas][8] 545 | 9. [Signup for Google Cloud][9] 546 | 10. [Google Cloud Shell][10] 547 | 548 | [1]: https://kubernetes.io/blog/2016/07/thousand-instances-of-cassandra-using-kubernetes-pet-set 549 | [2]: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/ 550 | [3]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ 551 | [4]: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ 552 | [5]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature 553 | [6]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ 554 | [7]: https://kubernetes.io/docs/concepts/services-networking/service/#headless-services 555 | [8]: https://cloud.google.com/compute/quotas 556 | [9]: https://cloud.google.com 557 | [10]: https://cloud.google.com/shell/docs/ 558 | 559 | **This is not an officially supported Google product** 560 | -------------------------------------------------------------------------------- /common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # "---------------------------------------------------------" 18 | # "- -" 19 | # "- Common commands for all scripts -" 20 | # "- -" 21 | # "---------------------------------------------------------" 22 | 23 | # gcloud and kubectl are required for this POC 24 | command -v gcloud >/dev/null 2>&1 || { \ 25 | echo >&2 "I require gcloud but it's not installed. Aborting."; exit 1; } 26 | command -v kubectl >/dev/null 2>&1 || { \ 27 | echo >&2 "I require kubectl but it's not installed. Aborting."; exit 1; } 28 | 29 | usage() { echo "Usage: $0 [-c ]" 1>&2; exit 1; } 30 | 31 | # parse -c flag for the CLUSTER_NAME using getopts 32 | while getopts ":c:" opt; do 33 | case ${opt} in 34 | c) 35 | CLUSTER_NAME=$OPTARG 36 | ;; 37 | \?) 38 | echo "Invalid flag on command line: $OPTARG" 1>&2 39 | ;; 40 | *) 41 | usage 42 | ;; 43 | esac 44 | done 45 | shift $((OPTIND -1)) 46 | 47 | # If user did not pass in -c flag then fail 48 | if [ -z "${CLUSTER_NAME}" ]; then 49 | usage 50 | fi 51 | 52 | # Get the default zone and use it or die 53 | ZONE=$(gcloud config get-value compute/zone) 54 | if [ -z "${ZONE}" ]; then 55 | echo "gcloud cli must be configured with a default zone." 1>&2 56 | echo "run 'gcloud config set compute/zone ZONE'." 1>&2 57 | echo "replace 'ZONE' with the zone name like us-west1-a." 1>&2 58 | exit 1; 59 | fi 60 | 61 | #Get the default region and use it or die 62 | REGION=$(gcloud config get-value compute/region) 63 | if [ -z "${REGION}" ]; then 64 | echo "gcloud cli must be configured with a default region." 1>&2 65 | echo "run 'gcloud config set compute/region REGION'." 1>&2 66 | echo "replace 'REGION' with the region name like us-west1." 1>&2 67 | exit 1; 68 | fi 69 | 70 | # Get a comma separated list of zones from the default region 71 | ZONESINREGION="" 72 | for FILTEREDZONE in $(gcloud compute zones list --filter="region:$REGION" --format="value(name)" --limit 3) 73 | do 74 | # Get a least 3 zones to run 3 nodes in 75 | ZONESINREGION+="$FILTEREDZONE," 76 | done 77 | #Remove the last comma from the starting 78 | ZONESINREGION=${ZONESINREGION%?} 79 | -------------------------------------------------------------------------------- /container/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM gcr.io/google-containers/debian-base-amd64:0.3 16 | 17 | ARG BUILD_DATE 18 | ARG VCS_REF 19 | ARG CASSANDRA_VERSION 20 | ARG CQLSH_CONTAINER 21 | 22 | LABEL \ 23 | org.label-schema.build-date=$BUILD_DATE \ 24 | org.label-schema.docker.dockerfile="/Dockerfile" \ 25 | org.label-schema.license="Apache License 2.0" \ 26 | org.label-schema.name="Cassandra container optimized for Kubernetes" \ 27 | org.label-schema.url="https://github.com/GoogleCloudPlatform/" \ 28 | org.label-schema.vcs-ref=$VCS_REF \ 29 | org.label-schema.vcs-type="Git" \ 30 | org.label-schema.vcs-url="https://github.com/GoogleCloudPlatform/gke-stateful-applications-demo" 31 | 32 | ENV \ 33 | CASSANDRA_CONF=/etc/cassandra \ 34 | CASSANDRA_DATA=/var/lib/cassandra \ 35 | CASSANDRA_LOGS=/var/log/cassandra \ 36 | CASSANDRA_RELEASE=3.11.3 \ 37 | CASSANDRA_SHA=d82e0670cb41b091e88fff55250ce945c4ea026c87a5517d3cf7b6b351d5e2ba \ 38 | JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ 39 | DI_VERSION=1.2.1 \ 40 | DI_SHA=057ecd4ac1d3c3be31f82fc0848bf77b1326a975b4f8423fe31607205a0fe945 \ 41 | JOLOKIA_VERSION=1.5.0 \ 42 | JOLOKIA_SHA=cd7e20a2887e013873d7321cea1e6bf6bd6ffcdd3cd3968d6950edd8d79bbfb8 \ 43 | PROMETHEUS_VERSION=0.10 \ 44 | PROMETHEUS_SHA=b144a5a22fc9ee62d8d198f0dcc622f851c77cf52fee4bd529afbc266af37e29 \ 45 | LOGENCODER_VERSION=4.10-SNAPSHOT \ 46 | LOGENCODER_SHA=89be27bea7adc05b68c052a27b08c594a9f8e354185acbfd7a7b5f04c7cd9e20 47 | 48 | COPY files / 49 | 50 | # hadolint ignore=DL3008,DL4006 51 | RUN \ 52 | set -ex \ 53 | && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \ 54 | && export CASSANDRA_VERSION=${CASSANDRA_VERSION:-$CASSANDRA_RELEASE} \ 55 | && export CASSANDRA_HOME=/usr/local/apache-cassandra-${CASSANDRA_VERSION} \ 56 | && apt-get update && apt-get -qq -y --force-yes install --no-install-recommends \ 57 | bash \ 58 | openjdk-8-jre-headless \ 59 | libjemalloc1 \ 60 | localepurge \ 61 | wget \ 62 | jq \ 63 | && wget -q -O - "http://search.maven.org/remotecontent?filepath=io/prometheus/jmx/jmx_prometheus_javaagent/${PROMETHEUS_VERSION}/jmx_prometheus_javaagent-${PROMETHEUS_VERSION}.jar" > /usr/local/share/prometheus-agent.jar \ 64 | && echo "$PROMETHEUS_SHA /usr/local/share/prometheus-agent.jar" | sha256sum -c - \ 65 | && wget -q -O - "http://search.maven.org/remotecontent?filepath=org/jolokia/jolokia-jvm/${JOLOKIA_VERSION}/jolokia-jvm-${JOLOKIA_VERSION}-agent.jar" > /usr/local/share/jolokia-agent.jar \ 66 | && echo "$JOLOKIA_SHA /usr/local/share/jolokia-agent.jar" | sha256sum -c - \ 67 | && mirror_url=$( wget -q -O - 'https://www.apache.org/dyn/closer.cgi?as_json=1' | jq --raw-output '.preferred' ) \ 68 | && wget -q -O - "${mirror_url}cassandra/${CASSANDRA_VERSION}/apache-cassandra-${CASSANDRA_VERSION}-bin.tar.gz" > /usr/local/apache-cassandra-bin.tar.gz \ 69 | && echo "$CASSANDRA_SHA /usr/local/apache-cassandra-bin.tar.gz" | sha256sum - \ 70 | && tar -xzf /usr/local/apache-cassandra-bin.tar.gz -C /usr/local \ 71 | && rm /usr/local/apache-cassandra-bin.tar.gz \ 72 | && ln -s $CASSANDRA_HOME /usr/local/apache-cassandra \ 73 | # TODO stage this in gcr 74 | && wget -q -O - "https://github.com/mstump/logstash-logback-encoder/releases/download/${LOGENCODER_VERSION}/logstash-logback-encoder-${LOGENCODER_VERSION}.jar" > /usr/local/apache-cassandra/lib/log-encoder.jar \ 75 | && echo "$LOGENCODER_SHA /usr/local/apache-cassandra/lib/log-encoder.jar" | sha256sum -c - \ 76 | && wget -q -O - https://github.com/Yelp/dumb-init/releases/download/v${DI_VERSION}/dumb-init_${DI_VERSION}_amd64 > /sbin/dumb-init \ 77 | && echo "$DI_SHA /sbin/dumb-init" | sha256sum -c - \ 78 | && adduser --disabled-password --no-create-home --gecos '' --disabled-login cassandra \ 79 | && mkdir -p /var/lib/cassandra/ /var/log/cassandra/ /etc/cassandra/triggers \ 80 | && chmod +x /sbin/dumb-init /ready-probe.sh \ 81 | && mv /logback-stdout.xml /logback-json-files.xml /logback-json-stdout.xml /logback-files.xml /cassandra.yaml /jvm.options /prometheus.yaml /etc/cassandra/ \ 82 | && mv /usr/local/apache-cassandra/conf/cassandra-env.sh /etc/cassandra/ \ 83 | && chown cassandra: /ready-probe.sh \ 84 | && if [ -n "$CQLSH_CONTAINER" ]; then apt-get -y --no-install-recommends install python; else rm -rf $CASSANDRA_HOME/pylib; fi \ 85 | && apt-get -y purge wget jq localepurge \ 86 | && apt-get -y autoremove \ 87 | && apt-get clean \ 88 | && rm -rf \ 89 | $CASSANDRA_HOME/*.txt \ 90 | $CASSANDRA_HOME/doc \ 91 | $CASSANDRA_HOME/javadoc \ 92 | $CASSANDRA_HOME/tools/*.yaml \ 93 | $CASSANDRA_HOME/tools/bin/*.bat \ 94 | $CASSANDRA_HOME/bin/*.bat \ 95 | doc \ 96 | man \ 97 | info \ 98 | locale \ 99 | common-licenses \ 100 | ~/.bashrc \ 101 | /var/lib/apt/lists/* \ 102 | /var/log/**/* \ 103 | /var/cache/debconf/* \ 104 | /etc/systemd \ 105 | /lib/lsb \ 106 | /lib/udev \ 107 | /usr/share/doc/ \ 108 | /usr/share/doc-base/ \ 109 | /usr/share/man/ \ 110 | /tmp/* \ 111 | /usr/lib/jvm/java-8-openjdk-amd64/jre/plugin \ 112 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/javaws \ 113 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/jjs \ 114 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/orbd \ 115 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/pack200 \ 116 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/policytool \ 117 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/rmid \ 118 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/rmiregistry \ 119 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/servertool \ 120 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/tnameserv \ 121 | /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/unpack200 \ 122 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/javaws.jar \ 123 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/deploy* \ 124 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/desktop \ 125 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/*javafx* \ 126 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/*jfx* \ 127 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libdecora_sse.so \ 128 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libprism_*.so \ 129 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libfxplugins.so \ 130 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libglass.so \ 131 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libgstreamer-lite.so \ 132 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjavafx*.so \ 133 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjfx*.so \ 134 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/jfxrt.jar \ 135 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/nashorn.jar \ 136 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/oblique-fonts \ 137 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/plugin.jar \ 138 | /usr/lib/jvm/java-8-openjdk-amd64/jre/man \ 139 | /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/images \ 140 | /usr/lib/jvm/java-8-openjdk-amd64/man \ 141 | /usr/lib/jvm/java-8-openjdk-amd64/jre/THIRD_PARTY_README \ 142 | /usr/lib/jvm/java-8-openjdk-amd64/jre/ASSEMBLY_EXCEPTION 143 | 144 | VOLUME ["/var/lib/cassandra"] 145 | 146 | # 1234: prometheus jmx_exporter 147 | # 7000: intra-node communication 148 | # 7001: TLS intra-node communication 149 | # 7199: JMX 150 | # 9042: CQL 151 | # 9160: thrift service 152 | # 8778: jolokia port 153 | EXPOSE 1234 7000 7001 7199 9042 9160 8778 154 | 155 | CMD ["/sbin/dumb-init", "/bin/bash", "/run.sh"] 156 | -------------------------------------------------------------------------------- /container/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | VERSION?=3.11.4 16 | REV?=v22 17 | PROJECT_ID?=pso-examples 18 | PROJECT?=gcr.io/${PROJECT_ID} 19 | CASSANDRA_HOST_IP?=$(strip $(shell ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p'|tail -n 1|awk '{print $1}')) 20 | 21 | all: build 22 | 23 | docker: 24 | docker build --no-cache=true --compress --squash --build-arg "CASSANDRA_VERSION=${VERSION}" -t ${PROJECT}/cassandra:${VERSION}-${REV} . 25 | 26 | docker-cqlsh: 27 | docker build --pull --build-arg "CASSANDRA_VERSION=${VERSION}" --build-arg "CQLSH_CONTAINER=1" -t ${PROJECT}/cassandra:${VERSION}-cqlsh-${REV} . 28 | 29 | docker-cached: 30 | docker build --compress --squash --build-arg "CASSANDRA_VERSION=${VERSION}" -t ${PROJECT}/cassandra:${VERSION}-${REV} . 31 | 32 | build: docker 33 | 34 | build-cqlsh: docker-cqlsh 35 | 36 | build-cached: docker-cached 37 | 38 | push: build build-cqlsh 39 | docker push ${PROJECT}/cassandra:${VERSION}-${REV} 40 | docker push ${PROJECT}/cassandra:${VERSION}-cqlsh-${REV} 41 | 42 | run: build-cached 43 | docker run -i -t --rm \ 44 | -e CASSANDRA_SEEDS='172.17.0.2' \ 45 | -e CASSANDRA_MEMTABLE_FLUSH_WRITERS=1 \ 46 | ${PROJECT}/cassandra:${VERSION} 47 | 48 | shell: build-cached 49 | docker run -i -t --rm \ 50 | -e CASSANDRA_SEEDS='172.17.0.2' \ 51 | -e CASSANDRA_MEMTABLE_FLUSH_WRITERS=1 \ 52 | ${PROJECT}/cassandra:${VERSION} \ 53 | /bin/bash 54 | 55 | push-cqlsh: build-cqlsh 56 | docker push ${PROJECT}/cassandra:${VERSION}-cqlsh-${REV} 57 | 58 | push-all: build build-cqlsh push push-cqlsh 59 | 60 | .PHONY: all build push docker docker-cqlsh build-cqlsh push push-all 61 | -------------------------------------------------------------------------------- /container/README.md: -------------------------------------------------------------------------------- 1 | # Cassandra Container for Kubernetes 2 | 3 | This project provides a container optimized to run Apache Cassandra on Kubernetes. 4 | Two containers are hosted; one without cqlsh or python, and another with cqlsh and python. 5 | 6 | The containers are available via: 7 | 8 | ```console 9 | docker pull gcr.io/pso-examples/cassandra:3.11.4-v22 10 | ``` 11 | Or 12 | ```console 13 | docker pull gcr.io/pso-examples/cassandra:3.11.4-cqlsh-v22 14 | ``` 15 | 16 | ## Building via Makefile 17 | 18 | The projects Makefile contains various targets for building and pushing both 19 | the production container and the development container. 20 | 21 | ### Container without cqlsh or python 22 | 23 | Use the default target. The example below also sets the docker repository name 24 | and the Cassandra version. See the top of the Makefile for other variables that 25 | can be set. 26 | 27 | ```console 28 | PROJECT=registry-url make 29 | ``` 30 | 31 | ### Container with cqlsh and python 32 | 33 | The following command builds the container which includes a working 34 | version of `cqlsh`. 35 | 36 | ```console 37 | make build-cqlsh 38 | ``` 39 | 40 | ## Configuring Cassandra 41 | 42 | The `run.sh` bash script in the [files](files) folder has many options that allow 43 | the container to run in and outside of Kubernetes. The setting of Configuration 44 | values are driven off of Environment Variables key value combination. For example 45 | `CASSANDRA_BROADCAST_ADDRESS` default to the value of `hostname -i`, but can 46 | be overriden. Please refer to [run.sh](files/runs.sh) for the full list of values. 47 | These values are either added to such things a the cassandra.yaml file, logging 48 | setup or GC values in jvm.options. These files are bundled and versioned in the 49 | files folder as well. 50 | 51 | Besides the base values one can pass in Environment Variables that named with 52 | the prefix `CASSANDRA_YAML_`. Any env var with that prefix is parsed and the value 53 | sets the corresponding YAML value in the `cassandra.yaml` configuration file. 54 | 55 | For example: 56 | 57 | 1. set an env var `CASSANDRA_YAML_phi_convict_threshold=10` 58 | 1. run.sh replaces the line `# phi_convict_threshold: 8" with "phi_convict_threshold: 10` 59 | 60 | ## Ready Probe 61 | 62 | The [ready probe](files/ready-probe.sh) file is used by Kubernetes to check the 63 | readiness of the container. 64 | -------------------------------------------------------------------------------- /container/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | steps: 16 | - name: 'gcr.io/cloud-builders/docker' 17 | args: [ 'build', '--build-arg', 'CQLSH_CONTAINER=1', '--build-arg', 'CASSANDRA_VERSION=${_CASSANDRA_VERSION}', '--no-cache=true', '--compress', '-t', 'gcr.io/pso-examples/cassandra:${_CASSANDRA_VERSION}-cqlsh-${_REV_}', '.' ] 18 | - name: 'gcr.io/cloud-builders/docker' 19 | args: [ 'build', '--build-arg', 'CASSANDRA_VERSION=${_CASSANDRA_VERSION}', '--no-cache=true', '--compress', '-t', 'gcr.io/pso-examples/cassandra:${_CASSANDRA_VERSION}-${_REV_}', '.' ] 20 | substitutions: 21 | _CASSANDRA_VERSION: '3.11.4' 22 | _REV_: 'v22' 23 | images: [ 24 | 'gcr.io/pso-examples/cassandra:${_CASSANDRA_VERSION}-cqlsh-${_REV_}', 25 | 'gcr.io/pso-examples/cassandra:${_CASSANDRA_VERSION}-${_REV_}' 26 | ] 27 | -------------------------------------------------------------------------------- /container/files/cassandra.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Cassandra storage config YAML 16 | 17 | # NOTE: 18 | # See http://wiki.apache.org/cassandra/StorageConfiguration for 19 | # full explanations of configuration directives 20 | # /NOTE 21 | 22 | # The name of the cluster. This is mainly used to prevent machines in 23 | # one logical cluster from joining another. 24 | cluster_name: 'Test Cluster' 25 | 26 | # This defines the number of tokens randomly assigned to this node on the ring 27 | # The more tokens, relative to other nodes, the larger the proportion of data 28 | # that this node will store. You probably want all nodes to have the same number 29 | # of tokens assuming they have equal hardware capability. 30 | # 31 | # If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility, 32 | # and will use the initial_token as described below. 33 | # 34 | # Specifying initial_token will override this setting on the node's initial start, 35 | # on subsequent starts, this setting will apply even if initial token is set. 36 | # 37 | # If you already have a cluster with 1 token per node, and wish to migrate to 38 | # multiple tokens per node, see http://wiki.apache.org/cassandra/Operations 39 | num_tokens: 32 40 | 41 | # initial_token allows you to specify tokens manually. While you can use # it with 42 | # vnodes (num_tokens > 1, above) -- in which case you should provide a 43 | # comma-separated list -- it's primarily used when adding nodes # to legacy clusters 44 | # that do not have vnodes enabled. 45 | # initial_token: 46 | 47 | # See http://wiki.apache.org/cassandra/HintedHandoff 48 | # May either be "true" or "false" to enable globally, or contain a list 49 | # of data centers to enable per-datacenter. 50 | # hinted_handoff_enabled: DC1,DC2 51 | hinted_handoff_enabled: true 52 | # this defines the maximum amount of time a dead host will have hints 53 | # generated. After it has been dead this long, new hints for it will not be 54 | # created until it has been seen alive and gone down again. 55 | max_hint_window_in_ms: 10800000 # 3 hours 56 | # Maximum throttle in KBs per second, per delivery thread. This will be 57 | # reduced proportionally to the number of nodes in the cluster. (If there 58 | # are two nodes in the cluster, each delivery thread will use the maximum 59 | # rate; if there are three, each will throttle to half of the maximum, 60 | # since we expect two nodes to be delivering hints simultaneously.) 61 | hinted_handoff_throttle_in_kb: 1024 62 | # Number of threads with which to deliver hints; 63 | # Consider increasing this number when you have multi-dc deployments, since 64 | # cross-dc handoff tends to be slower 65 | max_hints_delivery_threads: 2 66 | 67 | # Maximum throttle in KBs per second, total. This will be 68 | # reduced proportionally to the number of nodes in the cluster. 69 | batchlog_replay_throttle_in_kb: 1024 70 | 71 | # Authentication backend, implementing IAuthenticator; used to identify users 72 | # Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator, 73 | # PasswordAuthenticator}. 74 | # 75 | # - AllowAllAuthenticator performs no checks - set it to disable authentication. 76 | # - PasswordAuthenticator relies on username/password pairs to authenticate 77 | # users. It keeps usernames and hashed passwords in system_auth.credentials table. 78 | # Please increase system_auth keyspace replication factor if you use this authenticator. 79 | authenticator: AllowAllAuthenticator 80 | 81 | # Authorization backend, implementing IAuthorizer; used to limit access/provide permissions 82 | # Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer, 83 | # CassandraAuthorizer}. 84 | # 85 | # - AllowAllAuthorizer allows any action to any user - set it to disable authorization. 86 | # - CassandraAuthorizer stores permissions in system_auth.permissions table. Please 87 | # increase system_auth keyspace replication factor if you use this authorizer. 88 | authorizer: AllowAllAuthorizer 89 | 90 | # Validity period for permissions cache (fetching permissions can be an 91 | # expensive operation depending on the authorizer, CassandraAuthorizer is 92 | # one example). Defaults to 2000, set to 0 to disable. 93 | # Will be disabled automatically for AllowAllAuthorizer. 94 | permissions_validity_in_ms: 2000 95 | 96 | # Refresh interval for permissions cache (if enabled). 97 | # After this interval, cache entries become eligible for refresh. Upon next 98 | # access, an async reload is scheduled and the old value returned until it 99 | # completes. If permissions_validity_in_ms is non-zero, then this must be 100 | # also. 101 | # Defaults to the same value as permissions_validity_in_ms. 102 | # permissions_update_interval_in_ms: 1000 103 | 104 | # The partitioner is responsible for distributing groups of rows (by 105 | # partition key) across nodes in the cluster. You should leave this 106 | # alone for new clusters. The partitioner can NOT be changed without 107 | # reloading all data, so when upgrading you should set this to the 108 | # same partitioner you were already using. 109 | # 110 | # Besides Murmur3Partitioner, partitioners included for backwards 111 | # compatibility include RandomPartitioner, ByteOrderedPartitioner, and 112 | # OrderPreservingPartitioner. 113 | # 114 | partitioner: org.apache.cassandra.dht.Murmur3Partitioner 115 | 116 | # Directories where Cassandra should store data on disk. Cassandra 117 | # will spread data evenly across them, subject to the granularity of 118 | # the configured compaction strategy. 119 | # If not set, the default directory is $CASSANDRA_HOME/data/data. 120 | data_file_directories: 121 | - /var/lib/cassandra/data 122 | 123 | # commit log. when running on magnetic HDD, this should be a 124 | # separate spindle than the data directories. 125 | # If not set, the default directory is $CASSANDRA_HOME/data/commitlog. 126 | commitlog_directory: /var/lib/cassandra/commitlog 127 | 128 | # policy for data disk failures: 129 | # die: shut down gossip and client transports and kill the JVM for any fs errors or 130 | # single-sstable errors, so the node can be replaced. 131 | # stop_paranoid: shut down gossip and client transports even for single-sstable errors, 132 | # kill the JVM for errors during startup. 133 | # stop: shut down gossip and client transports, leaving the node effectively dead, but 134 | # can still be inspected via JMX, kill the JVM for errors during startup. 135 | # best_effort: stop using the failed disk and respond to requests based on 136 | # remaining available sstables. This means you WILL see obsolete 137 | # data at CL.ONE! 138 | # ignore: ignore fatal errors and let requests fail, as in pre-1.2 Cassandra 139 | disk_failure_policy: stop 140 | 141 | # policy for commit disk failures: 142 | # die: shut down gossip and Thrift and kill the JVM, so the node can be replaced. 143 | # stop: shut down gossip and Thrift, leaving the node effectively dead, but 144 | # can still be inspected via JMX. 145 | # stop_commit: shutdown the commit log, letting writes collect but 146 | # continuing to service reads, as in pre-2.0.5 Cassandra 147 | # ignore: ignore fatal errors and let the batches fail 148 | commit_failure_policy: stop 149 | 150 | # Maximum size of the key cache in memory. 151 | # 152 | # Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the 153 | # minimum, sometimes more. The key cache is fairly tiny for the amount of 154 | # time it saves, so it's worthwhile to use it at large numbers. 155 | # The row cache saves even more time, but must contain the entire row, 156 | # so it is extremely space-intensive. It's best to only use the 157 | # row cache if you have hot rows or static rows. 158 | # 159 | # NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. 160 | # 161 | # Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache. 162 | key_cache_size_in_mb: 163 | 164 | # Duration in seconds after which Cassandra should 165 | # save the key cache. Caches are saved to saved_caches_directory as 166 | # specified in this configuration file. 167 | # 168 | # Saved caches greatly improve cold-start speeds, and is relatively cheap in 169 | # terms of I/O for the key cache. Row cache saving is much more expensive and 170 | # has limited use. 171 | # 172 | # Default is 14400 or 4 hours. 173 | key_cache_save_period: 14400 174 | 175 | # Number of keys from the key cache to save 176 | # Disabled by default, meaning all keys are going to be saved 177 | # key_cache_keys_to_save: 100 178 | 179 | # Maximum size of the row cache in memory. 180 | # NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. 181 | # 182 | # Default value is 0, to disable row caching. 183 | row_cache_size_in_mb: 0 184 | 185 | # Duration in seconds after which Cassandra should 186 | # save the row cache. Caches are saved to saved_caches_directory as specified 187 | # in this configuration file. 188 | # 189 | # Saved caches greatly improve cold-start speeds, and is relatively cheap in 190 | # terms of I/O for the key cache. Row cache saving is much more expensive and 191 | # has limited use. 192 | # 193 | # Default is 0 to disable saving the row cache. 194 | row_cache_save_period: 0 195 | 196 | # Number of keys from the row cache to save 197 | # Disabled by default, meaning all keys are going to be saved 198 | # row_cache_keys_to_save: 100 199 | 200 | # Maximum size of the counter cache in memory. 201 | # 202 | # Counter cache helps to reduce counter locks' contention for hot counter cells. 203 | # In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before 204 | # write entirely. With RF > 1 a counter cache hit will still help to reduce the duration 205 | # of the lock hold, helping with hot counter cell updates, but will not allow skipping 206 | # the read entirely. Only the local (clock, count) tuple of a counter cell is kept 207 | # in memory, not the whole counter, so it's relatively cheap. 208 | # 209 | # NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. 210 | # 211 | # Default value is empty to make it "auto" (min(2.5% of Heap (in MB), 50MB)). Set to 0 to disable counter cache. 212 | # NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache. 213 | counter_cache_size_in_mb: 214 | 215 | # Duration in seconds after which Cassandra should 216 | # save the counter cache (keys only). Caches are saved to saved_caches_directory as 217 | # specified in this configuration file. 218 | # 219 | # Default is 7200 or 2 hours. 220 | counter_cache_save_period: 7200 221 | 222 | # Number of keys from the counter cache to save 223 | # Disabled by default, meaning all keys are going to be saved 224 | # counter_cache_keys_to_save: 100 225 | 226 | # The off-heap memory allocator. Affects storage engine metadata as 227 | # well as caches. Experiments show that JEMAlloc saves some memory 228 | # than the native GCC allocator (i.e., JEMalloc is more 229 | # fragmentation-resistant). 230 | # 231 | # Supported values are: NativeAllocator, JEMallocAllocator 232 | # 233 | # If you intend to use JEMallocAllocator you have to install JEMalloc as library and 234 | # modify cassandra-env.sh as directed in the file. 235 | # 236 | # Defaults to NativeAllocator 237 | # memory_allocator: NativeAllocator 238 | 239 | # saved caches 240 | # If not set, the default directory is $CASSANDRA_HOME/data/saved_caches. 241 | saved_caches_directory: /var/lib/cassandra/saved_caches 242 | 243 | # commitlog_sync may be either "periodic" or "batch." 244 | # 245 | # When in batch mode, Cassandra won't ack writes until the commit log 246 | # has been fsynced to disk. It will wait 247 | # commitlog_sync_batch_window_in_ms milliseconds between fsyncs. 248 | # This window should be kept short because the writer threads will 249 | # be unable to do extra work while waiting. (You may need to increase 250 | # concurrent_writes for the same reason.) 251 | # 252 | # commitlog_sync: batch 253 | # commitlog_sync_batch_window_in_ms: 2 254 | # 255 | # the other option is "periodic" where writes may be acked immediately 256 | # and the CommitLog is simply synced every commitlog_sync_period_in_ms 257 | # milliseconds. 258 | commitlog_sync: periodic 259 | commitlog_sync_period_in_ms: 10000 260 | 261 | # The size of the individual commitlog file segments. A commitlog 262 | # segment may be archived, deleted, or recycled once all the data 263 | # in it (potentially from each columnfamily in the system) has been 264 | # flushed to sstables. 265 | # 266 | # The default size is 32, which is almost always fine, but if you are 267 | # archiving commitlog segments (see commitlog_archiving.properties), 268 | # then you probably want a finer granularity of archiving; 8 or 16 MB 269 | # is reasonable. 270 | commitlog_segment_size_in_mb: 32 271 | 272 | # Reuse commit log files when possible. The default is false, and this 273 | # feature will be removed entirely in future versions of Cassandra. 274 | #commitlog_segment_recycling: false 275 | 276 | # any class that implements the SeedProvider interface and has a 277 | # constructor that takes a Map of parameters will do. 278 | seed_provider: 279 | # Addresses of hosts that are deemed contact points. 280 | # Cassandra nodes use this list of hosts to find each other and learn 281 | # the topology of the ring. You must change this if you are running 282 | # multiple nodes! 283 | - class_name: SEED_PROVIDER 284 | parameters: 285 | # seeds is actually a comma-delimited list of addresses. 286 | # Ex: ",," 287 | - seeds: "127.0.0.1" 288 | 289 | # For workloads with more data than can fit in memory, Cassandra's 290 | # bottleneck will be reads that need to fetch data from 291 | # disk. "concurrent_reads" should be set to (16 * number_of_drives) in 292 | # order to allow the operations to enqueue low enough in the stack 293 | # that the OS and drives can reorder them. Same applies to 294 | # "concurrent_counter_writes", since counter writes read the current 295 | # values before incrementing and writing them back. 296 | # 297 | # On the other hand, since writes are almost never IO bound, the ideal 298 | # number of "concurrent_writes" is dependent on the number of cores in 299 | # your system; (8 * number_of_cores) is a good rule of thumb. 300 | concurrent_reads: 32 301 | concurrent_writes: 32 302 | concurrent_counter_writes: 32 303 | 304 | # Total memory to use for sstable-reading buffers. Defaults to 305 | # the smaller of 1/4 of heap or 512MB. 306 | # file_cache_size_in_mb: 512 307 | 308 | # Total permitted memory to use for memtables. Cassandra will stop 309 | # accepting writes when the limit is exceeded until a flush completes, 310 | # and will trigger a flush based on memtable_cleanup_threshold 311 | # If omitted, Cassandra will set both to 1/4 the size of the heap. 312 | # memtable_heap_space_in_mb: 2048 313 | # memtable_offheap_space_in_mb: 2048 314 | 315 | # Ratio of occupied non-flushing memtable size to total permitted size 316 | # that will trigger a flush of the largest memtable. Lager mct will 317 | # mean larger flushes and hence less compaction, but also less concurrent 318 | # flush activity which can make it difficult to keep your disks fed 319 | # under heavy write load. 320 | # 321 | # memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) 322 | # memtable_cleanup_threshold: 0.11 323 | 324 | # Specify the way Cassandra allocates and manages memtable memory. 325 | # Options are: 326 | # heap_buffers: on heap nio buffers 327 | # offheap_buffers: off heap (direct) nio buffers 328 | # offheap_objects: native memory, eliminating nio buffer heap overhead 329 | memtable_allocation_type: heap_buffers 330 | 331 | # Total space to use for commitlogs. Since commitlog segments are 332 | # mmapped, and hence use up address space, the default size is 32 333 | # on 32-bit JVMs, and 8192 on 64-bit JVMs. 334 | # 335 | # If space gets above this value (it will round up to the next nearest 336 | # segment multiple), Cassandra will flush every dirty CF in the oldest 337 | # segment and remove it. So a small total commitlog space will tend 338 | # to cause more flush activity on less-active columnfamilies. 339 | # commitlog_total_space_in_mb: 8192 340 | 341 | # This sets the amount of memtable flush writer threads. These will 342 | # be blocked by disk io, and each one will hold a memtable in memory 343 | # while blocked. 344 | # 345 | # memtable_flush_writers defaults to the smaller of (number of disks, 346 | # number of cores), with a minimum of 2 and a maximum of 8. 347 | # 348 | # If your data directories are backed by SSD, you should increase this 349 | # to the number of cores. 350 | #memtable_flush_writers: 8 351 | 352 | # A fixed memory pool size in MB for for SSTable index summaries. If left 353 | # empty, this will default to 5% of the heap size. If the memory usage of 354 | # all index summaries exceeds this limit, SSTables with low read rates will 355 | # shrink their index summaries in order to meet this limit. However, this 356 | # is a best-effort process. In extreme conditions Cassandra may need to use 357 | # more than this amount of memory. 358 | index_summary_capacity_in_mb: 359 | 360 | # How frequently index summaries should be resampled. This is done 361 | # periodically to redistribute memory from the fixed-size pool to sstables 362 | # proportional their recent read rates. Setting to -1 will disable this 363 | # process, leaving existing index summaries at their current sampling level. 364 | index_summary_resize_interval_in_minutes: 60 365 | 366 | # Whether to, when doing sequential writing, fsync() at intervals in 367 | # order to force the operating system to flush the dirty 368 | # buffers. Enable this to avoid sudden dirty buffer flushing from 369 | # impacting read latencies. Almost always a good idea on SSDs; not 370 | # necessarily on platters. 371 | trickle_fsync: false 372 | trickle_fsync_interval_in_kb: 10240 373 | 374 | # TCP port, for commands and data 375 | # For security reasons, you should not expose this port to the internet. Firewall it if needed. 376 | storage_port: 7000 377 | 378 | # SSL port, for encrypted communication. Unused unless enabled in 379 | # encryption_options 380 | # For security reasons, you should not expose this port to the internet. Firewall it if needed. 381 | ssl_storage_port: 7001 382 | 383 | # Address or interface to bind to and tell other Cassandra nodes to connect to. 384 | # You _must_ change this if you want multiple nodes to be able to communicate! 385 | # 386 | # Set listen_address OR listen_interface, not both. Interfaces must correspond 387 | # to a single address, IP aliasing is not supported. 388 | # 389 | # Leaving it blank leaves it up to InetAddress.getLocalHost(). This 390 | # will always do the Right Thing _if_ the node is properly configured 391 | # (hostname, name resolution, etc), and the Right Thing is to use the 392 | # address associated with the hostname (it might not be). 393 | # 394 | # Setting listen_address to 0.0.0.0 is always wrong. 395 | # 396 | # If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address 397 | # you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4 398 | # address will be used. If true the first ipv6 address will be used. Defaults to false preferring 399 | # ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. 400 | listen_address: localhost 401 | # listen_interface: eth0 402 | # listen_interface_prefer_ipv6: false 403 | 404 | # Address to broadcast to other Cassandra nodes 405 | # Leaving this blank will set it to the same value as listen_address 406 | # broadcast_address: 1.2.3.4 407 | 408 | # Internode authentication backend, implementing IInternodeAuthenticator; 409 | # used to allow/disallow connections from peer nodes. 410 | # internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator 411 | 412 | # Whether to start the native transport server. 413 | # Please note that the address on which the native transport is bound is the 414 | # same as the rpc_address. The port however is different and specified below. 415 | start_native_transport: true 416 | # port for the CQL native transport to listen for clients on 417 | # For security reasons, you should not expose this port to the internet. Firewall it if needed. 418 | native_transport_port: 9042 419 | # The maximum threads for handling requests when the native transport is used. 420 | # This is similar to rpc_max_threads though the default differs slightly (and 421 | # there is no native_transport_min_threads, idle threads will always be stopped 422 | # after 30 seconds). 423 | # native_transport_max_threads: 128 424 | # 425 | # The maximum size of allowed frame. Frame (requests) larger than this will 426 | # be rejected as invalid. The default is 256MB. 427 | # native_transport_max_frame_size_in_mb: 256 428 | 429 | # The maximum number of concurrent client connections. 430 | # The default is -1, which means unlimited. 431 | # native_transport_max_concurrent_connections: -1 432 | 433 | # The maximum number of concurrent client connections per source ip. 434 | # The default is -1, which means unlimited. 435 | # native_transport_max_concurrent_connections_per_ip: -1 436 | 437 | # Whether to start the thrift rpc server. 438 | start_rpc: true 439 | 440 | # The address or interface to bind the Thrift RPC service and native transport 441 | # server to. 442 | # 443 | # Set rpc_address OR rpc_interface, not both. Interfaces must correspond 444 | # to a single address, IP aliasing is not supported. 445 | # 446 | # Leaving rpc_address blank has the same effect as on listen_address 447 | # (i.e. it will be based on the configured hostname of the node). 448 | # 449 | # Note that unlike listen_address, you can specify 0.0.0.0, but you must also 450 | # set broadcast_rpc_address to a value other than 0.0.0.0. 451 | # 452 | # For security reasons, you should not expose this port to the internet. Firewall it if needed. 453 | # 454 | # If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address 455 | # you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 456 | # address will be used. If true the first ipv6 address will be used. Defaults to false preferring 457 | # ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. 458 | rpc_address: localhost 459 | # rpc_interface: eth1 460 | # rpc_interface_prefer_ipv6: false 461 | 462 | # port for Thrift to listen for clients on 463 | rpc_port: 9160 464 | 465 | # RPC address to broadcast to drivers and other Cassandra nodes. This cannot 466 | # be set to 0.0.0.0. If left blank, this will be set to the value of 467 | # rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must 468 | # be set. 469 | # broadcast_rpc_address: 1.2.3.4 470 | 471 | # enable or disable keepalive on rpc/native connections 472 | rpc_keepalive: true 473 | 474 | # Cassandra provides two out-of-the-box options for the RPC Server: 475 | # 476 | # sync -> One thread per thrift connection. For a very large number of clients, memory 477 | # will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size 478 | # per thread, and that will correspond to your use of virtual memory (but physical memory 479 | # may be limited depending on use of stack space). 480 | # 481 | # hsha -> Stands for "half synchronous, half asynchronous." All thrift clients are handled 482 | # asynchronously using a small number of threads that does not vary with the amount 483 | # of thrift clients (and thus scales well to many clients). The rpc requests are still 484 | # synchronous (one thread per active request). If hsha is selected then it is essential 485 | # that rpc_max_threads is changed from the default value of unlimited. 486 | # 487 | # The default is sync because on Windows hsha is about 30% slower. On Linux, 488 | # sync/hsha performance is about the same, with hsha of course using less memory. 489 | # 490 | # Alternatively, can provide your own RPC server by providing the fully-qualified class name 491 | # of an o.a.c.t.TServerFactory that can create an instance of it. 492 | rpc_server_type: sync 493 | 494 | # Uncomment rpc_min|max_thread to set request pool size limits. 495 | # 496 | # Regardless of your choice of RPC server (see above), the number of maximum requests in the 497 | # RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync 498 | # RPC server, it also dictates the number of clients that can be connected at all). 499 | # 500 | # The default is unlimited and thus provides no protection against clients overwhelming the server. You are 501 | # encouraged to set a maximum that makes sense for you in production, but do keep in mind that 502 | # rpc_max_threads represents the maximum number of client requests this server may execute concurrently. 503 | # 504 | # rpc_min_threads: 16 505 | # rpc_max_threads: 2048 506 | 507 | # uncomment to set socket buffer sizes on rpc connections 508 | # rpc_send_buff_size_in_bytes: 509 | # rpc_recv_buff_size_in_bytes: 510 | 511 | # Uncomment to set socket buffer size for internode communication 512 | # Note that when setting this, the buffer size is limited by net.core.wmem_max 513 | # and when not setting it it is defined by net.ipv4.tcp_wmem 514 | # See: 515 | # /proc/sys/net/core/wmem_max 516 | # /proc/sys/net/core/rmem_max 517 | # /proc/sys/net/ipv4/tcp_wmem 518 | # /proc/sys/net/ipv4/tcp_wmem 519 | # and: man tcp 520 | # internode_send_buff_size_in_bytes: 521 | # internode_recv_buff_size_in_bytes: 522 | 523 | # Frame size for thrift (maximum message length). 524 | thrift_framed_transport_size_in_mb: 15 525 | 526 | # Set to true to have Cassandra create a hard link to each sstable 527 | # flushed or streamed locally in a backups/ subdirectory of the 528 | # keyspace data. Removing these links is the operator's 529 | # responsibility. 530 | incremental_backups: false 531 | 532 | # Whether or not to take a snapshot before each compaction. Be 533 | # careful using this option, since Cassandra won't clean up the 534 | # snapshots for you. Mostly useful if you're paranoid when there 535 | # is a data format change. 536 | snapshot_before_compaction: false 537 | 538 | # Whether or not a snapshot is taken of the data before keyspace truncation 539 | # or dropping of column families. The STRONGLY advised default of true 540 | # should be used to provide data safety. If you set this flag to false, you will 541 | # lose data on truncation or drop. 542 | auto_snapshot: true 543 | 544 | # When executing a scan, within or across a partition, we need to keep the 545 | # tombstones seen in memory so we can return them to the coordinator, which 546 | # will use them to make sure other replicas also know about the deleted rows. 547 | # With workloads that generate a lot of tombstones, this can cause performance 548 | # problems and even exaust the server heap. 549 | # (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) 550 | # Adjust the thresholds here if you understand the dangers and want to 551 | # scan more tombstones anyway. These thresholds may also be adjusted at runtime 552 | # using the StorageService mbean. 553 | tombstone_warn_threshold: 1000 554 | tombstone_failure_threshold: 100000 555 | 556 | # Granularity of the collation index of rows within a partition. 557 | # Increase if your rows are large, or if you have a very large 558 | # number of rows per partition. The competing goals are these: 559 | # 1) a smaller granularity means more index entries are generated 560 | # and looking up rows withing the partition by collation column 561 | # is faster 562 | # 2) but, Cassandra will keep the collation index in memory for hot 563 | # rows (as part of the key cache), so a larger granularity means 564 | # you can cache more hot rows 565 | column_index_size_in_kb: 64 566 | 567 | 568 | # Log WARN on any batch size exceeding this value. 5kb per batch by default. 569 | # Caution should be taken on increasing the size of this threshold as it can lead to node instability. 570 | batch_size_warn_threshold_in_kb: 5 571 | 572 | 573 | # Log WARN on any batches not of type LOGGED than span across more partitions than this limit 574 | unlogged_batch_across_partitions_warn_threshold: 10 575 | 576 | # Number of simultaneous compactions to allow, NOT including 577 | # validation "compactions" for anti-entropy repair. Simultaneous 578 | # compactions can help preserve read performance in a mixed read/write 579 | # workload, by mitigating the tendency of small sstables to accumulate 580 | # during a single long running compactions. The default is usually 581 | # fine and if you experience problems with compaction running too 582 | # slowly or too fast, you should look at 583 | # compaction_throughput_mb_per_sec first. 584 | # 585 | # concurrent_compactors defaults to the smaller of (number of disks, 586 | # number of cores), with a minimum of 2 and a maximum of 8. 587 | # 588 | # If your data directories are backed by SSD, you should increase this 589 | # to the number of cores. 590 | #concurrent_compactors: 1 591 | 592 | # Throttles compaction to the given total throughput across the entire 593 | # system. The faster you insert data, the faster you need to compact in 594 | # order to keep the sstable count down, but in general, setting this to 595 | # 16 to 32 times the rate you are inserting data is more than sufficient. 596 | # Setting this to 0 disables throttling. Note that this account for all types 597 | # of compaction, including validation compaction. 598 | compaction_throughput_mb_per_sec: 16 599 | 600 | # Log a warning when compacting partitions larger than this value 601 | compaction_large_partition_warning_threshold_mb: 100 602 | 603 | # When compacting, the replacement sstable(s) can be opened before they 604 | # are completely written, and used in place of the prior sstables for 605 | # any range that has been written. This helps to smoothly transfer reads 606 | # between the sstables, reducing page cache churn and keeping hot rows hot 607 | sstable_preemptive_open_interval_in_mb: 50 608 | 609 | # Throttles all outbound streaming file transfers on this node to the 610 | # given total throughput in Mbps. This is necessary because Cassandra does 611 | # mostly sequential IO when streaming data during bootstrap or repair, which 612 | # can lead to saturating the network connection and degrading rpc performance. 613 | # When unset, the default is 200 Mbps or 25 MB/s. 614 | # stream_throughput_outbound_megabits_per_sec: 200 615 | 616 | # Throttles all streaming file transfer between the datacenters, 617 | # this setting allows users to throttle inter dc stream throughput in addition 618 | # to throttling all network stream traffic as configured with 619 | # stream_throughput_outbound_megabits_per_sec 620 | # When unset, the default is 200 Mbps or 25 MB/s 621 | # inter_dc_stream_throughput_outbound_megabits_per_sec: 200 622 | 623 | # How long the coordinator should wait for read operations to complete 624 | read_request_timeout_in_ms: 5000 625 | # How long the coordinator should wait for seq or index scans to complete 626 | range_request_timeout_in_ms: 10000 627 | # How long the coordinator should wait for writes to complete 628 | write_request_timeout_in_ms: 2000 629 | # How long the coordinator should wait for counter writes to complete 630 | counter_write_request_timeout_in_ms: 5000 631 | # How long a coordinator should continue to retry a CAS operation 632 | # that contends with other proposals for the same row 633 | cas_contention_timeout_in_ms: 1000 634 | # How long the coordinator should wait for truncates to complete 635 | # (This can be much longer, because unless auto_snapshot is disabled 636 | # we need to flush first so we can snapshot before removing the data.) 637 | truncate_request_timeout_in_ms: 60000 638 | # The default timeout for other, miscellaneous operations 639 | request_timeout_in_ms: 10000 640 | 641 | # Enable operation timeout information exchange between nodes to accurately 642 | # measure request timeouts. If disabled, replicas will assume that requests 643 | # were forwarded to them instantly by the coordinator, which means that 644 | # under overload conditions we will waste that much extra time processing 645 | # already-timed-out requests. 646 | # 647 | # Warning: before enabling this property make sure to ntp is installed 648 | # and the times are synchronized between the nodes. 649 | cross_node_timeout: false 650 | 651 | # Set socket timeout for streaming operation. 652 | # The stream session is failed if no data/ack is received by any of the participants 653 | # within that period, which means this should also be sufficient to stream a large 654 | # sstable or rebuild table indexes. 655 | # Default value is 86400000ms, which means stale streams timeout after 24 hours. 656 | # A value of zero means stream sockets should never time out. 657 | # streaming_socket_timeout_in_ms: 86400000 658 | 659 | # phi value that must be reached for a host to be marked down. 660 | # most users should never need to adjust this. 661 | # phi_convict_threshold: 8 662 | 663 | # endpoint_snitch -- Set this to a class that implements 664 | # IEndpointSnitch. The snitch has two functions: 665 | # - it teaches Cassandra enough about your network topology to route 666 | # requests efficiently 667 | # - it allows Cassandra to spread replicas around your cluster to avoid 668 | # correlated failures. It does this by grouping machines into 669 | # "datacenters" and "racks." Cassandra will do its best not to have 670 | # more than one replica on the same "rack" (which may not actually 671 | # be a physical location) 672 | # 673 | # CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH 674 | # ONCE DATA IS INSERTED INTO THE CLUSTER. This would cause data loss. 675 | # This means that if you start with the default SimpleSnitch, which 676 | # locates every node on "rack1" in "datacenter1", your only options 677 | # if you need to add another datacenter are GossipingPropertyFileSnitch 678 | # (and the older PFS). From there, if you want to migrate to an 679 | # incompatible snitch like Ec2Snitch you can do it by adding new nodes 680 | # under Ec2Snitch (which will locate them in a new "datacenter") and 681 | # decommissioning the old ones. 682 | # 683 | # Out of the box, Cassandra provides 684 | # - SimpleSnitch: 685 | # Treats Strategy order as proximity. This can improve cache 686 | # locality when disabling read repair. Only appropriate for 687 | # single-datacenter deployments. 688 | # - GossipingPropertyFileSnitch 689 | # This should be your go-to snitch for production use. The rack 690 | # and datacenter for the local node are defined in 691 | # cassandra-rackdc.properties and propagated to other nodes via 692 | # gossip. If cassandra-topology.properties exists, it is used as a 693 | # fallback, allowing migration from the PropertyFileSnitch. 694 | # - PropertyFileSnitch: 695 | # Proximity is determined by rack and data center, which are 696 | # explicitly configured in cassandra-topology.properties. 697 | # - Ec2Snitch: 698 | # Appropriate for EC2 deployments in a single Region. Loads Region 699 | # and Availability Zone information from the EC2 API. The Region is 700 | # treated as the datacenter, and the Availability Zone as the rack. 701 | # Only private IPs are used, so this will not work across multiple 702 | # Regions. 703 | # - Ec2MultiRegionSnitch: 704 | # Uses public IPs as broadcast_address to allow cross-region 705 | # connectivity. (Thus, you should set seed addresses to the public 706 | # IP as well.) You will need to open the storage_port or 707 | # ssl_storage_port on the public IP firewall. (For intra-Region 708 | # traffic, Cassandra will switch to the private IP after 709 | # establishing a connection.) 710 | # - RackInferringSnitch: 711 | # Proximity is determined by rack and data center, which are 712 | # assumed to correspond to the 3rd and 2nd octet of each node's IP 713 | # address, respectively. Unless this happens to match your 714 | # deployment conventions, this is best used as an example of 715 | # writing a custom Snitch class and is provided in that spirit. 716 | # 717 | # You can use a custom Snitch by setting this to the full class name 718 | # of the snitch, which will be assumed to be on your classpath. 719 | endpoint_snitch: SimpleSnitch 720 | 721 | # controls how often to perform the more expensive part of host score 722 | # calculation 723 | dynamic_snitch_update_interval_in_ms: 100 724 | # controls how often to reset all host scores, allowing a bad host to 725 | # possibly recover 726 | dynamic_snitch_reset_interval_in_ms: 600000 727 | # if set greater than zero and read_repair_chance is < 1.0, this will allow 728 | # 'pinning' of replicas to hosts in order to increase cache capacity. 729 | # The badness threshold will control how much worse the pinned host has to be 730 | # before the dynamic snitch will prefer other replicas over it. This is 731 | # expressed as a double which represents a percentage. Thus, a value of 732 | # 0.2 means Cassandra would continue to prefer the static snitch values 733 | # until the pinned host was 20% worse than the fastest. 734 | dynamic_snitch_badness_threshold: 0.1 735 | 736 | # request_scheduler -- Set this to a class that implements 737 | # RequestScheduler, which will schedule incoming client requests 738 | # according to the specific policy. This is useful for multi-tenancy 739 | # with a single Cassandra cluster. 740 | # NOTE: This is specifically for requests from the client and does 741 | # not affect inter node communication. 742 | # org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place 743 | # org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of 744 | # client requests to a node with a separate queue for each 745 | # request_scheduler_id. The scheduler is further customized by 746 | # request_scheduler_options as described below. 747 | request_scheduler: org.apache.cassandra.scheduler.NoScheduler 748 | 749 | # Scheduler Options vary based on the type of scheduler 750 | # NoScheduler - Has no options 751 | # RoundRobin 752 | # - throttle_limit -- The throttle_limit is the number of in-flight 753 | # requests per client. Requests beyond 754 | # that limit are queued up until 755 | # running requests can complete. 756 | # The value of 80 here is twice the number of 757 | # concurrent_reads + concurrent_writes. 758 | # - default_weight -- default_weight is optional and allows for 759 | # overriding the default which is 1. 760 | # - weights -- Weights are optional and will default to 1 or the 761 | # overridden default_weight. The weight translates into how 762 | # many requests are handled during each turn of the 763 | # RoundRobin, based on the scheduler id. 764 | # 765 | # request_scheduler_options: 766 | # throttle_limit: 80 767 | # default_weight: 5 768 | # weights: 769 | # Keyspace1: 1 770 | # Keyspace2: 5 771 | 772 | # request_scheduler_id -- An identifier based on which to perform 773 | # the request scheduling. Currently the only valid option is keyspace. 774 | # request_scheduler_id: keyspace 775 | 776 | # Enable or disable inter-node encryption 777 | # Default settings are TLS v1, RSA 1024-bit keys (it is imperative that 778 | # users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher 779 | # suite for authentication, key exchange and encryption of the actual data transfers. 780 | # Use the DHE/ECDHE ciphers if running in FIPS 140 compliant mode. 781 | # NOTE: No custom encryption options are enabled at the moment 782 | # The available internode options are : all, none, dc, rack 783 | # 784 | # If set to dc cassandra will encrypt the traffic between the DCs 785 | # If set to rack cassandra will encrypt the traffic between the racks 786 | # 787 | # The passwords used in these options must match the passwords used when generating 788 | # the keystore and truststore. For instructions on generating these files, see: 789 | # http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore 790 | # 791 | server_encryption_options: 792 | internode_encryption: none 793 | keystore: conf/.keystore 794 | keystore_password: cassandra 795 | truststore: conf/.truststore 796 | truststore_password: cassandra 797 | # More advanced defaults below: 798 | # protocol: TLS 799 | # algorithm: SunX509 800 | # store_type: JKS 801 | # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] 802 | # require_client_auth: false 803 | 804 | # enable or disable client/server encryption. 805 | client_encryption_options: 806 | enabled: false 807 | # If enabled and optional is set to true encrypted and unencrypted connections are handled. 808 | optional: false 809 | keystore: conf/.keystore 810 | keystore_password: cassandra 811 | # require_client_auth: false 812 | # Set trustore and truststore_password if require_client_auth is true 813 | # truststore: conf/.truststore 814 | # truststore_password: cassandra 815 | # More advanced defaults below: 816 | # protocol: TLS 817 | # algorithm: SunX509 818 | # store_type: JKS 819 | # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] 820 | 821 | # internode_compression controls whether traffic between nodes is 822 | # compressed. 823 | # can be: all - all traffic is compressed 824 | # dc - traffic between different datacenters is compressed 825 | # none - nothing is compressed. 826 | internode_compression: dc 827 | 828 | # Enable or disable tcp_nodelay for inter-dc communication. 829 | # Disabling it will result in larger (but fewer) network packets being sent, 830 | # reducing overhead from the TCP protocol itself, at the cost of increasing 831 | # latency if you block for cross-datacenter responses. 832 | inter_dc_tcp_nodelay: false 833 | 834 | # GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level 835 | # Adjust the threshold based on your application throughput requirement 836 | # By default, Cassandra logs GC Pauses greater than 200 ms at INFO level 837 | # gc_warn_threshold_in_ms: 1000 838 | 839 | otc_coalescing_strategy: TIMEHORIZON 840 | otc_coalescing_window_us: 200 841 | -------------------------------------------------------------------------------- /container/files/jvm.options: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # jvm.options # 3 | # # 4 | # - all flags defined here will be used by cassandra to startup the JVM # 5 | # - one flag should be specified per line # 6 | # - lines that do not start with '-' will be ignored # 7 | # - only static flags are accepted (no variables or parameters) # 8 | # - dynamic flags will be appended to these on cassandra-env # 9 | ########################################################################### 10 | 11 | ###################### 12 | # STARTUP PARAMETERS # 13 | ###################### 14 | 15 | # Uncomment any of the following properties to enable specific startup parameters 16 | 17 | # In a multi-instance deployment, multiple Cassandra instances will independently assume that all 18 | # CPU processors are available to it. This setting allows you to specify a smaller set of processors 19 | # and perhaps have affinity. 20 | #-Dcassandra.available_processors=number_of_processors 21 | 22 | # The directory location of the cassandra.yaml file. 23 | #-Dcassandra.config=directory 24 | 25 | # Sets the initial partitioner token for a node the first time the node is started. 26 | #-Dcassandra.initial_token=token 27 | 28 | # Set to false to start Cassandra on a node but not have the node join the cluster. 29 | #-Dcassandra.join_ring=true|false 30 | 31 | # Set to false to clear all gossip state for the node on restart. Use when you have changed node 32 | # information in cassandra.yaml (such as listen_address). 33 | #-Dcassandra.load_ring_state=true|false 34 | 35 | # Enable pluggable metrics reporter. See Pluggable metrics reporting in Cassandra 2.0.2. 36 | #-Dcassandra.metricsReporterConfigFile=file 37 | 38 | # Set the port on which the CQL native transport listens for clients. (Default: 9042) 39 | #-Dcassandra.native_transport_port=port 40 | 41 | # Overrides the partitioner. (Default: org.apache.cassandra.dht.Murmur3Partitioner) 42 | #-Dcassandra.partitioner=partitioner 43 | 44 | # To replace a node that has died, restart a new node in its place specifying the address of the 45 | # dead node. The new node must not have any data in its data directory, that is, it must be in the 46 | # same state as before bootstrapping. 47 | #-Dcassandra.replace_address=listen_address or broadcast_address of dead node 48 | 49 | # Allow restoring specific tables from an archived commit log. 50 | #-Dcassandra.replayList=table 51 | 52 | # Allows overriding of the default RING_DELAY (1000ms), which is the amount of time a node waits 53 | # before joining the ring. 54 | #-Dcassandra.ring_delay_ms=ms 55 | 56 | # Set the port for the Thrift RPC service, which is used for client connections. (Default: 9160) 57 | #-Dcassandra.rpc_port=port 58 | 59 | # Set the SSL port for encrypted communication. (Default: 7001) 60 | #-Dcassandra.ssl_storage_port=port 61 | 62 | # Enable or disable the native transport server. See start_native_transport in cassandra.yaml. 63 | # cassandra.start_native_transport=true|false 64 | 65 | # Enable or disable the Thrift RPC server. (Default: true) 66 | #-Dcassandra.start_rpc=true/false 67 | 68 | # Set the port for inter-node communication. (Default: 7000) 69 | #-Dcassandra.storage_port=port 70 | 71 | # Set the default location for the trigger JARs. (Default: conf/triggers) 72 | #-Dcassandra.triggers_dir=directory 73 | 74 | # For testing new compaction and compression strategies. It allows you to experiment with different 75 | # strategies and benchmark write performance differences without affecting the production workload. 76 | #-Dcassandra.write_survey=true 77 | 78 | # To disable configuration via JMX of auth caches (such as those for credentials, permissions and 79 | # roles). This will mean those config options can only be set (persistently) in cassandra.yaml 80 | # and will require a restart for new values to take effect. 81 | #-Dcassandra.disable_auth_caches_remote_configuration=true 82 | 83 | ######################## 84 | # GENERAL JVM SETTINGS # 85 | ######################## 86 | 87 | # enable assertions. disabling this in production will give a modest 88 | # performance benefit (around 5%). 89 | #-ea 90 | 91 | # enable thread priorities, primarily so we can give periodic tasks 92 | # a lower priority to avoid interfering with client workload 93 | -XX:+UseThreadPriorities 94 | 95 | # allows lowering thread priority without being root on linux - probably 96 | # not necessary on Windows but doesn't harm anything. 97 | # see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workar 98 | -XX:ThreadPriorityPolicy=42 99 | 100 | # Enable heap-dump if there's an OOM 101 | -XX:+HeapDumpOnOutOfMemoryError 102 | 103 | # Per-thread stack size. 104 | -Xss256k 105 | 106 | # Larger interned string table, for gossip's benefit (CASSANDRA-6410) 107 | -XX:StringTableSize=1000003 108 | 109 | # Make sure all memory is faulted and zeroed on startup. 110 | # This helps prevent soft faults in containers and makes 111 | # transparent hugepage allocation more effective. 112 | -XX:+AlwaysPreTouch 113 | 114 | # Disable biased locking as it does not benefit Cassandra. 115 | -XX:-UseBiasedLocking 116 | 117 | # Enable thread-local allocation blocks and allow the JVM to automatically 118 | # resize them at runtime. 119 | -XX:+UseTLAB 120 | -XX:+ResizeTLAB 121 | 122 | # http://www.evanjones.ca/jvm-mmap-pause.html 123 | -XX:+PerfDisableSharedMem 124 | 125 | # Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See 126 | # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version: 127 | # comment out this entry to enable IPv6 support). 128 | -Djava.net.preferIPv4Stack=true 129 | 130 | ### Debug options 131 | 132 | # uncomment to enable flight recorder 133 | #-XX:+UnlockCommercialFeatures 134 | #-XX:+FlightRecorder 135 | 136 | # uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414 137 | #-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414 138 | 139 | # uncomment to have Cassandra JVM log internal method compilation (developers only) 140 | #-XX:+UnlockDiagnosticVMOptions 141 | #-XX:+LogCompilation 142 | 143 | ################# 144 | # HEAP SETTINGS # 145 | ################# 146 | 147 | # Heap size is automatically calculated by cassandra-env based on this 148 | # formula: max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB)) 149 | # That is: 150 | # - calculate 1/2 ram and cap to 1024MB 151 | # - calculate 1/4 ram and cap to 8192MB 152 | # - pick the max 153 | # 154 | # For production use you may wish to adjust this for your environment. 155 | # If that's the case, uncomment the -Xmx and Xms options below to override the 156 | # automatic calculation of JVM heap memory. 157 | # 158 | # It is recommended to set min (-Xms) and max (-Xmx) heap sizes to 159 | # the same value to avoid stop-the-world GC pauses during resize, and 160 | # so that we can lock the heap in memory on startup to prevent any 161 | # of it from being swapped out. 162 | #-Xms4G 163 | #-Xmx4G 164 | 165 | # Young generation size is automatically calculated by cassandra-env 166 | # based on this formula: min(100 * num_cores, 1/4 * heap size) 167 | # 168 | # The main trade-off for the young generation is that the larger it 169 | # is, the longer GC pause times will be. The shorter it is, the more 170 | # expensive GC will be (usually). 171 | # 172 | # It is not recommended to set the young generation size if using the 173 | # G1 GC, since that will override the target pause-time goal. 174 | # More info: http://www.oracle.com/technetwork/articles/java/g1gc-1984535.html 175 | # 176 | # The example below assumes a modern 8-core+ machine for decent 177 | # times. If in doubt, and if you do not particularly want to tweak, go 178 | # 100 MB per physical CPU core. 179 | #-Xmn800M 180 | 181 | ################# 182 | # GC SETTINGS # 183 | ################# 184 | 185 | ### CMS Settings 186 | 187 | #-XX:+UseParNewGC 188 | #-XX:+UseConcMarkSweepGC 189 | #-XX:+CMSParallelRemarkEnabled 190 | #-XX:SurvivorRatio=8 191 | #-XX:MaxTenuringThreshold=1 192 | #-XX:CMSInitiatingOccupancyFraction=75 193 | #-XX:+UseCMSInitiatingOccupancyOnly 194 | #-XX:CMSWaitDuration=10000 195 | #-XX:+CMSParallelInitialMarkEnabled 196 | #-XX:+CMSEdenChunksRecordAlways 197 | # some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541 198 | #-XX:+CMSClassUnloadingEnabled 199 | 200 | ### G1 Settings (experimental, comment previous section and uncomment section below to enable) 201 | 202 | ## Use the Hotspot garbage-first collector. 203 | -XX:+UseG1GC 204 | # 205 | ## Have the JVM do less remembered set work during STW, instead 206 | ## preferring concurrent GC. Reduces p99.9 latency. 207 | -XX:G1RSetUpdatingPauseTimePercent=5 208 | # 209 | ## Main G1GC tunable: lowering the pause target will lower throughput and vise versa. 210 | ## 200ms is the JVM default and lowest viable setting 211 | ## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml. 212 | #-XX:MaxGCPauseMillis=500 213 | 214 | ## Optional G1 Settings 215 | 216 | # Save CPU time on large (>= 16GB) heaps by delaying region scanning 217 | # until the heap is 70% full. The default in Hotspot 8u40 is 40%. 218 | #-XX:InitiatingHeapOccupancyPercent=70 219 | 220 | # For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores. 221 | # Otherwise equal to the number of cores when 8 or less. 222 | # Machines with > 10 cores should try setting these to <= full cores. 223 | #-XX:ParallelGCThreads=16 224 | 225 | # By default, ConcGCThreads is 1/4 of ParallelGCThreads. 226 | # Setting both to the same value can reduce STW durations. 227 | #-XX:ConcGCThreads=16 228 | 229 | -Dcassandra.max_queued_native_transport_requests=6144 230 | 231 | -javaagent:/usr/local/share/jolokia-agent.jar=host=0.0.0.0 232 | -javaagent:/usr/local/share/prometheus-agent.jar=0.0.0.0:1234:/etc/cassandra/prometheus.yaml 233 | 234 | # Debug 235 | -XX:+PrintCommandLineFlags 236 | -------------------------------------------------------------------------------- /container/files/logback-files.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | INFO 33 | 34 | ${cassandra.logdir}/system.log 35 | 36 | ${cassandra.logdir}/system.log.%i.zip 37 | 1 38 | 20 39 | 40 | 41 | 20MB 42 | 43 | 44 | %-5level [%thread] %date{ISO8601} %F:%L - %msg%n 45 | 46 | 47 | 48 | 49 | 50 | 51 | ${cassandra.logdir}/debug.log 52 | 53 | ${cassandra.logdir}/debug.log.%i.zip 54 | 1 55 | 20 56 | 57 | 58 | 20MB 59 | 60 | 61 | %-5level [%thread] %date{ISO8601} %F:%L - %msg%n 62 | 63 | 64 | 65 | 66 | 67 | 68 | 1024 69 | 0 70 | true 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | INFO 79 | 80 | 81 | %-5level [%thread] %date{ISO8601} %F:%L - %msg%n 82 | 83 | 84 | 85 | 88 | 89 | 90 | 91 | 92 | 93 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /container/files/logback-json-files.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | INFO 33 | 34 | ${cassandra.logdir}/system.log 35 | 36 | ${cassandra.logdir}/system.log.%i.zip 37 | 1 38 | 20 39 | 40 | 41 | 20MB 42 | 43 | 44 | {"type":"cassandra"} 45 | 46 | 47 | 48 | 49 | 50 | 51 | ${cassandra.logdir}/debug.log 52 | 53 | ${cassandra.logdir}/debug.log.%i.zip 54 | 1 55 | 20 56 | 57 | 58 | 20MB 59 | 60 | 61 | {"type":"cassandra"} 62 | 63 | 64 | 65 | 66 | 67 | 68 | 1024 69 | 0 70 | true 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | INFO 79 | 80 | 81 | {"type":"cassandra"} 82 | 83 | 84 | 85 | 88 | 89 | 90 | 91 | 92 | 93 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /container/files/logback-json-stdout.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | {"type":"cassandra"} 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /container/files/logback-stdout.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | %-5level %date{HH:mm:ss,SSS} %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /container/files/prometheus.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | lowercaseOutputLabelNames: true 16 | lowercaseOutputName: true 17 | rules: 18 | - pattern: org.apache.cassandra.metrics<>(Count|Value) 19 | name: cassandra_$1_$3 20 | labels: 21 | address: "$2" 22 | 23 | - pattern: org.apache.cassandra.metrics<>(Count|Value) 24 | name: cassandra_$1_$5 25 | labels: 26 | "$1": "$4" 27 | "$2": "$3" 28 | 29 | # org.apache.cassandra.metrics<>98thPercentile 30 | - pattern: org\.apache\.cassandra\.metrics<>([a-zA-Z0-9]+)? 31 | name: cassandra_$1_$5 32 | labels: 33 | "$1": "$4" 34 | "$2": "$3" 35 | metric: "$6" 36 | 37 | # org.apache.cassandra.db<>UnleveledSSTables 38 | # org.apache.cassandra.db<>DroppableTombstoneRatio 39 | - pattern: org\.apache\.cassandra\.db<>([a-zA-Z0-9]+)? 40 | name: cassandra_$1 41 | labels: 42 | keyspace: "$2" 43 | table: "$3" 44 | metric: "$4" 45 | 46 | # org.apache.cassandra.db<>CoreCompactorThreads 47 | - pattern: org\.apache\.cassandra\.db<>([a-zA-Z0-9]+)? 48 | name: cassandra_$1 49 | labels: 50 | metric: "$2" 51 | 52 | - pattern: com.datastax.bdp<>(Count|Value) 53 | name: dse_$1_$3 54 | labels: 55 | address: "$2" 56 | 57 | - pattern: com.datastax.bdp<>([a-zA-Z0-9]+)? 58 | name: dse_$1_$5 59 | labels: 60 | "$1": "$4" 61 | "$2": "$3" 62 | metric: "$6" 63 | 64 | # solr/sit1_pb.transactions<>95thPcRequestTime 65 | - pattern: solr\/(\S*)?<>([a-zA-Z0-9]+)? 66 | name: dse_search_$3 67 | labels: 68 | columnfamily: "$1" 69 | metric: "$4" 70 | -------------------------------------------------------------------------------- /container/files/ready-probe.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | if [[ $(/usr/local/apache-cassandra/bin/nodetool status | grep "$POD_IP") == *"UN"* ]]; then 18 | if [[ $DEBUG ]]; then echo "Up"; fi 19 | exit 0 20 | else 21 | if [[ $DEBUG ]]; then echo "Not Up"; fi 22 | exit 1 23 | fi 24 | -------------------------------------------------------------------------------- /container/files/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | # Base variables for Cassandra install 20 | CASSANDRA_HOME=/usr/local/apache-cassandra 21 | CASSANDRA_BIN=$CASSANDRA_HOME/bin/cassandra 22 | CASSANDRA_CONF_DIR=/etc/cassandra 23 | CASSANDRA_CFG=$CASSANDRA_CONF_DIR/cassandra.yaml 24 | CASSANDRA_CONF_DIR=/etc/cassandra 25 | 26 | # Get the pod ip address 27 | # This is used in a number of configurations and passed in the k8s manifest 28 | if [ -z "$POD_IP" ]; then 29 | POD_IP=$(hostname -I| awk '{print $1}') 30 | fi 31 | 32 | # Add the hostname to hosts file 33 | if hostname -i; 34 | then 35 | # fix for host networking 36 | echo "$POD_IP $HOSTNAME" >> /etc/hosts 37 | fi 38 | 39 | # we are doing StatefulSet or just setting our seeds 40 | if [ -z "$CASSANDRA_SEEDS" ]; then 41 | CASSANDRA_SEEDS=$POD_IP 42 | fi 43 | 44 | # The following vars relate to their counterparts in $CASSANDRA_CFG 45 | # for instance rpc_address 46 | CASSANDRA_AUTO_BOOTSTRAP="${CASSANDRA_AUTO_BOOTSTRAP:-true}" 47 | CASSANDRA_BROADCAST_ADDRESS=${POD_IP:-$HOSTNAME} 48 | CASSANDRA_BROADCAST_RPC_ADDRESS=${POD_IP:-$HOSTNAME} 49 | CASSANDRA_CLUSTER_NAME="${CASSANDRA_CLUSTER_NAME:='Test Cluster'}" 50 | CASSANDRA_DC="${CASSANDRA_DC}" 51 | CASSANDRA_DISK_OPTIMIZATION_STRATEGY="${CASSANDRA_DISK_OPTIMIZATION_STRATEGY:-ssd}" 52 | CASSANDRA_ENDPOINT_SNITCH="${CASSANDRA_ENDPOINT_SNITCH:-SimpleSnitch}" 53 | CASSANDRA_INTERNODE_COMPRESSION="${CASSANDRA_INTERNODE_COMPRESSION:-dc}" 54 | CASSANDRA_LISTEN_ADDRESS=${POD_IP:-$HOSTNAME} 55 | CASSANDRA_LOG_GC="${CASSANDRA_LOG_GC:-false}" 56 | CASSANDRA_LOG_GC_VERBOSE="${CASSANDRA_GC_VERBOSE:-false}" 57 | CASSANDRA_LOG_JSON="${CASSANDRA_LOG_JSON:-false}" 58 | CASSANDRA_LOG_PATH="${CASSANDRA_LOG_PATH:-/var/log/cassandra}" 59 | CASSANDRA_LOG_TO_FILES="${CASSANDRA_LOG_TO_FILES:-false}" 60 | CASSANDRA_MIGRATION_WAIT="${CASSANDRA_MIGRATION_WAIT:-1}" 61 | CASSANDRA_NUM_TOKENS="${CASSANDRA_NUM_TOKENS:-32}" 62 | CASSANDRA_RACK="${CASSANDRA_RACK}" 63 | CASSANDRA_RING_DELAY="${CASSANDRA_RING_DELAY:-30000}" 64 | CASSANDRA_RPC_ADDRESS="${CASSANDRA_RPC_ADDRESS:-0.0.0.0}" 65 | CASSANDRA_SEEDS="${CASSANDRA_SEEDS:false}" 66 | CASSANDRA_SEED_PROVIDER="${CASSANDRA_SEED_PROVIDER:-org.apache.cassandra.locator.SimpleSeedProvider}" 67 | CASSANDRA_TRICKLE_FSYNC="${CASSANDRA_TRICKLE_FSYNC:-false}" 68 | 69 | # Turn off JMX auth 70 | CASSANDRA_OPEN_JMX="${CASSANDRA_OPEN_JMX:-true}" 71 | 72 | echo Starting Cassandra on "${CASSANDRA_LISTEN_ADDRESS}" 73 | echo CASSANDRA_CONF_DIR "${CASSANDRA_CONF_DIR}" 74 | echo CASSANDRA_AUTO_BOOTSTRAP "${CASSANDRA_AUTO_BOOTSTRAP}" 75 | echo CASSANDRA_BROADCAST_ADDRESS "${CASSANDRA_BROADCAST_ADDRESS}" 76 | echo CASSANDRA_BROADCAST_RPC_ADDRESS "${CASSANDRA_BROADCAST_RPC_ADDRESS}" 77 | echo CASSANDRA_CFG "${CASSANDRA_CFG}" 78 | echo CASSANDRA_CLUSTER_NAME "${CASSANDRA_CLUSTER_NAME}" 79 | echo CASSANDRA_COMPACTION_THROUGHPUT_MB_PER_SEC "${CASSANDRA_COMPACTION_THROUGHPUT_MB_PER_SEC}" 80 | echo CASSANDRA_CONCURRENT_COMPACTORS "${CASSANDRA_CONCURRENT_COMPACTORS}" 81 | echo CASSANDRA_CONCURRENT_READS "${CASSANDRA_CONCURRENT_READS}" 82 | echo CASSANDRA_CONCURRENT_WRITES "${CASSANDRA_CONCURRENT_WRITES}" 83 | echo CASSANDRA_COUNTER_CACHE_SIZE_IN_MB "${CASSANDRA_COUNTER_CACHE_SIZE_IN_MB}" 84 | echo CASSANDRA_DC "${CASSANDRA_DC}" 85 | echo CASSANDRA_DISK_OPTIMIZATION_STRATEGY "${CASSANDRA_DISK_OPTIMIZATION_STRATEGY}" 86 | echo CASSANDRA_ENDPOINT_SNITCH "${CASSANDRA_ENDPOINT_SNITCH}" 87 | echo CASSANDRA_GC_WARN_THRESHOLD_IN_MS "${CASSANDRA_GC_WARN_THRESHOLD_IN_MS}" 88 | echo CASSANDRA_INTERNODE_COMPRESSION "${CASSANDRA_INTERNODE_COMPRESSION}" 89 | echo CASSANDRA_KEY_CACHE_SIZE_IN_MB "${CASSANDRA_KEY_CACHE_SIZE_IN_MB}" 90 | echo CASSANDRA_LISTEN_ADDRESS "${CASSANDRA_LISTEN_ADDRESS}" 91 | echo CASSANDRA_LISTEN_INTERFACE "${CASSANDRA_LISTEN_INTERFACE}" 92 | echo CASSANDRA_LOG_JSON "${CASSANDRA_LOG_JSON}" 93 | echo CASSANDRA_LOG_GC "${CASSANDRA_LOG_GC}" 94 | echo CASSANDRA_LOG_GC_VERBOSE "${CASSANDRA_LOG_GC_VERBOSE}" 95 | echo CASSANDRA_LOG_PATH "${CASSANDRA_LOG_PATH}" 96 | echo CASSANDRA_LOG_TO_FILES "${CASSANDRA_LOG_TO_FILES}" 97 | echo CASSANDRA_MEMTABLE_ALLOCATION_TYPE "${CASSANDRA_MEMTABLE_ALLOCATION_TYPE}" 98 | echo CASSANDRA_MEMTABLE_CLEANUP_THRESHOLD "${CASSANDRA_MEMTABLE_CLEANUP_THRESHOLD}" 99 | echo CASSANDRA_MEMTABLE_FLUSH_WRITERS "${CASSANDRA_MEMTABLE_FLUSH_WRITERS}" 100 | echo CASSANDRA_MIGRATION_WAIT "${CASSANDRA_MIGRATION_WAIT}" 101 | echo CASSANDRA_NUM_TOKENS "${CASSANDRA_NUM_TOKENS}" 102 | echo CASSANDRA_OPEN_JMX "${CASSANDRA_OPEN_JMX}" 103 | echo CASSANDRA_RACK "${CASSANDRA_RACK}" 104 | echo CASSANDRA_RING_DELAY "${CASSANDRA_RING_DELAY}" 105 | echo CASSANDRA_RPC_ADDRESS "${CASSANDRA_RPC_ADDRESS}" 106 | echo CASSANDRA_RPC_INTERFACE "${CASSANDRA_RPC_INTERFACE}" 107 | echo CASSANDRA_SEEDS "${CASSANDRA_SEEDS}" 108 | echo CASSANDRA_SEED_PROVIDER "${CASSANDRA_SEED_PROVIDER}" 109 | echo CASSANDRA_TRICKLE_FSYNC "${CASSANDRA_TRICKLE_FSYNC}" 110 | 111 | # set the storage directory 112 | # shellcheck disable=SC2016 113 | sed -ri 's/^cassandra_storagedir.*/cassandra_storagedir=$CASSANDRA_DATA/' "$CASSANDRA_HOME/bin/cassandra.in.sh" 114 | 115 | # if DC and RACK are set, use GossipingPropertyFileSnitch 116 | if [[ $CASSANDRA_DC && $CASSANDRA_RACK ]]; then 117 | echo "dc=$CASSANDRA_DC" > $CASSANDRA_CONF_DIR/cassandra-rackdc.properties 118 | echo "rack=$CASSANDRA_RACK" >> $CASSANDRA_CONF_DIR/cassandra-rackdc.properties 119 | CASSANDRA_ENDPOINT_SNITCH="GossipingPropertyFileSnitch" 120 | fi 121 | 122 | # Set the heap options in jvm.options 123 | if [ -n "$CASSANDRA_MAX_HEAP" ]; then 124 | sed -ri "s/^(#)?-Xmx[0-9]+.*/-Xmx$CASSANDRA_MAX_HEAP/" "$CASSANDRA_CONF_DIR/jvm.options" 125 | sed -ri "s/^(#)?-Xms[0-9]+.*/-Xms$CASSANDRA_MAX_HEAP/" "$CASSANDRA_CONF_DIR/jvm.options" 126 | fi 127 | 128 | # If replacing a node 129 | # This is not fully functional and should be improved 130 | if [ -n "$CASSANDRA_REPLACE_NODE" ]; then 131 | echo "-Dcassandra.replace_address=$CASSANDRA_REPLACE_NODE/" >> "$CASSANDRA_CONF_DIR/jvm.options" 132 | fi 133 | 134 | # Setup C* racks 135 | # Add rack 136 | # TODO need to test this 137 | for rackdc in dc rack; do 138 | var="CASSANDRA_${rackdc^^}" 139 | val="${!var}" 140 | if [ "$val" ]; then 141 | sed -ri 's/^('"$rackdc"'=).*/\1 '"$val"'/' "$CASSANDRA_CONF_DIR/cassandra-rackdc.properties" 142 | fi 143 | done 144 | 145 | # Replace various values in the $CASSANDRA_CFG config file 146 | # These values map directly to the cassandra.yaml file that is vendorred 147 | # In this directory 148 | for yaml in \ 149 | broadcast_address \ 150 | broadcast_rpc_address \ 151 | cluster_name \ 152 | disk_optimization_strategy \ 153 | endpoint_snitch \ 154 | listen_address \ 155 | num_tokens \ 156 | rpc_address \ 157 | start_rpc \ 158 | key_cache_size_in_mb \ 159 | concurrent_reads \ 160 | concurrent_writes \ 161 | memtable_cleanup_threshold \ 162 | memtable_allocation_type \ 163 | memtable_flush_writers \ 164 | concurrent_compactors \ 165 | compaction_throughput_mb_per_sec \ 166 | counter_cache_size_in_mb \ 167 | internode_compression \ 168 | endpoint_snitch \ 169 | gc_warn_threshold_in_ms \ 170 | listen_interface \ 171 | rpc_interface \ 172 | trickle_fsync \ 173 | ; do 174 | var="CASSANDRA_${yaml^^}" 175 | val="${!var}" 176 | if [ "$val" ]; then 177 | sed -ri 's/^(#\s*)?('"$yaml"':).*/\2 '"$val"'/' "$CASSANDRA_CFG" 178 | fi 179 | done 180 | 181 | # Loop through the containers environment variables search for a VAR prefixed 182 | # with CASSANDRA_YAML_ 183 | # When a correctly name var is found such as CASSANDRA_YAML_FOO, then replace the 184 | # 'foo' yaml key's value with the value set in the env var. 185 | # For example 186 | # CASSANDRA_YAML_phi_convict_threshold=10 is set the StatefulSet manifest 187 | # "# phi_convict_threshold: 8" line in the C* config file will be modified to 188 | # "phi_convict_threshold: 10". 189 | # TODO: do a to lower on the $yaml value to ensure that CASSANDRA_YAML_FOO and 190 | # TODO: CASSANDRA_YAML_foo both work. 191 | while IFS='=' read -r name ; do 192 | if [[ $name == 'CASSANDRA_YAML_'* ]]; then 193 | val="${!name}" 194 | yaml=$(echo "${name,,}" | cut -c 16-) 195 | echo "FOUND $name $yaml $val" 196 | sed -ri 's/^(#\s*)?('"$yaml"':).*/\2 '"$val"'/' "$CASSANDRA_CFG" 197 | fi 198 | done < <(env) 199 | 200 | # setting auto bootstrap for starting nodes 201 | echo "auto_bootstrap: ${CASSANDRA_AUTO_BOOTSTRAP}" >> $CASSANDRA_CFG 202 | 203 | # set the seed to itself. This is only for the first pod, otherwise 204 | # it will be able to get seeds from the seed provider 205 | if [[ $CASSANDRA_SEEDS == 'false' ]]; then 206 | sed -ri 's/- seeds:.*/- seeds: "'"$POD_IP"'"/' $CASSANDRA_CFG 207 | else # if we have seeds set them. Probably StatefulSet 208 | sed -ri 's/- seeds:.*/- seeds: "'"$CASSANDRA_SEEDS"'"/' $CASSANDRA_CFG 209 | fi 210 | 211 | # Set the seed provider in the CASSANDRA_CFG 212 | sed -ri 's/- class_name: SEED_PROVIDER/- class_name: '"$CASSANDRA_SEED_PROVIDER"'/' $CASSANDRA_CFG 213 | 214 | # setup JVM options in cassandra-env 215 | sed -ri 's/JVM_OPTS.*Xloggc.*//' $CASSANDRA_CONF_DIR/cassandra-env.sh 216 | 217 | # Turn on JVM Garbage Collection logging 218 | if [[ $CASSANDRA_LOG_GC == 'true' ]]; then 219 | { 220 | echo "-XX:+PrintGCDetails"; 221 | echo "-XX:+PrintGCDateStamps"; 222 | echo "-XX:+PrintHeapAtGC"; 223 | echo "-XX:+PrintTenuringDistribution"; 224 | echo "-XX:+PrintGCApplicationStoppedTime"; 225 | echo "-XX:+PrintPromotionFailure"; } >> $CASSANDRA_CONF_DIR/jvm.options 226 | 227 | if [[ $CASSANDRA_LOG_GC_VERBOSE == 'true' ]]; then 228 | echo "-XX:PrintFLSStatistics=1" >> $CASSANDRA_CONF_DIR/jvm.options 229 | fi 230 | 231 | if [[ $CASSANDRA_LOG_TO_FILES == 'true' ]]; then 232 | { 233 | echo "-Xloggc:${CASSANDRA_LOG_PATH}/gc.log"; 234 | echo "-XX:+UseGCLogFileRotation"; 235 | echo "-XX:NumberOfGCLogFiles=10"; 236 | echo "-XX:GCLogFileSize=10M"; } >> $CASSANDRA_CONF_DIR/jvm.options 237 | fi 238 | fi 239 | 240 | # configure logging 241 | sed -ri 's/.*cassandra_parms=.*-Dlogback.configurationFile.*//' $CASSANDRA_BIN 242 | sed -ri 's/.*cassandra_parms=.*-Dcassandra.logdir.*//' $CASSANDRA_BIN 243 | echo "-Dcassandra.logdir=${CASSANDRA_LOG_PATH}" >> $CASSANDRA_CONF_DIR/jvm.options 244 | if [[ $CASSANDRA_LOG_TO_FILES == 'true' ]]; then 245 | if [[ $CASSANDRA_LOG_JSON == 'true' ]]; then 246 | echo "-Dlogback.configurationFile=${CASSANDRA_CONF_DIR}/logback-json-files.xml" >> $CASSANDRA_CONF_DIR/jvm.options 247 | else 248 | echo "-Dlogback.configurationFile=${CASSANDRA_CONF_DIR}/logback-files.xml" >> $CASSANDRA_CONF_DIR/jvm.options 249 | fi 250 | else 251 | # TODO can we clean this up?? 252 | if [[ $CASSANDRA_LOG_JSON == 'true' ]]; then 253 | echo "-Dlogback.configurationFile=${CASSANDRA_CONF_DIR}/logback-json-stdout.xml" >> $CASSANDRA_CONF_DIR/jvm.options 254 | else 255 | echo "-Dlogback.configurationFile=${CASSANDRA_CONF_DIR}/logback-stdout.xml" >> $CASSANDRA_CONF_DIR/jvm.options 256 | fi 257 | fi 258 | 259 | # getting WARNING messages with Migration Service 260 | # Setting -D options with jvm for migration and ring delay 261 | echo "-Dcassandra.migration_task_wait_in_seconds=${CASSANDRA_MIGRATION_WAIT}" >> $CASSANDRA_CONF_DIR/jvm.options 262 | # Setting ring delay can speed up the deployment of C* pods 263 | echo "-Dcassandra.ring_delay_ms=${CASSANDRA_RING_DELAY}" >> $CASSANDRA_CONF_DIR/jvm.options 264 | 265 | # Enable jmx 266 | if [[ $CASSANDRA_OPEN_JMX == 'true' ]]; then 267 | export LOCAL_JMX=no 268 | sed -ri 's/ -Dcom\.sun\.management\.jmxremote\.authenticate=true/ -Dcom\.sun\.management\.jmxremote\.authenticate=false/' $CASSANDRA_CONF_DIR/cassandra-env.sh 269 | sed -ri 's/ -Dcom\.sun\.management\.jmxremote\.password\.file=\/etc\/cassandra\/jmxremote\.password//' $CASSANDRA_CONF_DIR/cassandra-env.sh 270 | 271 | { 272 | echo "JVM_OPTS=\"\$JVM_OPTS -Dcom.sun.management.jmxremote\""; 273 | echo "JVM_OPTS=\"\$JVM_OPTS -Dcom.sun.management.jmxremote.ssl=false\""; 274 | echo "JVM_OPTS=\"\$JVM_OPTS -Dcom.sun.management.jmxremote.local.only=false\""; 275 | echo "JVM_OPTS=\"\$JVM_OPTS -Dcom.sun.management.jmxremote.port=7199\""; 276 | echo "JVM_OPTS=\"\$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=7199\""; 277 | echo "JVM_OPTS=\"\$JVM_OPTS -Djava.rmi.server.hostname=$POD_IP\""; } >> $CASSANDRA_CONF_DIR/cassandra-env.sh 278 | fi 279 | 280 | # setup perms on the PVC and log drives 281 | # TODO can we do this in one line? 282 | chmod 700 "${CASSANDRA_DATA}" 283 | chmod 700 "${CASSANDRA_LOG_PATH}" 284 | 285 | # set owner on various paths 286 | chown -c -R cassandra: "${CASSANDRA_DATA}" "${CASSANDRA_CONF_DIR}" "${CASSANDRA_LOG_PATH}" 287 | 288 | # TODO setup a DEBUG env var on these echos 289 | # Can be a lot of logging 290 | echo "/etc/resolv.conf" 291 | cat /etc/resolv.conf 292 | 293 | echo "$CASSANDRA_CFG" 294 | cat $CASSANDRA_CFG 295 | 296 | # start cassandra 297 | su cassandra -c "$CASSANDRA_BIN -f" 298 | -------------------------------------------------------------------------------- /create.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # "---------------------------------------------------------" 18 | # "- -" 19 | # "- Create starts a GKE Cluster and installs -" 20 | # "- a Cassandra StatefulSet -" 21 | # "- -" 22 | # "---------------------------------------------------------" 23 | 24 | set -o errexit 25 | set -o nounset 26 | set -o pipefail 27 | 28 | ROOT=$(dirname "${BASH_SOURCE[0]}") 29 | CLUSTER_NAME="" 30 | ZONE="" 31 | GKE_VERSION=$(gcloud container get-server-config \ 32 | --format="value(validMasterVersions[0])") 33 | 34 | # shellcheck disable=SC1090 35 | source "$ROOT"/common.sh 36 | 37 | 38 | if [[ "$(gcloud services list --format='value(serviceConfig.name)' \ 39 | --filter='serviceConfig.name:container.googleapis.com' 2>&1)" != \ 40 | 'container.googleapis.com' ]]; then 41 | echo "Enabling the Kubernetes Engine API" 42 | gcloud services enable container.googleapis.com 43 | else 44 | echo "The Kubernetes Engine API is already enabled" 45 | fi 46 | 47 | # Create a GKE cluster 48 | # Only setting num of node to "1", because it is a regional cluster the create 49 | # call will create a nodepool that has "1" node in every zone. 50 | echo "Creating cluster" 51 | gcloud container clusters create "$CLUSTER_NAME" \ 52 | --zone "$ZONE" \ 53 | --node-locations "$ZONESINREGION" \ 54 | --cluster-version "$GKE_VERSION" \ 55 | --machine-type "n1-standard-4" \ 56 | --num-nodes=1 \ 57 | --node-taints app=cassandra:NoSchedule \ 58 | --enable-network-policy \ 59 | --enable-ip-alias 60 | 61 | # Get the kubectl credentials for the GKE cluster. 62 | gcloud container clusters get-credentials "$CLUSTER_NAME" --zone "$ZONE" 63 | 64 | # Create cassandra cluster using the manifests in the 'manifests' directory. 65 | # need to set namespace explicitly 66 | kubectl --namespace=default create -f "$ROOT"/manifests 67 | 68 | # Create new nodepool that will not schedule Cassandra Pods 69 | # We create the nodepool after the kubectl command because this may cause 70 | # the control plane to be upgraded. 71 | gcloud container node-pools create nodepool-cassdemo-2 \ 72 | --zone "$ZONE" \ 73 | --num-nodes=1 \ 74 | --cluster "$CLUSTER_NAME" 75 | -------------------------------------------------------------------------------- /delete.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # "---------------------------------------------------------" 18 | # "- -" 19 | # "- Delete uninstalls Cassandra and deletes -" 20 | # "- the GKE cluster -" 21 | # "- -" 22 | # "---------------------------------------------------------" 23 | 24 | # Do not set errexit as it makes partial deletes impossible 25 | set -o nounset 26 | set -o pipefail 27 | 28 | ROOT=$(dirname "${BASH_SOURCE[0]}") 29 | CLUSTER_NAME="" 30 | ZONE="" 31 | 32 | # shellcheck disable=SC1090 33 | source "$ROOT"/common.sh 34 | 35 | # Get credentials for the k8s cluster 36 | gcloud container clusters get-credentials "$CLUSTER_NAME" --zone "$ZONE" 37 | 38 | # Delete cassandra 39 | echo "Deleting Cassandra" 40 | kubectl --namespace=default delete -f "$ROOT"/manifests 41 | # You have to wait the default pod grace period before you can delete the pvcs 42 | echo "Sleeping 60 seconds before deleting PVCs. The default pod grace period." 43 | sleep 60 44 | # delete the pvcs 45 | kubectl --namespace=default delete pvc -l app=cassandra 46 | 47 | # Cleanup the cluster 48 | echo "Deleting cluster" 49 | gcloud container clusters delete "$CLUSTER_NAME" --zone "$ZONE" --async --quiet 50 | -------------------------------------------------------------------------------- /images/gke-cassandra.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/gke-stateful-applications-demo/afe278de56109fe4b0b51e40998a089415c62f6a/images/gke-cassandra.png -------------------------------------------------------------------------------- /manifests/cassandra-pdb.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # PodDisruptionBudget See 17 | # https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ 18 | # 19 | # This PDB forces a minumum of two C* nodes are available. 20 | # 21 | apiVersion: policy/v1beta1 22 | kind: PodDisruptionBudget 23 | metadata: 24 | name: cassandra-pdb 25 | spec: 26 | minAvailable: 2 27 | selector: 28 | matchLabels: 29 | app: cassandra 30 | -------------------------------------------------------------------------------- /manifests/cassandra-service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Headless Services for C* 17 | # See 18 | # https://kubernetes.io/docs/concepts/services-networking/service/#headless-services 19 | # 20 | apiVersion: v1 21 | kind: Service 22 | metadata: 23 | labels: 24 | app: cassandra 25 | name: cassandra 26 | spec: 27 | clusterIP: None 28 | ports: 29 | - port: 9042 30 | selector: 31 | app: cassandra 32 | -------------------------------------------------------------------------------- /manifests/cassandra-statefulset.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # StatefulSet for C* 16 | # See https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/ 17 | apiVersion: "apps/v1" 18 | kind: StatefulSet 19 | metadata: 20 | name: cassandra 21 | labels: 22 | app: cassandra 23 | spec: 24 | serviceName: cassandra 25 | replicas: 3 # starting off with a small cluster of 3 nodes 26 | updateStrategy: 27 | type: RollingUpdate 28 | selector: 29 | matchLabels: 30 | app: cassandra 31 | template: 32 | metadata: 33 | labels: 34 | app: cassandra 35 | spec: 36 | affinity: 37 | podAntiAffinity: 38 | # The following sections are two examples of using anti pod affinity. 39 | # The uncommented section is enabled, or you can use the other example. 40 | # Both cannot be used at the same time, and thus the required rule. 41 | # is commented out. 42 | # 43 | # This is using pod anti-affinity to ensure that the scheduler tries 44 | # it best to spread pods 45 | # see https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature) 46 | preferredDuringSchedulingIgnoredDuringExecution: 47 | - weight: 100 48 | podAffinityTerm: 49 | labelSelector: 50 | matchExpressions: 51 | - key: app 52 | operator: In 53 | values: 54 | - cassandra 55 | topologyKey: kubernetes.io/hostname 56 | # Use pod anti-affinity to force the scheduler to not deploy two 57 | # Cassandra pod on the same node. 58 | # requiredDuringSchedulingIgnoredDuringExecution: 59 | # - labelSelector: 60 | # matchExpressions: 61 | # - key: app 62 | # operator: In 63 | # values: 64 | # - nginx 65 | # topologyKey: kubernetes.io/hostname 66 | 67 | # Allow the pods to be schedule on nodes that are tainted 68 | # see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ 69 | tolerations: 70 | - key: app 71 | operator: Equal 72 | value: cassandra 73 | effect: NoSchedule 74 | terminationGracePeriodSeconds: 1800 75 | containers: 76 | - name: cassandra 77 | image: gcr.io/pso-examples/cassandra:3.11.4-v8 78 | imagePullPolicy: Always 79 | ports: 80 | - containerPort: 7000 81 | name: intra-node 82 | - containerPort: 7001 83 | name: tls-intra-node 84 | - containerPort: 7199 85 | name: jmx 86 | - containerPort: 9042 87 | name: cql 88 | resources: # these are not production ready values 89 | limits: 90 | cpu: "1024m" 91 | # remember this memory value needs to include: 92 | # - heap 93 | # - off heap (file io for instance) 94 | # - and running nodetool for lifecycle 95 | memory: 1Gi 96 | requests: 97 | cpu: "1024m" 98 | memory: 1Gi 99 | securityContext: 100 | capabilities: 101 | add: 102 | - IPC_LOCK 103 | # this is still needs improvement for Cassandra 104 | lifecycle: 105 | preStop: 106 | exec: 107 | command: 108 | - /bin/sh 109 | - -c 110 | - nodetool drain 111 | env: 112 | # Various ENV VARS that can be set, a lot more are accessible via 113 | # the container. 114 | # Refer to the README.md in the container folder for more information. 115 | - name: MAX_HEAP_SIZE 116 | value: 512M 117 | - name: HEAP_NEWSIZE 118 | value: 100M 119 | - name: CASSANDRA_SEEDS 120 | value: "cassandra-0.cassandra.default.svc.cluster.local" 121 | - name: CASSANDRA_CLUSTER_NAME 122 | value: "K8Demo" 123 | - name: CASSANDRA_DC 124 | value: "DC1-K8Demo" 125 | - name: CASSANDRA_RACK 126 | value: "Rack1-K8Demo" 127 | # we are using the downward api to allow the pod get the correct 128 | # IP address. 129 | # See https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/ 130 | - name: POD_IP 131 | valueFrom: 132 | fieldRef: 133 | fieldPath: status.podIP 134 | readinessProbe: 135 | exec: 136 | command: 137 | - /bin/bash 138 | - -c 139 | - /ready-probe.sh 140 | initialDelaySeconds: 15 141 | timeoutSeconds: 5 142 | livenessProbe: 143 | exec: 144 | command: [ "/bin/sh", "-c", "/usr/local/apache-cassandra/bin/nodetool status" ] 145 | initialDelaySeconds: 90 146 | periodSeconds: 30 147 | # These volume mounts are persistent. They are like inline claims, 148 | # but not exactly because the names need to match exactly one of 149 | # the stateful pod volumes. 150 | volumeMounts: 151 | - name: cassandra-data 152 | mountPath: /cassandra_data 153 | # These are converted to volume claims by the controller 154 | # and mounted at the paths mentioned above. 155 | # Use SSD GCEPersistentDisk 156 | # For more info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ 157 | volumeClaimTemplates: 158 | - metadata: 159 | name: cassandra-data 160 | annotations: 161 | # StorageClass is contained in his demo as well 162 | volume.beta.kubernetes.io/storage-class: fast 163 | spec: 164 | accessModes: [ "ReadWriteOnce" ] 165 | resources: 166 | requests: 167 | storage: 10Gi 168 | -------------------------------------------------------------------------------- /manifests/cassandra-storageclass.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # 16 | # Storage class to use gce SSDs 17 | # See https://kubernetes.io/docs/concepts/storage/storage-classes/#gce 18 | # 19 | kind: StorageClass 20 | apiVersion: storage.k8s.io/v1 21 | metadata: 22 | name: fast 23 | provisioner: kubernetes.io/gce-pd 24 | parameters: 25 | type: pd-ssd 26 | replication-type: regional-pd 27 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.BUILD.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.Dockerfile.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.Makefile.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.WORKSPACE.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.bazel.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.bzl.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.css.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.go.preamble: -------------------------------------------------------------------------------- 1 | // +build 2 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.html.preamble: -------------------------------------------------------------------------------- 1 | 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.java.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.js.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.py.preamble: -------------------------------------------------------------------------------- 1 | #! 2 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.py.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.scss.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.sh.preamble: -------------------------------------------------------------------------------- 1 | #! 2 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.sh.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.tf.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.ts.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.xml.preamble: -------------------------------------------------------------------------------- 1 | 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.yaml.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/make.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This function checks to make sure that every 18 | # shebang has a '- e' flag, which causes it 19 | # to exit on error 20 | function check_bash() { 21 | find . -name "*.sh" | while IFS= read -d '' -r file; 22 | do 23 | if [[ "$file" != *"bash -e"* ]]; 24 | then 25 | echo "$file is missing shebang with -e"; 26 | exit 1; 27 | fi; 28 | done; 29 | } 30 | 31 | # This function makes sure that the required files for 32 | # releasing to OSS are present 33 | function basefiles() { 34 | echo "Checking for required files" 35 | test -f CONTRIBUTING.md || echo "Missing CONTRIBUTING.md" 36 | test -f LICENSE || echo "Missing LICENSE" 37 | test -f README.md || echo "Missing README.md" 38 | } 39 | 40 | # This function runs the hadolint linter on 41 | # every file named 'Dockerfile' 42 | function docker() { 43 | echo "Running hadolint on Dockerfiles" 44 | find . -name "Dockerfile" -exec hadolint {} \; 45 | } 46 | 47 | # This function runs 'terraform validate' against all 48 | # files ending in '.tf' 49 | function check_terraform() { 50 | echo "Running terraform validate" 51 | #shellcheck disable=SC2156 52 | find . -name "*.tf" -exec bash -c 'terraform validate --check-variables=false $(dirname "{}")' \; 53 | } 54 | 55 | # This function runs 'go fmt' and 'go vet' on eery file 56 | # that ends in '.go' 57 | function golang() { 58 | echo "Running go fmt and go vet" 59 | find . -name "*.go" -exec go fmt {} \; 60 | find . -name "*.go" -exec go vet {} \; 61 | } 62 | 63 | # This function runs the flake8 linter on every file 64 | # ending in '.py' 65 | function check_python() { 66 | echo "Running flake8" 67 | find . -name "*.py" -exec flake8 {} \; 68 | } 69 | 70 | # This function runs the shellcheck linter on every 71 | # file ending in '.sh' 72 | function check_shell() { 73 | echo "Running shellcheck" 74 | find . -name "*.sh" -exec shellcheck -x {} \; 75 | } 76 | 77 | # This function makes sure that there is no trailing whitespace 78 | # in any files in the project. 79 | # There are some exclusions 80 | function check_trailing_whitespace() { 81 | echo "The following lines have trailing whitespace" 82 | grep -r '[[:blank:]]$' --exclude-dir=".terraform" --exclude="*.png" --exclude-dir=".git" --exclude="*.pyc" . 83 | rc=$? 84 | if [ $rc = 0 ]; then 85 | exit 1 86 | fi 87 | } 88 | -------------------------------------------------------------------------------- /test/verify_boilerplate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.7 2 | # Copyright 2018 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # Verifies that all source files contain the necessary copyright boilerplate 16 | # snippet. 17 | 18 | # This code is based on existing work 19 | # https://partner-code.googlesource.com/helmsman-cardinal/+/refs/heads/master/helmsman-template-project/test/verify_boilerplate.py 20 | 21 | 22 | """ 23 | A runnable module to test the presence of boilerplate 24 | text in files within a repo. 25 | """ 26 | 27 | from __future__ import print_function 28 | from subprocess import run, CalledProcessError 29 | import argparse 30 | import glob 31 | import os 32 | import re 33 | import sys 34 | 35 | 36 | # These directories will be omitted from header checks 37 | SKIPPED_PATHS = [ 38 | 'Godeps', 'third_party', '_gopath', '_output', 39 | '.git', 'vendor', '__init__.py', 'node_modules', 40 | 'bazel-out', 'external', '3rdparty' 41 | ] 42 | 43 | # A map of regular expressions used in boilerplate validation. 44 | # The date regex is used in validating the date referenced 45 | # is the boilerplate, by ensuring it is an acceptable year. 46 | REGEXES = { 47 | # beware the Y2100 problem 48 | "date": re.compile(r'(20\d\d)') 49 | } 50 | 51 | 52 | def get_args(): 53 | """Parses command line arguments. 54 | Configures and runs argparse.ArgumentParser to extract command line 55 | arguments. 56 | Returns: 57 | An argparse.Namespace containing the arguments parsed from the 58 | command line 59 | """ 60 | parser = argparse.ArgumentParser() 61 | 62 | parser.add_argument("filenames", 63 | help="""A list of files to check, all in repo are 64 | checked if this is unspecified.""", 65 | nargs='*') 66 | 67 | parser.add_argument("-f", "--force-extension", 68 | default="", 69 | help="""Force an extension to compare against. Useful 70 | for files without extensions, such as runnable shell 71 | scripts .""") 72 | 73 | parser.add_argument( 74 | "-r", "--rootdir", 75 | default=None, 76 | help="""Root directory of repository. If not specified, the script will 77 | attempt to draw this value from git.""") 78 | 79 | parser.add_argument("-b", "--boilerplate-dir", 80 | default=None, 81 | help="""Directory with boilerplate files. Defaults to 82 | [root]/test/boilerplate.""") 83 | 84 | args = parser.parse_args() 85 | 86 | if not args.rootdir: 87 | ask_git = run( 88 | ["git", "rev-parse", "--show-toplevel"], 89 | capture_output=True, text=True) 90 | try: 91 | ask_git.check_returncode() 92 | except CalledProcessError: 93 | print("""No root specfied and directory does not seem to be a git 94 | repository, or git is not installed.""", file=sys.stderr) 95 | sys.exit(1) 96 | args.rootdir = ask_git.stdout.strip() 97 | 98 | if not args.boilerplate_dir: 99 | args.boilerplate_dir = os.path.join(args.rootdir, "test/boilerplate") 100 | 101 | return args 102 | 103 | 104 | def get_references(args): 105 | """Reads each reference boilerplate file's contents into an array, then 106 | adds that array to a dictionary keyed by the file extension. 107 | 108 | Returns: 109 | A dictionary of boilerplate lines, keyed by file extension. 110 | For example, boilerplate.py.txt would result in the 111 | k,v pair {".py": py_lines} where py_lines is an array 112 | containing each line of the file. 113 | """ 114 | references = {} 115 | 116 | # Find all paths for boilerplate references 117 | boilerplate_paths = glob.glob( 118 | os.path.join(args.boilerplate_dir, "boilerplate.*.txt")) 119 | 120 | # Read all boilerplate references into dictionary 121 | for path in boilerplate_paths: 122 | with open(path, 'r') as ref_file: 123 | extension = os.path.basename(path).split(".")[1] 124 | ref = ref_file.read().splitlines() 125 | references[extension] = ref 126 | 127 | return references 128 | 129 | 130 | # Improvement: combine this function with `get_references` 131 | def get_preambles(args): 132 | """Reads each preamble boilerplate file's contents into an array, then 133 | adds that array to a dictionary keyed by the file extension. 134 | 135 | Returns: 136 | A dictionary of boilerplate lines, keyed by file extension. 137 | For example, boilerplate.py.preamble would result 138 | in the k,v pair {".py": py_lines} where py_lines is 139 | an array containing each line of the file 140 | (ex: "#!/usr/bin/env python3.7") 141 | """ 142 | preambles = {} 143 | 144 | # Find all paths for boilerplate preambles 145 | boilerplate_paths = glob.glob( 146 | os.path.join(args.boilerplate_dir, "boilerplate.*.preamble")) 147 | 148 | # Read all boilerplate preambles into dictionary 149 | for path in boilerplate_paths: 150 | with open(path, 'r') as ref_file: 151 | extension = os.path.basename(path).split(".")[1] 152 | ref = ref_file.read().splitlines() 153 | preambles[extension] = ref 154 | 155 | return preambles 156 | 157 | 158 | def has_valid_header(filename, references, preambles, regexs, args): 159 | """Test whether a file has the correct boilerplate header. 160 | Tests each file against the boilerplate stored in refs for that file type 161 | (based on extension), or by the entire filename (eg Dockerfile, Makefile). 162 | Some heuristics are applied to remove build tags and shebangs, but little 163 | variance in header formatting is tolerated. 164 | Args: 165 | filename: A string containing the name of the file to test 166 | references: A map of reference boilerplate text, 167 | keyed by file extension 168 | preambles: A map of preamble boilerplate text, keyed by file extension 169 | regexs: a map of compiled regex objects used in verifying boilerplate 170 | Returns: 171 | True if the file has the correct boilerplate header, otherwise returns 172 | False. 173 | """ 174 | # Read the entire file. 175 | with open(filename, 'r') as test_file: 176 | data = test_file.read() 177 | 178 | # Select the appropriate reference based on the extension, 179 | # or if none, the file name. 180 | basename, extension = get_file_parts(filename) 181 | if args.force_extension: 182 | extension = args.force_extension 183 | elif extension: 184 | extension = extension 185 | else: 186 | extension = basename 187 | ref = references[extension] 188 | print("Verifying boilerplate in file: %s as %s" % ( 189 | os.path.relpath(filename, args.rootdir), 190 | extension)) 191 | 192 | preamble = preambles.get(extension) 193 | if preamble: 194 | preamble = re.escape("\n".join(preamble)) 195 | regflags = re.MULTILINE | re.IGNORECASE 196 | regex = re.compile(r"^(%s.*\n)\n*" % preamble, regflags) 197 | (data, _) = regex.subn("", data, 1) 198 | 199 | data = data.splitlines() 200 | 201 | # if our test file is smaller than the reference it surely fails! 202 | if len(ref) > len(data): 203 | return False 204 | # truncate our file to the same number of lines as the reference file 205 | data = data[:len(ref)] 206 | 207 | # if we don't match the reference at this point, fail 208 | if ref != data: 209 | return False 210 | 211 | return True 212 | 213 | 214 | def get_file_parts(filename): 215 | """Extracts the basename and extension parts of a filename. 216 | Identifies the extension as everything after the last period in filename. 217 | Args: 218 | filename: string containing the filename 219 | Returns: 220 | A tuple of: 221 | A string containing the basename 222 | A string containing the extension in lowercase 223 | """ 224 | extension = os.path.splitext(filename)[1].split(".")[-1].lower() 225 | basename = os.path.basename(filename) 226 | return basename, extension 227 | 228 | 229 | def normalize_files(files, args): 230 | """Extracts the files that require boilerplate checking from the files 231 | argument. 232 | A new list will be built. Each path from the original files argument will 233 | be added unless it is within one of SKIPPED_DIRS. All relative paths will 234 | be converted to absolute paths by prepending the root_dir path parsed from 235 | the command line, or its default value. 236 | Args: 237 | files: a list of file path strings 238 | Returns: 239 | A modified copy of the files list where any any path in a skipped 240 | directory is removed, and all paths have been made absolute. 241 | """ 242 | newfiles = [f for f in files if not any(s in f for s in SKIPPED_PATHS)] 243 | 244 | for idx, pathname in enumerate(newfiles): 245 | if not os.path.isabs(pathname): 246 | newfiles[idx] = os.path.join(args.rootdir, pathname) 247 | return newfiles 248 | 249 | 250 | def get_files(extensions, args): 251 | """Generates a list of paths whose boilerplate should be verified. 252 | If a list of file names has been provided on the command line, it will be 253 | treated as the initial set to search. Otherwise, all paths within rootdir 254 | will be discovered and used as the initial set. 255 | Once the initial set of files is identified, it is normalized via 256 | normalize_files() and further stripped of any file name whose extension is 257 | not in extensions. 258 | Args: 259 | extensions: a list of file extensions indicating which file types 260 | should have their boilerplate verified 261 | Returns: 262 | A list of absolute file paths 263 | """ 264 | files = [] 265 | if args.filenames: 266 | files = args.filenames 267 | else: 268 | for root, dirs, walkfiles in os.walk(args.rootdir): 269 | # don't visit certain dirs. This is just a performance improvement 270 | # as we would prune these later in normalize_files(). But doing it 271 | # cuts down the amount of filesystem walking we do and cuts down 272 | # the size of the file list 273 | for dpath in SKIPPED_PATHS: 274 | if dpath in dirs: 275 | dirs.remove(dpath) 276 | for name in walkfiles: 277 | pathname = os.path.join(root, name) 278 | files.append(pathname) 279 | files = normalize_files(files, args) 280 | outfiles = [] 281 | for pathname in files: 282 | basename, extension = get_file_parts(pathname) 283 | extension_present = extension in extensions or basename in extensions 284 | if args.force_extension or extension_present: 285 | outfiles.append(pathname) 286 | return outfiles 287 | 288 | 289 | def main(args): 290 | """Identifies and verifies files that should have the desired boilerplate. 291 | Retrieves the lists of files to be validated and tests each one in turn. 292 | If all files contain correct boilerplate, this function terminates 293 | normally. Otherwise it prints the name of each non-conforming file and 294 | exists with a non-zero status code. 295 | """ 296 | refs = get_references(args) 297 | preambles = get_preambles(args) 298 | filenames = get_files(refs.keys(), args) 299 | nonconforming_files = [] 300 | for filename in filenames: 301 | if not has_valid_header(filename, refs, preambles, REGEXES, args): 302 | nonconforming_files.append(filename) 303 | if nonconforming_files: 304 | print('%d files have incorrect boilerplate headers:' % len( 305 | nonconforming_files)) 306 | for filename in sorted(nonconforming_files): 307 | print(os.path.relpath(filename, args.rootdir)) 308 | sys.exit(1) 309 | else: 310 | print('All files examined have correct boilerplate.') 311 | 312 | 313 | if __name__ == "__main__": 314 | ARGS = get_args() 315 | main(ARGS) 316 | -------------------------------------------------------------------------------- /update_image_tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # "------------------------------------------------------------------------" 18 | # "- -" 19 | # "- This script generates the cassandra-statefulset.yaml with new tag -" 20 | # "- -" 21 | # "------------------------------------------------------------------------" 22 | 23 | set -o errexit 24 | set -o nounset 25 | set -o pipefail 26 | 27 | PROJECT_ID=$1 28 | APP_NAME=$2 29 | IMAGE_TAG=$3 30 | MANIFEST_FILE=$4 31 | sed -i -e "s#gcr.io/${PROJECT_ID}/${APP_NAME}:.*#gcr.io/${PROJECT_ID}/${APP_NAME}:${IMAGE_TAG}#" \ 32 | "${MANIFEST_FILE}" 33 | -------------------------------------------------------------------------------- /validate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # "---------------------------------------------------------" 18 | # "- -" 19 | # "- Validation script checks if Cassandra -" 20 | # "- deployed successfully. -" 21 | # "- -" 22 | # "---------------------------------------------------------" 23 | 24 | # Do no set exit on error, since the rollout status command may fail 25 | set -o nounset 26 | set -o pipefail 27 | 28 | ROOT=$(dirname "${BASH_SOURCE[0]}") 29 | CLUSTER_NAME="" 30 | ZONE="" 31 | 32 | # shellcheck disable=SC1090 33 | source "$ROOT"/common.sh 34 | 35 | # Get credentials for the k8s cluster 36 | gcloud container clusters get-credentials "$CLUSTER_NAME" --zone "$ZONE" 37 | 38 | # Get rollout status for the statefulset and grep for the complete message 39 | MESSAGE="statefulset rolling update complete" 40 | ROLLOUT=$(kubectl --namespace=default rollout status --timeout="10m" -f \ 41 | "$ROOT"/manifests/cassandra-statefulset.yaml) 42 | 43 | # Test the ROLLOUT variable to see if the grep has returned the expected value. 44 | # Depending on the test print success or failure messages. 45 | if [[ $ROLLOUT = *"$MESSAGE"* ]]; then 46 | echo "Validation Passed: the Statefulset has been deployed" 47 | else 48 | echo "Validation Failed: Statefulset has not been deployed" 49 | exit 1 50 | fi 51 | --------------------------------------------------------------------------------