├── .circleci └── config.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── SECURITY.md ├── bin ├── create-release └── easy-cass-lab ├── build.gradle.kts ├── config └── bash_profile ├── core ├── build.gradle.kts └── src │ └── main │ └── kotlin │ └── com │ └── rustyrazorblade │ └── easycasslab │ └── core │ └── YamlDelegate.kt ├── dashboards ├── build.gradle ├── dashboards │ ├── caches.jsonnet │ ├── jvm.jsonnet │ ├── overview.jsonnet │ ├── resources.jsonnet │ └── stress.jsonnet ├── docker-compose.yml └── monitoring-environment │ ├── grafana.ini │ ├── prometheus.yml │ └── provisioning │ ├── dashboards │ └── cassandra.yaml │ └── datasources │ └── datasource.yml ├── docker-grafonnet ├── build.gradle └── docker │ ├── Dockerfile │ └── entrypoint.sh ├── docs ├── development.html └── index.html ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── manual ├── build.gradle └── src │ ├── development.adoc │ ├── includes │ └── help.txt │ └── index.adoc ├── packer ├── Dockerfile ├── README.md ├── base │ ├── aliases.sh │ ├── base.pkr.hcl │ └── install │ │ ├── install_async_profiler.sh │ │ ├── install_bcc.sh │ │ ├── install_fio.sh │ │ ├── install_otel_collector.sh │ │ ├── install_python.sh │ │ └── prepare_instance.sh ├── cassandra │ ├── axonops-sudoers │ ├── bin │ │ ├── cass-yaml-search │ │ ├── cassandra-pid │ │ ├── flamegraph │ │ ├── iostat-plot │ │ ├── patch-config │ │ ├── restart-cassandra-and-wait │ │ ├── set-java-version │ │ ├── setup-axonops │ │ ├── sjk-gc │ │ ├── sjk-mx │ │ ├── use-cassandra │ │ └── wait-for-up-normal │ ├── cassandra.in.sh │ ├── cassandra.pkr.hcl │ ├── cassandra_versions.yaml │ ├── config │ │ └── cassandra-sidecar.yaml │ ├── environment │ ├── htoprc │ ├── install │ │ ├── install_axon.sh │ │ ├── install_cassandra.sh │ │ ├── install_easy_cass_stress.sh │ │ ├── install_sidecar.sh │ │ └── prepare_instance.sh │ ├── patch-jvm-options.py │ ├── services │ │ ├── cassandra-sidecar.service │ │ └── cassandra.service │ └── test │ │ ├── jvm-server.options │ │ ├── jvm.patch.options │ │ └── test.options └── docker-compose.yml ├── settings.gradle └── src ├── integration-test └── kotlin │ └── com │ └── rustyrazorblade │ └── easycasslab │ └── MainTest.kt ├── main ├── kotlin │ └── com │ │ └── rustyrazorblade │ │ └── easycasslab │ │ ├── Cassandra.kt │ │ ├── CommandLineParser.kt │ │ ├── Containers.kt │ │ ├── Context.kt │ │ ├── Docker.kt │ │ ├── EC2.kt │ │ ├── Main.kt │ │ ├── ResourceFile.kt │ │ ├── Utils.kt │ │ ├── commands │ │ ├── BuildBaseImage.kt │ │ ├── BuildCassandraImage.kt │ │ ├── BuildImage.kt │ │ ├── Clean.kt │ │ ├── ConfigureAxonOps.kt │ │ ├── Down.kt │ │ ├── DownloadConfig.kt │ │ ├── Hosts.kt │ │ ├── ICommand.kt │ │ ├── Init.kt │ │ ├── ListVersions.kt │ │ ├── Repl.kt │ │ ├── Restart.kt │ │ ├── SetupInstance.kt │ │ ├── Start.kt │ │ ├── StartAxonOps.kt │ │ ├── Stop.kt │ │ ├── Up.kt │ │ ├── UpdateConfig.kt │ │ ├── UploadAuthorizedKeys.kt │ │ ├── UseCassandra.kt │ │ ├── Version.kt │ │ ├── WriteConfig.kt │ │ ├── converters │ │ │ └── AZConverter.kt │ │ ├── delegates │ │ │ ├── BuildArgs.kt │ │ │ └── Hosts.kt │ │ └── formatters │ │ │ └── HostOutput.kt │ │ ├── configuration │ │ ├── CassandraVersion.kt │ │ ├── CassandraYaml.kt │ │ ├── ClusterState.kt │ │ ├── Dashboards.kt │ │ ├── Host.kt │ │ ├── HostInfo.kt │ │ ├── Prometheus.kt │ │ ├── Seeds.kt │ │ ├── ServerType.kt │ │ ├── TFState.kt │ │ └── User.kt │ │ ├── containers │ │ ├── Packer.kt │ │ └── Terraform.kt │ │ ├── ssh │ │ ├── ConnectionManager.kt │ │ ├── ISSHClient.kt │ │ ├── MockSSHClient.kt │ │ ├── Response.kt │ │ └── SSHClient.kt │ │ └── terraform │ │ ├── AWSConfiguration.kt │ │ └── AWSResources.kt └── resources │ ├── com │ └── rustyrazorblade │ │ └── easycasslab │ │ ├── commands │ │ ├── axonops-dashboards.json │ │ └── setup_instance.sh │ │ └── configuration │ │ └── env.sh │ └── log4j2.yaml └── test ├── kotlin └── com │ └── rustyrazorblade │ └── easycasslab │ ├── ContextTest.kt │ ├── DockerTest.kt │ ├── commands │ └── converters │ │ └── AZConverterTest.kt │ ├── configuration │ ├── CassandraVersionTest.kt │ ├── CassandraYamlTest.kt │ ├── HostTest.kt │ ├── PrometheusTest.kt │ └── SeedsTest.kt │ └── containers │ └── PackerTest.kt └── resources ├── com └── rustyrazorblade │ └── easycasslab │ └── configuration │ ├── cassandra.yaml │ ├── extra_versions │ └── extra_version.yaml │ ├── seeds.txt │ ├── terraform-one-node.tfstate │ └── terraform.tfstate └── log4j2-test.properties /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | jobs: 4 | build: 5 | machine: 6 | image: ubuntu-1604:201903-01 7 | # docker_layer_caching: true 8 | 9 | working_directory: ~/repo 10 | 11 | steps: 12 | - run: sudo apt-get update 13 | - run: sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common 14 | 15 | - run: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 16 | - run: sudo apt-key fingerprint 0EBFCD88 17 | - run: | 18 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" 19 | 20 | - run: sudo apt-get update 21 | - checkout 22 | - run: ./gradlew test --stacktrace 23 | - store_test_results: 24 | path: build/reports/tests/test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | *.iml 3 | *.pyc 4 | .DS_Store 5 | .vagrant.v1* 6 | .tags* 7 | /.gradle/ 8 | /out/ 9 | build/ 10 | .jira-url 11 | /test/ 12 | /.terraform* 13 | /build_cassandra.sh* 14 | /copy_provisioning_resources.sh* 15 | /create_provisioning_resources.sh 16 | /parallel_ssh.sh* 17 | /hosts.txt 18 | /known_hosts 19 | /provisioning/ 20 | /seeds.txt 21 | /stress_ips.txt 22 | /terraform.tf.json 23 | /terraform.tfstate 24 | /terraform.tfvars 25 | /terraform.tfstate.backup 26 | logs/ 27 | sshConfig 28 | /scripts 29 | logs 30 | /authorized_keys 31 | 32 | 33 | # Created by https://www.gitignore.io/api/eclipse 34 | # Edit at https://www.gitignore.io/?templates=eclipse 35 | 36 | ### Eclipse ### 37 | 38 | .metadata 39 | tmp/ 40 | *.tmp 41 | *.bak 42 | *.swp 43 | *~.nib 44 | local.properties 45 | .settings/ 46 | .loadpath 47 | .recommenders 48 | 49 | # External tool builders 50 | .externalToolBuilders/ 51 | 52 | # Locally stored "Eclipse launch configurations" 53 | *.launch 54 | 55 | # PyDev specific (Python IDE for Eclipse) 56 | *.pydevproject 57 | 58 | # CDT-specific (C/C++ Development Tooling) 59 | .cproject 60 | 61 | # CDT- autotools 62 | .autotools 63 | 64 | # Java annotation processor (APT) 65 | .factorypath 66 | 67 | # PDT-specific (PHP Development Tools) 68 | .buildpath 69 | 70 | # sbteclipse plugin 71 | .target 72 | 73 | # Tern plugin 74 | .tern-project 75 | 76 | # TeXlipse plugin 77 | .texlipse 78 | 79 | # STS (Spring Tool Suite) 80 | .springBeans 81 | 82 | # Code Recommenders 83 | .recommenders/ 84 | 85 | # Annotation Processing 86 | .apt_generated/ 87 | 88 | # Scala IDE specific (Scala & Java development for Eclipse) 89 | .cache-main 90 | .scala_dependencies 91 | .worksheet 92 | 93 | ### Eclipse Patch ### 94 | # Eclipse Core 95 | .project 96 | 97 | # JDT-specific (Eclipse Java Development Tools) 98 | .classpath 99 | 100 | # Annotation Processing 101 | .apt_generated 102 | 103 | .sts4-cache/ 104 | 105 | # End of https://www.gitignore.io/api/eclipse 106 | src/main/resources/com/rustyrazorblade/easycasslab/instances/importers/instances.json 107 | /prometheus.yml 108 | /env.sh 109 | /artifacts 110 | easy-cass-lab.ipr 111 | easy-cass-lab.iws 112 | 113 | /prometheus_labels.yml 114 | /src/main/dashboards/ 115 | out/ 116 | packer/ec2_ubuntu.pem 117 | 118 | cassandra.patch.yaml 119 | 120 | environment.sh 121 | 122 | disk_setup.sh 123 | 124 | /setup_instance.sh 125 | cassandra_versions_extra.yaml 126 | state.json 127 | /cassandra_versions.yaml 128 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | . 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributions are welcome and encouraged, although it is a good idea to check before writing any code for a new feature to make sure it lines up with the project roadmap. 2 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Original work prior to 2024 2 | Copyright 2018 The Last Pickle 3 | 4 | Adopted by Rustyrazorblade in 2024 5 | Copyright 2024 Rustyrazorblade Consulting 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | The project does not have a separate security policy from the regular release schedule. We will release often, incorporating bug fixes, new features, and security fixes all in one. 4 | -------------------------------------------------------------------------------- /bin/create-release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Release script to be used for the main repo. 4 | # Requires the GitHub client (gh) to be installed. 5 | # No actions should be taken until the release is confirmed! 6 | 7 | # Variables 8 | VERSION_FILE="gradle.properties" 9 | VERSION_KEY="version" 10 | DEBUG=false 11 | GH_RELEASE_DEBUG="" 12 | 13 | HOMEBREW_FORMULA="${RRB_HOMEBREW_REPO}/Formula/easy-cass-lab.rb" 14 | 15 | 16 | # Parse command line options 17 | while getopts "d" opt; do 18 | case $opt in 19 | d) 20 | DEBUG=true 21 | ;; 22 | \?) 23 | echo "Invalid option: -$OPTARG" >&2 24 | exit 1 25 | ;; 26 | esac 27 | done 28 | 29 | ##### Helper Functions ##### 30 | function get_current_version { 31 | grep "^$VERSION_KEY" "$VERSION_FILE" | cut -d'=' -f2 32 | } 33 | 34 | function bump_version { 35 | echo $(( $1 + 1 )) 36 | } 37 | 38 | function update_version_file { 39 | echo "Updating $VERSION_FILE, version=$1" 40 | sed -i '' "s/^$VERSION_KEY=.*/$VERSION_KEY=$1/" "$VERSION_FILE" 41 | if [[ "$DEBUG" = false ]]; then 42 | git add "$VERSION_FILE" 43 | git commit -m "Bump version to $1" 44 | else 45 | echo "Skipping commit" 46 | fi 47 | } 48 | 49 | ##### End Helper Functions ##### 50 | 51 | ##### Sanity Checks. Debug mode outputs all commands. ##### 52 | 53 | if [ "$DEBUG" = true ]; then 54 | set -x 55 | fi 56 | 57 | ##### Check if gh is installed #### 58 | if ! command -v gh &> /dev/null; then 59 | echo "gh CLI is not installed. Please install it and try again." 60 | exit 1 61 | fi 62 | 63 | ##### ensure the formula exists ##### 64 | if [ ! -f "$HOMEBREW_FORMULA" ]; then 65 | echo "Error: File $HOMEBREW_FORMULA does not exist." 66 | exit 1 67 | fi 68 | 69 | ##### ensure docker is running 70 | if ! docker info > /dev/null 2>&1; then 71 | echo "Error: Docker is not running." 72 | exit 1 73 | fi 74 | 75 | ##### End Sanity Checks ##### 76 | 77 | ####### Main script ####### 78 | 79 | if [ "$DEBUG" = false ]; then 80 | if ! git diff-index --quiet HEAD --; then 81 | echo "You have uncommitted changes. Please commit or stash them before running this script." 82 | exit 1 83 | fi 84 | else 85 | GH_RELEASE_DEBUG="--draft" 86 | fi 87 | 88 | current_version=$(get_current_version) 89 | new_version=$(bump_version "$current_version") 90 | 91 | echo "Current version: $current_version, new version: $new_version" 92 | 93 | echo "Building..." 94 | 95 | gw distTar 96 | RELEASE_FILE="build/distributions/easy-cass-lab-${current_version}.tar.gz" 97 | 98 | if [ ! -f "$RELEASE_FILE" ]; then 99 | echo "Release file $RELEASE_FILE does not exist, exiting." 100 | exit 1 101 | fi 102 | 103 | #### SHA256 and URL for the repo update 104 | ARTIFACT_SHA=$(sha2 -256 -q $RELEASE_FILE) 105 | 106 | echo "SHA: $ARTIFACT_SHA" 107 | 108 | echo "Current version: $current_version, next version: $new_version. Enter to continue, control-c to quit." 109 | 110 | read 111 | 112 | if [ "$DEBUG" = false ]; then 113 | bin/easy-cass-lab build-image --release 114 | echo "Press enter to continue" 115 | read 116 | else 117 | echo "In debug mode, skipping image building" 118 | fi 119 | 120 | 121 | ##### Copy image to other regions ##### 122 | 123 | ##### Create release in GitHub ##### 124 | if [ "$DEBUG" = false ]; then 125 | gh release create "v${current_version}" --generate-notes $RELEASE_FILE 126 | else 127 | echo "Skipping gh create create because we're in debug mode." 128 | fi 129 | 130 | ##### Update Homebrew 131 | 132 | # Update the `version` and `sha256` fields. 133 | sed -i '' "s/^\([[:space:]]*\)version.*/\1version \"${current_version}\"/" $HOMEBREW_FORMULA 134 | sed -i '' "s/^\([[:space:]]*\)sha256.*/\1sha256 \"${ARTIFACT_SHA}\"/" $HOMEBREW_FORMULA 135 | 136 | # commit and push 137 | if [ "$DEBUG" = false ]; then 138 | ( 139 | cd $RRB_HOMEBREW_REPO 140 | git commit -am "Version bump to $current_version" 141 | git push 142 | ) 143 | fi 144 | 145 | ##### Done with the release, now we increment gradle's version ##### 146 | 147 | update_version_file "$new_version" 148 | 149 | echo "Release $new_version created successfully." 150 | 151 | -------------------------------------------------------------------------------- /bin/easy-cass-lab: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script is for local development 3 | # The release bin script is generated by the gradle build 4 | 5 | dir=$(dirname $0) 6 | 7 | APP_HOME="$(dirname "$dir")" 8 | APP_HOME=$(readlink -f $APP_HOME) 9 | 10 | EASY_CASS_LAB_USER_DATA=~/.easy-cass-lab/ 11 | 12 | VERSION=$(grep '^version' "${APP_HOME}/gradle.properties" | cut -d '=' -f2) 13 | 14 | JAR="${APP_HOME}/build/libs/easy-cass-lab-${VERSION}-all.jar" 15 | 16 | java -Deasycasslab.version=$VERSION -Deasycasslab.apphome=$APP_HOME -jar $JAR "$@" 17 | 18 | -------------------------------------------------------------------------------- /config/bash_profile: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | alias v="ls -lahG" -------------------------------------------------------------------------------- /core/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | idea 3 | java 4 | kotlin("jvm") 5 | } 6 | 7 | kotlin { 8 | jvmToolchain(17) 9 | } 10 | 11 | dependencies { 12 | implementation("org.apache.logging.log4j:log4j-api-kotlin:${rootProject.extra["log4j_api_version"]}") 13 | implementation("org.apache.logging.log4j:log4j-core:${rootProject.extra["log4j_core_version"]}") 14 | implementation("org.apache.logging.log4j:log4j-slf4j18-impl:${rootProject.extra["slf4j_version"]}") 15 | 16 | // https://mvnrepository.com/artifact/org.jetbrains.kotlin/kotlin-stdlib 17 | implementation("org.jetbrains.kotlin:kotlin-stdlib:${rootProject.extra["kotlin_version"]}") 18 | 19 | implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${rootProject.extra["jackson_dataformat_version"]}") 20 | implementation("com.fasterxml.jackson.module:jackson-module-kotlin:${rootProject.extra["jackson_kotlin_version"]}") 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/kotlin/com/rustyrazorblade/easycasslab/core/YamlDelegate.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.core 2 | 3 | import com.fasterxml.jackson.databind.DeserializationFeature 4 | import com.fasterxml.jackson.databind.ObjectMapper 5 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 6 | import com.fasterxml.jackson.module.kotlin.registerKotlinModule 7 | import kotlin.reflect.KProperty 8 | 9 | /** 10 | * Making yaml mapping easy 11 | * Usage example: 12 | * 13 | * val yaml : ObjectMapper by YamlDelegate 14 | */ 15 | class YamlDelegate(val ignoreUnknown : Boolean = false) { 16 | operator fun getValue(thisRef: Any?, property: KProperty<*>) : ObjectMapper { 17 | if(ignoreUnknown) { 18 | return com.rustyrazorblade.easycasslab.core.YamlDelegate.Companion.yaml.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) 19 | } 20 | return com.rustyrazorblade.easycasslab.core.YamlDelegate.Companion.yaml 21 | } 22 | 23 | companion object { 24 | val yaml = ObjectMapper(YAMLFactory()).registerKotlinModule() 25 | } 26 | } 27 | 28 | -------------------------------------------------------------------------------- /dashboards/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'docker-compose' 3 | id 'java' 4 | id 'distribution' 5 | } 6 | 7 | version "1.0" 8 | 9 | def dashboardSourceLocation = "$buildDir/dashboards" 10 | def dashboardOutput = "$dashboardSourceLocation/com/rustyrazorblade/dashboards" 11 | 12 | task startDependencies(type:Exec) { 13 | group = "dashboard" 14 | description = "Start Cassandra and wait for it to be up" 15 | 16 | environment "DASHBOARD_DIR", dashboardOutput 17 | environment "BUILD_DIR", dashboardOutput 18 | 19 | commandLine "docker-compose", "run", "start_dependencies" 20 | } 21 | 22 | task preview(type:Exec) { 23 | group = "dashboard" 24 | description = "Start the dashboard dev environment" 25 | commandLine "docker-compose", "up" 26 | 27 | environment "DASHBOARD_DIR", dashboardOutput 28 | environment "BUILD_DIR", dashboardOutput 29 | 30 | dependsOn startDependencies 31 | } 32 | 33 | task stopPreview(type:Exec) { 34 | group = "dashboard" 35 | description = "Stops the docker compose environment" 36 | 37 | environment "DASHBOARD_DIR", dashboardOutput 38 | environment "BUILD_DIR", dashboardOutput 39 | 40 | commandLine "docker-compose", "stop" 41 | } 42 | 43 | task generateDashboards(type: Exec) { 44 | commandLine "docker-compose", "up", "--exit-code-from", "grafonnet", "grafonnet" 45 | 46 | dependsOn(":docker-grafonnet:buildDocker") 47 | 48 | standardOutput = new ByteArrayOutputStream() 49 | errorOutput = new ByteArrayOutputStream() 50 | group = "dashboard" 51 | description = "Regenerate the Grafana Dashboards using Docker." 52 | 53 | inputs.files(project.fileTree(dir: "dashboards")) 54 | 55 | outputs.dir(dashboardOutput) 56 | .withPropertyName("dashboards") 57 | 58 | environment "BUILD_DIR", dashboardOutput 59 | environment "DASHBOARD_DIR", dashboardOutput 60 | 61 | ignoreExitValue true 62 | doLast { 63 | logger.info(standardOutput.toString()) 64 | logger.error(errorOutput.toString()) 65 | // if (executionResult.getExitValue() != 0) { 66 | // print(standardOutput) 67 | // throw new GradleScriptException("non zero exit code") 68 | // } 69 | } 70 | } 71 | 72 | installDist.dependsOn(":dashboards:generateDashboards") 73 | distTar.dependsOn(":dashboards:generateDashboards") 74 | distZip.dependsOn(":dashboards:generateDashboards") 75 | 76 | sourceSets { 77 | main { 78 | output.dir(dashboardSourceLocation , builtBy: 'generateDashboards') 79 | } 80 | } 81 | 82 | distributions { 83 | main { 84 | distributionBaseName = "easy-cass-lab-dashboards" 85 | 86 | contents { 87 | from dashboardOutput 88 | } 89 | } 90 | } 91 | 92 | -------------------------------------------------------------------------------- /dashboards/dashboards/caches.jsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local dashboard = grafana.dashboard; 3 | local row = grafana.row; 4 | local graphPanel = grafana.graphPanel; 5 | local tablePanel = grafana.tablePanel; 6 | local singleStatPanel = grafana.singlestat; 7 | local textPanel = grafana.text; 8 | local prometheus = grafana.prometheus; 9 | local template = grafana.template; 10 | 11 | 12 | // used in the single stat panels where higher is better - cache hit rates for example 13 | local reversedColors =[ 14 | '#d44a3a', 15 | 'rgba(237, 129, 40, 0.89)', 16 | '#299c46', 17 | ]; 18 | 19 | local smallGrid = { 20 | 'w': 4, 21 | 'h': 4 22 | }; 23 | 24 | local singleStatCachePanel(name, scope) = 25 | singleStatPanel.new( 26 | name, 27 | span=2, 28 | colors=reversedColors, 29 | datasource='$PROMETHEUS_DS', 30 | format='percentunit', 31 | decimals=0, 32 | gaugeShow=true, 33 | gaugeMaxValue=1, 34 | timeFrom='', 35 | valueMaps=[ 36 | { 37 | 'op': '=', 38 | 'text': '0', 39 | 'value': 'null' 40 | } 41 | ], 42 | thresholds='0.80,0.90', 43 | ) 44 | .addTarget( 45 | prometheus.target( 46 | 'avg(irate(org_apache_cassandra_metrics_cache_count{scope="%(scope)s", name="Hits"}[1m]) / on (instance) irate(org_apache_cassandra_metrics_cache_count{scope="%(scope)s", name="Requests"}[1m]))' % {scope:scope} 47 | ) 48 | ); 49 | 50 | local cacheGraphPanel(name, scope) = 51 | graphPanel.new(name, 52 | datasource='$PROMETHEUS_DS', 53 | format='percentunit', 54 | decimals=1, 55 | ) 56 | .addTarget( 57 | prometheus.target( 58 | 'org_apache_cassandra_metrics_cache_oneminuterate{scope="%(scope)s", name="Hits"} / on (instance) org_apache_cassandra_metrics_cache_oneminuterate{scope="%(scope)s", name="Requests"} ' % {scope:scope}, 59 | legendFormat='{{instance}}' 60 | ) 61 | ); 62 | 63 | 64 | dashboard.new( 65 | 'Caches', 66 | schemaVersion=14, 67 | refresh='1m', 68 | time_from='now-15m', 69 | editable=true, 70 | tags=['Cassandra', 'Resources', 'Cache'], 71 | ) 72 | .addTemplate( 73 | grafana.template.datasource( 74 | 'PROMETHEUS_DS', 75 | 'prometheus', 76 | 'Prometheus', 77 | hide='label', 78 | ) 79 | ) 80 | .addTemplate( 81 | template.new( 82 | 'node', 83 | '$PROMETHEUS_DS', 84 | 'label_values(node_cpu_seconds_total{cpu="0", mode="user", environment="$environment", cluster="$cluster", datacenter=~"$datacenter", rack=~"$rack"}, node)', 85 | label='Node', 86 | refresh='time', 87 | current='all', 88 | includeAll=true, 89 | multi=true, 90 | ) 91 | ) 92 | 93 | .addRow( 94 | row.new(title='Quick Stats') 95 | .addPanel(singleStatCachePanel('Key Cache Hit Rate', 'KeyCache'), smallGrid) 96 | .addPanel(singleStatCachePanel('Counter Cache Hit Rate', 'CounterCache'), smallGrid) 97 | .addPanel(singleStatCachePanel('Row Cache Hit Rate', 'RowCache'), smallGrid) 98 | .addPanel(singleStatCachePanel('Chunk Cache Hit Rate', 'ChunkCache'), smallGrid) 99 | ) 100 | .addRow( 101 | row.new(title='Key Cache') 102 | .addPanel(cacheGraphPanel('Key Cache', 'KeyCache')) 103 | 104 | ) 105 | .addRow( 106 | row.new(title='Counter Cache') 107 | .addPanel(cacheGraphPanel('Counter Cache', 'CounterCache')) 108 | ) 109 | .addRow( 110 | row.new(title='Row Cache') 111 | .addPanel(cacheGraphPanel('Row Cache', 'RowCache')) 112 | 113 | ) 114 | .addRow( 115 | row.new(title='Chunk Cache') 116 | .addPanel(cacheGraphPanel('Chunk Cache', 'ChunkCache')) 117 | ) 118 | -------------------------------------------------------------------------------- /dashboards/dashboards/jvm.jsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local dashboard = grafana.dashboard; 3 | local row = grafana.row; 4 | local graphPanel = grafana.graphPanel; 5 | local tablePanel = grafana.tablePanel; 6 | local singleStatPanel = grafana.singlestat; 7 | local textPanel = grafana.text; 8 | local prometheus = grafana.prometheus; 9 | local template = grafana.template; 10 | 11 | dashboard.new( 12 | 'JVM', 13 | schemaVersion=14, 14 | refresh='1m', 15 | time_from='now-15m', 16 | editable=true, 17 | tags=['Cassandra', 'Resources', 'Network', 'Disk'], 18 | ) 19 | .addTemplate( 20 | grafana.template.datasource( 21 | 'PROMETHEUS_DS', 22 | 'prometheus', 23 | 'Prometheus', 24 | hide='label', 25 | ) 26 | ) 27 | .addTemplate( 28 | template.new( 29 | 'node', 30 | '$PROMETHEUS_DS', 31 | 'label_values(node_cpu_seconds_total{cpu="0", mode="user", environment="$environment", cluster="$cluster", datacenter=~"$datacenter", rack=~"$rack"}, node)', 32 | label='Node', 33 | refresh='time', 34 | current='all', 35 | includeAll=true, 36 | multi=true, 37 | ) 38 | ) 39 | .addRow( 40 | row.new("Regions") 41 | .addPanel( 42 | graphPanel.new( 43 | "Eden Usage", 44 | description="Eden Usage", 45 | datasource='$PROMETHEUS_DS' 46 | ) 47 | .addTarget( 48 | prometheus.target( 49 | 'jvm_memory_pool_bytes_used{pool="Par Eden Space"} / 2^20', 50 | legendFormat="{{instance}}", 51 | ) 52 | ) 53 | ) 54 | .addPanel( 55 | graphPanel.new( 56 | "CMS Old Gen Usage", 57 | description="CMS Old Gen Usage", 58 | datasource='$PROMETHEUS_DS' 59 | ) 60 | .addTarget( 61 | prometheus.target( 62 | 'jvm_memory_pool_bytes_used{pool="CMS Old Gen"} / 2^20', 63 | legendFormat="{{instance}}", 64 | ) 65 | ) 66 | ) 67 | ) -------------------------------------------------------------------------------- /dashboards/dashboards/resources.jsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local dashboard = grafana.dashboard; 3 | local row = grafana.row; 4 | local graphPanel = grafana.graphPanel; 5 | local tablePanel = grafana.tablePanel; 6 | local singleStatPanel = grafana.singlestat; 7 | local textPanel = grafana.text; 8 | local prometheus = grafana.prometheus; 9 | local template = grafana.template; 10 | 11 | 12 | 13 | dashboard.new( 14 | 'System Resources', 15 | schemaVersion=14, 16 | refresh='1m', 17 | time_from='now-15m', 18 | editable=true, 19 | tags=['Cassandra', 'Resources', 'Network', 'Disk'], 20 | ) 21 | .addTemplate( 22 | grafana.template.datasource( 23 | 'PROMETHEUS_DS', 24 | 'prometheus', 25 | 'Prometheus', 26 | hide='label', 27 | ) 28 | ) 29 | .addTemplate( 30 | template.new( 31 | 'node', 32 | '$PROMETHEUS_DS', 33 | 'label_values(node_cpu_seconds_total{cpu="0", mode="user", environment="$environment", cluster="$cluster", datacenter=~"$datacenter", rack=~"$rack"}, node)', 34 | label='Node', 35 | refresh='time', 36 | current='all', 37 | includeAll=true, 38 | multi=true, 39 | ) 40 | ) 41 | .addRow( 42 | row.new("Disk") 43 | .addPanel( 44 | graphPanel.new( 45 | "Disk Throughput", 46 | description="Disk Throughput", 47 | datasource='$PROMETHEUS_DS' 48 | ) 49 | .addTarget( 50 | prometheus.target( 51 | 'sum(rate(node_disk_read_bytes_total[1m]) ) by (instance)', 52 | legendFormat="{{instance}}" 53 | ) 54 | ) 55 | ) 56 | ) 57 | .addRow( 58 | row.new("CPU") 59 | .addPanel( 60 | graphPanel.new( 61 | "CPU", 62 | description="CPU Usage", 63 | datasource='$PROMETHEUS_DS', 64 | ) 65 | .addTarget( 66 | prometheus.target( 67 | '100-(100 * avg(rate(node_cpu_seconds_total{mode="idle"}[30s])) by (instance))', 68 | legendFormat="{{instance}}" 69 | ) 70 | ) 71 | ) 72 | ) 73 | .addRow( 74 | row.new("Network") 75 | .addPanel( 76 | graphPanel.new( 77 | "Network Traffit - Transmit", 78 | description="Network Transmit Usage", 79 | datasource='$PROMETHEUS_DS', 80 | format="bytes", 81 | 82 | ) 83 | .addTarget( 84 | prometheus.target( 85 | 'sum(rate(node_network_transmit_bytes_total[1m])) by (instance)', 86 | legendFormat="{{instance}}" 87 | ) 88 | ) 89 | ) 90 | .addPanel( 91 | graphPanel.new( 92 | "Network Traffit - Receiving", 93 | description="Network Receiving Usage", 94 | datasource='$PROMETHEUS_DS', 95 | format="bytes", 96 | 97 | ) 98 | .addTarget( 99 | prometheus.target( 100 | 'sum(rate(node_network_receive_bytes_total[1m])) by (instance)', 101 | legendFormat="{{instance}}" 102 | ) 103 | ) 104 | ) 105 | ) -------------------------------------------------------------------------------- /dashboards/dashboards/stress.jsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local dashboard = grafana.dashboard; 3 | local row = grafana.row; 4 | local graphPanel = grafana.graphPanel; 5 | local tablePanel = grafana.tablePanel; 6 | local singleStatPanel = grafana.singlestat; 7 | local textPanel = grafana.text; 8 | local prometheus = grafana.prometheus; 9 | local template = grafana.template; 10 | 11 | 12 | /* 13 | # HELP selects Generated from Dropwizard metric import (metric=selects, type=com.codahale.metrics.Timer) 14 | # TYPE selects summary 15 | selects{quantile="0.5",} 0.002851058 16 | selects{quantile="0.75",} 0.00331815 17 | selects{quantile="0.95",} 0.004561816000000001 18 | selects{quantile="0.98",} 0.004561816000000001 19 | selects{quantile="0.99",} 0.004561816000000001 20 | selects{quantile="0.999",} 0.004561816000000001 21 | selects_count 9.0 22 | # HELP mutations Generated from Dropwizard metric import (metric=mutations, type=com.codahale.metrics.Timer) 23 | # TYPE mutations summary 24 | mutations{quantile="0.5",} 0.003054039 25 | mutations{quantile="0.75",} 0.0034644000000000003 26 | mutations{quantile="0.95",} 0.011488619 27 | mutations{quantile="0.98",} 0.011488619 28 | mutations{quantile="0.99",} 0.011488619 29 | mutations{quantile="0.999",} 0.011488619 30 | mutations_count 13.0 31 | # HELP errors_total Generated from Dropwizard metric import (metric=errors, type=com.codahale.metrics.Meter) 32 | # TYPE errors_total counter 33 | errors_total 0.0 34 | # HELP populateMutations Generated from Dropwizard metric import (metric=populateMutations, type=com.codahale.metrics.Timer) 35 | # TYPE populateMutations summary 36 | populateMutations{quantile="0.5",} 0.0 37 | populateMutations{quantile="0.75",} 0.0 38 | populateMutations{quantile="0.95",} 0.0 39 | populateMutations{quantile="0.98",} 0.0 40 | populateMutations{quantile="0.99",} 0.0 41 | populateMutations{quantile="0.999",} 0.0 42 | populateMutations_count 0.0 43 | */ 44 | 45 | dashboard.new( 46 | 'tlp-stress', 47 | schemaVersion=14, 48 | refresh='5s', 49 | time_from='now-5m', 50 | editable=true, 51 | tags=['Cassandra', 'Overview', 'Stress'] 52 | ) 53 | .addTemplate( 54 | grafana.template.datasource( 55 | 'PROMETHEUS_DS', 56 | 'prometheus', 57 | 'Prometheus', 58 | hide='label', 59 | ) 60 | ) 61 | .addRow( 62 | row.new(title="Stress Overview") 63 | .addPanel( 64 | singleStatPanel.new( 65 | "Aggregate Writes / Second", 66 | description="Aggregate Writes / Second", 67 | postfix=" writes/s", 68 | sparklineShow=true, 69 | timeFrom='30s', 70 | datasource='$PROMETHEUS_DS', 71 | valueName="current", 72 | ) 73 | .addTarget( 74 | prometheus.target( 75 | 'sum(irate(mutations_count{job="stress"}[15s]))' 76 | ) 77 | ) 78 | ) 79 | .addPanel( 80 | singleStatPanel.new( 81 | "Aggregate Reads / Second", 82 | description="Aggregate Reads / Second", 83 | postfix=" reads/s", 84 | sparklineShow=true, 85 | timeFrom='30s', 86 | datasource='$PROMETHEUS_DS', 87 | valueName="current", 88 | ) 89 | .addTarget( 90 | prometheus.target( 91 | 'sum(irate(selects_count{job="stress"}[15s]))' 92 | ) 93 | ) 94 | ) 95 | .addPanel( 96 | singleStatPanel.new( 97 | "Aggregate Errors / Second", 98 | description="Aggregate Errors / Second", 99 | postfix=" errors/s", 100 | sparklineShow=true, 101 | timeFrom='30s', 102 | datasource='$PROMETHEUS_DS', 103 | valueName="current", 104 | ) 105 | .addTarget( 106 | prometheus.target( 107 | 'sum(irate(errors_total{job="stress"}[15s]))' 108 | ) 109 | ) 110 | ) 111 | ) 112 | .addRow( 113 | row.new(title="Latency") 114 | .addPanel( 115 | graphPanel.new( 116 | 'Write Latency (p99)', 117 | description='p99 Write Latency (ms)', 118 | format='short', 119 | datasource='$PROMETHEUS_DS', 120 | transparent=true, 121 | fill=0, 122 | legend_show=true, 123 | legend_values=true, 124 | legend_current=true, 125 | legend_alignAsTable=true, 126 | legend_sort='current', 127 | legend_sortDesc=true, 128 | shared_tooltip=false, 129 | decimals=2, 130 | min=0, 131 | ) 132 | .addTarget( 133 | prometheus.target('mutations{quantile="0.99", job="stress"} * 1000', 134 | legendFormat="{{instance}}" 135 | ) 136 | ) 137 | ) 138 | .addPanel( 139 | graphPanel.new( 140 | 'Read Latency (p99)', 141 | description='p99 Read Latency (ms)', 142 | format='short', 143 | datasource='$PROMETHEUS_DS', 144 | transparent=true, 145 | fill=0, 146 | legend_show=true, 147 | legend_values=true, 148 | legend_current=true, 149 | legend_alignAsTable=true, 150 | legend_sort='current', 151 | legend_sortDesc=true, 152 | shared_tooltip=false, 153 | decimals=2, 154 | min=0, 155 | ) 156 | .addTarget( 157 | prometheus.target('selects{quantile="0.99", job="stress"} * 1000', 158 | legendFormat="{{instance}}" 159 | ) 160 | ) 161 | ) 162 | 163 | ) -------------------------------------------------------------------------------- /dashboards/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.0" 2 | 3 | services: 4 | grafonnet: 5 | image: "thelastpickle/grafonnet:1.0" 6 | volumes: 7 | - "./dashboards/:/input:ro" 8 | - "${BUILD_DIR}:/output" 9 | 10 | prometheus: 11 | image: prom/prometheus:v2.11.1 12 | restart: unless-stopped 13 | ports: 14 | - "9090:9090" 15 | networks: 16 | - easy_cass_lab_net 17 | volumes: 18 | - "./monitoring-environment/prometheus.yml:/etc/prometheus/prometheus.yml:ro" 19 | 20 | node_exporter: 21 | image: prom/node-exporter:v0.18.1 22 | ports: 23 | - "9100:9100" 24 | volumes: 25 | - "/proc:/host/proc:ro" 26 | - "/sys:/host/sys:ro" 27 | - "/:/rootfs:ro" 28 | command: 29 | - "--path.procfs=/host/proc" 30 | - "--path.rootfs=/rootfs" 31 | - "--path.sysfs=/host/sys" 32 | - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)" 33 | - "--collector.meminfo" 34 | depends_on: 35 | - cassandra 36 | - cassandra2 37 | networks: 38 | - easy_cass_lab_net 39 | 40 | node_exporter2: 41 | image: prom/node-exporter:v0.18.1 42 | ports: 43 | - "9100" 44 | volumes: 45 | - "/proc:/host/proc:ro" 46 | - "/sys:/host/sys:ro" 47 | - "/:/rootfs:ro" 48 | command: 49 | - "--path.procfs=/host/proc" 50 | - "--path.rootfs=/rootfs" 51 | - "--path.sysfs=/host/sys" 52 | - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)" 53 | - "--collector.meminfo" 54 | depends_on: 55 | - cassandra 56 | - cassandra2 57 | networks: 58 | - easy_cass_lab_net 59 | 60 | grafana: 61 | image: grafana/grafana 62 | restart: unless-stopped 63 | ports: 64 | - "3000:3000" 65 | volumes: 66 | - "./monitoring-environment/grafana.ini:/etc/grafana/grafana.ini:ro" 67 | - "./monitoring-environment/provisioning/:/etc/grafana/provisioning/" 68 | - "${DASHBOARD_DIR}:/var/lib/grafana/dashboards:ro" 69 | networks: 70 | - easy_cass_lab_net 71 | environment: 72 | # Attempts to get a nice nodes up/total panel... 73 | - "GF_PANELS_DISABLE_SANITIZE_HTML=true" 74 | 75 | cassandra: 76 | image: cassandra:3.11.4 77 | ports: 78 | - "9501:9501" 79 | - "9042:9042" 80 | - "7000" 81 | volumes: 82 | - "../src/main/resources/com/rustyrazorblade/easycasslab/commands/origin/provisioning/cassandra/jmx_prometheus_javaagent-0.12.0.jar.txt:/usr/share/jmx_prometheus_javaagent-0.12.0.jar:ro" 83 | - "../src/main/resources/com/rustyrazorblade/easycasslab/commands/origin/provisioning/cassandra/config.yaml:/usr/share/config.yaml:ro" 84 | environment: 85 | JVM_EXTRA_OPTS: '-javaagent:/usr/share/jmx_prometheus_javaagent-0.12.0.jar=9501:/usr/share/config.yaml -Dcassandra.consistent.rangemovement=false -Dcassandra.ring_delay_ms=100' 86 | CASSANDRA_NUM_TOKENS: 1 87 | CASSANDRA_SEEDS: "cassandra, cassandra2" 88 | networks: 89 | - easy_cass_lab_net 90 | 91 | cassandra2: 92 | image: cassandra:3.11.4 93 | command: /bin/bash -c "echo 'Waiting for seed node' && sleep 30 && /docker-entrypoint.sh cassandra -f" 94 | ports: 95 | - "9501" 96 | - "9042" 97 | - "7000" 98 | volumes: 99 | - "../src/main/resources/com/rustyrazorblade/easycasslab/commands/origin/provisioning/cassandra/jmx_prometheus_javaagent-0.12.0.jar.txt:/usr/share/jmx_prometheus_javaagent-0.12.0.jar:ro" 100 | - "../src/main/resources/com/rustyrazorblade/easycasslab/commands/origin/provisioning/cassandra/config.yaml:/usr/share/config.yaml:ro" 101 | networks: 102 | - easy_cass_lab_net 103 | environment: 104 | JVM_EXTRA_OPTS: '-javaagent:/usr/share/jmx_prometheus_javaagent-0.12.0.jar=9501:/usr/share/config.yaml -Dcassandra.consistent.rangemovement=false -Dcassandra.ring_delay_ms=100' 105 | CASSANDRA_NUM_TOKENS: 1 106 | CASSANDRA_SEEDS: "cassandra" 107 | 108 | stress: 109 | image: rustyrazorblade/easy-cass-stress-stress:latest 110 | ports: 111 | - "9500:9500" 112 | networks: 113 | - easy_cass_lab_net 114 | environment: 115 | - "TLP_STRESS_CASSANDRA_HOST=cassandra" 116 | command: "run KeyValue --rate 100 -d 1d -r .8" 117 | depends_on: 118 | - cassandra 119 | - cassandra2 120 | 121 | stress2: 122 | image: rustyrazorblade/easy-cass-stress:latest 123 | ports: 124 | - "9500" 125 | networks: 126 | - easy_cass_lab_net 127 | environment: 128 | - "EASY_CASS_STRESS_CASSANDRA_HOST=cassandra" 129 | command: "run BasicTimeSeries --rate 100 -d 1d -r .9" 130 | depends_on: 131 | - cassandra 132 | - cassandra2 133 | - stress 134 | 135 | start_dependencies: 136 | image: dadarek/wait-for-dependencies 137 | depends_on: 138 | - cassandra 139 | - cassandra2 140 | command: cassandra:9042 cassandra2:9042 141 | networks: 142 | - easy_cass_lab_net 143 | 144 | networks: 145 | easy_cass_lab_net: 146 | driver: bridge -------------------------------------------------------------------------------- /dashboards/monitoring-environment/grafana.ini: -------------------------------------------------------------------------------- 1 | 2 | [auth.anonymous] 3 | enabled = true 4 | 5 | # Organization name that should be used for unauthenticated users 6 | org_name = Main Org. 7 | 8 | # Role for unauthenticated users, other valid values are `Editor` and `Admin` 9 | org_role = Admin 10 | -------------------------------------------------------------------------------- /dashboards/monitoring-environment/prometheus.yml: -------------------------------------------------------------------------------- 1 | # used for docker testing only 2 | global: 3 | scrape_interval: 15s 4 | 5 | scrape_configs: 6 | - job_name: 'prometheus' 7 | static_configs: 8 | - targets: ['localhost:9090'] 9 | 10 | - job_name: 'cassandra' 11 | scrape_interval: 5s 12 | static_configs: 13 | - targets: ['cassandra:9501'] 14 | labels: 15 | environment: PickleLabs 16 | cluster: FunkyTesting 17 | datacenter: AwesomeDC 18 | rack: RadRack 19 | node: Cassandra_01 20 | 21 | - targets: ['cassandra2:9501'] 22 | labels: 23 | environment: PickleLabs 24 | cluster: FunkyTesting 25 | datacenter: AwesomeDC 26 | rack: RadRack2 27 | node: Cassandra_02 28 | 29 | - job_name: 'node_exporter' 30 | scrape_interval: 5s 31 | static_configs: 32 | - targets: ['node_exporter:9100'] 33 | labels: 34 | environment: PickleLabs 35 | cluster: FunkyTesting 36 | datacenter: AwesomeDC 37 | rack: RadRack 38 | node: Cassandra_01 39 | - targets: ['node_exporter2:9100'] 40 | labels: 41 | environment: PickleLabs 42 | cluster: FunkyTesting 43 | datacenter: AwesomeDC 44 | rack: RadRack 45 | node: Cassandra_02 46 | 47 | - job_name: 'stress' 48 | scrape_interval: 5s 49 | static_configs: 50 | - targets: ['stress:9500', 'stress2:9500'] 51 | labels: 52 | environment: PickleLabs 53 | cluster: FunkyTesting 54 | datacenter: AwesomeDC 55 | rack: RadRack 56 | node: stress01 -------------------------------------------------------------------------------- /dashboards/monitoring-environment/provisioning/dashboards/cassandra.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: "cassandra" 5 | orgId: 1 6 | folder: "cassandra" 7 | type: file 8 | editable: true 9 | updateIntervalInSeconds: 5 10 | options: 11 | path: "/var/lib/grafana/dashboards" 12 | -------------------------------------------------------------------------------- /dashboards/monitoring-environment/provisioning/datasources/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | url: http://prometheus:9090 6 | type: prometheus 7 | access: proxy 8 | -------------------------------------------------------------------------------- /docker-grafonnet/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'com.bmuschko.docker-remote-api' 3 | } 4 | 5 | version "4.0" 6 | 7 | import com.bmuschko.gradle.docker.tasks.image.* 8 | 9 | docker { 10 | registryCredentials { 11 | url = 'https://index.docker.io/v1/' 12 | username = System.getenv("DOCKER_USERNAME") 13 | password = System.getenv("DOCKER_PASSWORD") 14 | email = System.getenv("DOCKER_EMAIL") 15 | } 16 | } 17 | 18 | ext { 19 | container = "thelastpickle/grafonnet" 20 | } 21 | 22 | task buildDocker(type: DockerBuildImage) { 23 | images = ["$container:latest".toString(), "$container:$version".toString()] 24 | inputDir = file("docker") 25 | } 26 | 27 | task push(type: DockerPushImage) { 28 | dependsOn buildDocker 29 | 30 | // imageName container 31 | } 32 | -------------------------------------------------------------------------------- /docker-grafonnet/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sparkprime/jsonnet 2 | 3 | RUN apk update && apk add git && git clone https://github.com/grafana/grafonnet-lib.git && \ 4 | cd grafonnet-lib && git checkout f3ee1d810858cf556d25f045b53cb0f1fd10b94e 5 | 6 | COPY entrypoint.sh /entrypoint.sh 7 | RUN chmod 755 /entrypoint.sh 8 | 9 | ENTRYPOINT /entrypoint.sh 10 | -------------------------------------------------------------------------------- /docker-grafonnet/docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | echo "Generating dashboards" 5 | 6 | for fp in $(ls /input); do 7 | echo "Generating $fp" 8 | 9 | /usr/local/bin/jsonnet -J /grafonnet-lib/ /input/$fp -o /output/${fp%net} 10 | done -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=12 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustyrazorblade/easy-cass-lab/9948adc90e5fd43b681aaef02b03796955069459/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 1>&2 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 48 | echo. 1>&2 49 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 50 | echo location of your Java installation. 1>&2 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 1>&2 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 62 | echo. 1>&2 63 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 64 | echo location of your Java installation. 1>&2 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /manual/build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | mavenCentral() 4 | } 5 | dependencies { 6 | // classpath 'org.asciidoctor:asciidoctor-gradle-plugin:1.5.9.2' 7 | } 8 | } 9 | 10 | 11 | plugins { 12 | id 'org.asciidoctor.jvm.convert' version '4.0.2' 13 | } 14 | 15 | asciidoctor { 16 | sourceDir = file("src") 17 | outputDir = "${rootProject.projectDir}/docs/" 18 | attributes "EASY_CASS_LAB_VERSION" : rootProject.version.toString() 19 | } 20 | -------------------------------------------------------------------------------- /manual/src/development.adoc: -------------------------------------------------------------------------------- 1 | = easy-cass-lab Development Guide 2 | Jon Haddad 3 | :toc: left 4 | :icons: font 5 | 6 | Hello there. If you're reading this, you've probably decided to contribute to easy-cass-lab or use the tools for your own work. Very cool. 7 | 8 | INFO:: These docs are a WIP and we are aware of a number of sections that are yet to be completed. 9 | 10 | == Overview 11 | 12 | easy-cass-lab is broken into several subprojects to make each compontent a bit more managable. Some of these are docker (which are prefixed with `docker-`), some are text generation (such as the manual you're reading now), and others are simply a bit of code that download an artifact. Each subproject is versioned and may potentially create an artifact which can be downloaded separately, such as the `dashboards` subproject. 13 | 14 | 15 | == Docker 16 | 17 | Each container is versioned and can be built locally using the following: 18 | 19 | [source,bash] 20 | ---- 21 | ./gradlew :PROJECT-NAME:buildDocker 22 | ---- 23 | 24 | where `PROJECT-NAME` is one of the subproject directories you see in the top level. 25 | 26 | === Setup 27 | 28 | We recommend updating your local Docker service to use 8GB of memory. This is necessary when running dashboard previews locally. The preview is configured to run multiple Cassandra containers at once. 29 | 30 | 31 | == Publishing 32 | 33 | . First check circle ci to ensure the build is clean and green. 34 | . Ensure the following are set: `BINTRAY_USER`, `BINTRAY_KEY`, `DOCKER_USERNAME`, `DOCKER_PASSWORD`, `DOCKER_EMAIL`. These will be removed 35 | . Perform the following: 36 | 37 | [source,bash] 38 | ---- 39 | ./gradlew :manual:asciidoctor :manual:publish 40 | 41 | # end of optional stuff 42 | 43 | ./gradlew buildAll uploadAll 44 | ---- 45 | 46 | . Bump the version in `build.gradle`. 47 | -------------------------------------------------------------------------------- /manual/src/includes/help.txt: -------------------------------------------------------------------------------- 1 | Usage: easy-cass-lab [options] [command] [command options] 2 | Options: 3 | --help, -h 4 | Shows this help. 5 | Default: false 6 | Commands: 7 | init Initialize this directory for easy-cass-lab 8 | Usage: init [options] Client, Ticket, Purpose 9 | Options: 10 | --ami 11 | AMI 12 | Default: ami-51537029 13 | --cassandra, -c 14 | Number of Cassandra instances 15 | Default: 3 16 | --instance 17 | Instance Type 18 | Default: c5d.2xlarge 19 | --monitoring, -m 20 | Enable monitoring (beta) 21 | Default: false 22 | --region 23 | Region 24 | Default: us-west-2 25 | --stress, -s 26 | Number of stress instances 27 | Default: 0 28 | --up 29 | Start instances automatically 30 | Default: false 31 | 32 | up Starts instances 33 | Usage: up [options] 34 | Options: 35 | --auto-approve, -a, --yes 36 | Auto approve changes 37 | Default: false 38 | 39 | start Start cassandra on all nodes via service command 40 | Usage: start [options] 41 | Options: 42 | --all, -a 43 | Start all services on all instances. This overrides all other 44 | options 45 | Default: false 46 | --monitoring, -m 47 | Start services on monitoring instances 48 | Default: false 49 | 50 | stop Stop cassandra on all nodes via service command 51 | Usage: stop [options] 52 | Options: 53 | --all, -a 54 | Start all services on all instances. This overrides all other 55 | options 56 | Default: false 57 | --monitoring, -m 58 | Start services on monitoring instances 59 | Default: false 60 | 61 | install Install Everything 62 | Usage: install 63 | 64 | down Shut down a cluster 65 | Usage: down [options] 66 | Options: 67 | --auto-approve, -a, --yes 68 | Auto approve changes 69 | Default: false 70 | 71 | build Create a custom named Cassandra build from a working directory. 72 | Usage: build [options] Path to build 73 | Options: 74 | -n 75 | Name of build 76 | 77 | ls List available builds 78 | Usage: ls 79 | 80 | use Use a Cassandra build 81 | Usage: use [options] 82 | Options: 83 | --config, -c 84 | Configuration settings to change in the cassandra.yaml file 85 | specified in the format key:value,... 86 | Default: [] 87 | 88 | clean null 89 | Usage: clean 90 | 91 | hosts null 92 | Usage: hosts 93 | 94 | 95 | Done 96 | -------------------------------------------------------------------------------- /packer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | ENV DEBIAN_FRONTEND=noninteractive 3 | RUN apt-get update && apt-get install -y ubuntu-server curl sudo 4 | RUN useradd -m cassandra 5 | 6 | RUN apt-get install -y openjdk-8-jdk openjdk-8-dbg openjdk-11-jdk openjdk-11-dbg openjdk-17-jdk openjdk-17-dbg 7 | 8 | RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - \ 9 | && apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ 10 | && apt-get update && apt-get install -y packer 11 | 12 | COPY cassandra/cassandra_versions.yaml /etc/cassandra_versions.yaml 13 | 14 | -------------------------------------------------------------------------------- /packer/README.md: -------------------------------------------------------------------------------- 1 | Build the docker container from scratch: 2 | 3 | ```shell 4 | docker build --no-cache -t ecl . 5 | ``` 6 | 7 | Build the docker container using cached layers: 8 | 9 | ```shell 10 | docker build -t ecl . 11 | ``` 12 | 13 | To start the test env: 14 | 15 | ```shell 16 | docker compose run --rm ecl 17 | ``` 18 | 19 | Test an install script: 20 | 21 | ```shell 22 | bash 23 | ``` 24 | -------------------------------------------------------------------------------- /packer/base/aliases.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export PATH="$PATH:/usr/share/bcc/tools:/usr/local/cassandra/current/bin:/usr/local/cassandra/current/tools/bin:/usr/local/async-profiler/bin:/usr/local/easy-cass-stress/bin" 4 | export ART="/mnt/cassandra/artifacts" 5 | export CASSANDRA_YAML="/usr/local/cassandra/current/conf/cassandra.yaml" 6 | # use ubuntu users's logs directory for nodetool commands 7 | export CASSANDRA_LOG_DIR=/home/ubuntu/logs 8 | 9 | alias nt="nodetool" 10 | alias v="ls -lahG" 11 | alias ts="tail -f -n1000 /mnt/cassandra/logs/system.log" 12 | alias as="sudo tail -f /var/log/axonops/axon-agent.log" 13 | alias l="cd /mnt/cassandra/logs" 14 | alias d="cd /mnt/cassandra/data" 15 | alias c="cqlsh $(hostname)" 16 | alias drop-cache="echo 3 | sudo tee /proc/sys/vm/drop_caches" 17 | alias heap-dump="sudo jmap -dump:live,format=b,file=/mnt/cassandra/artifacts/heapdump-$(date +%s).hprof $(cassandra-pid)" 18 | alias js="sudo jstack $(cassandra-pid) | tee /mnt/cassandra/artifacts/jstack-$(date +%s).txt" 19 | alias cdd="cd /mnt/cassandra/data" 20 | alias cdl="cd /mnt/cassandra/logs" 21 | alias cda="cd /mnt/cassandra/artifacts" 22 | alias cdc="cd /usr/local/cassandra/current/conf" 23 | 24 | -------------------------------------------------------------------------------- /packer/base/base.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | amazon = { 4 | version = ">= 1.2.8" 5 | source = "github.com/hashicorp/amazon" 6 | } 7 | } 8 | } 9 | 10 | variable "arch" { 11 | type = string 12 | default = "amd64" 13 | } 14 | 15 | variable "region" { 16 | type = string 17 | default = "us-west-2" 18 | } 19 | 20 | variable "release_version" { 21 | type = string 22 | default = "" 23 | } 24 | 25 | locals { 26 | timestamp = regex_replace(timestamp(), "[- TZ:]", "") 27 | version = var.release_version != "" ? var.release_version : local.timestamp 28 | # We need to use a Graviton instance type for arm 29 | instance_type = var.arch == "amd64" ? "c3.xlarge" : "c8g.2xlarge" 30 | } 31 | 32 | source "amazon-ebs" "ubuntu" { 33 | ami_name = "rustyrazorblade/images/easy-cass-lab-base-${var.arch}-${local.version}" 34 | instance_type = local.instance_type 35 | region = "${var.region}" 36 | source_ami_filter { 37 | filters = { 38 | name = "ubuntu/images/*ubuntu-jammy-22.04-${var.arch}-server-*.1" 39 | root-device-type = "ebs" 40 | virtualization-type = "hvm" 41 | } 42 | most_recent = true 43 | owners = ["099720109477"] 44 | } 45 | ssh_username = "ubuntu" 46 | launch_block_device_mappings { 47 | device_name = "/dev/sda1" 48 | volume_size = 16 49 | volume_type = "gp2" 50 | delete_on_termination = true 51 | } 52 | } 53 | 54 | build { 55 | name = "easy-cass-lab" 56 | sources = [ 57 | "source.amazon-ebs.ubuntu" 58 | ] 59 | 60 | provisioner "shell" { 61 | script = "install/prepare_instance.sh" 62 | } 63 | 64 | provisioner "shell" { 65 | inline = [ 66 | # bpftrace was removed b/c it breaks bcc tools, need to build latest from source 67 | "sudo wget https://github.com/mikefarah/yq/releases/download/v4.41.1/yq_linux_${var.arch} -O /usr/local/bin/yq", 68 | "sudo chmod +x /usr/local/bin/yq", 69 | ] 70 | } 71 | 72 | # install pyenv and python 73 | provisioner "shell" { 74 | script = "install/install_python.sh" 75 | } 76 | 77 | provisioner "shell" { 78 | script = "install/install_fio.sh" 79 | } 80 | 81 | # install async profiler 82 | provisioner "shell" { 83 | script = "install/install_async_profiler.sh" 84 | } 85 | 86 | 87 | provisioner "shell" { 88 | script = "install/install_bcc.sh" 89 | } 90 | 91 | # install OpenTelemetry Collector 92 | provisioner "shell" { 93 | script = "install/install_otel_collector.sh" 94 | } 95 | 96 | provisioner "shell" { 97 | inline = [ 98 | "sudo apt install openjdk-8-jdk openjdk-8-dbg openjdk-11-jdk openjdk-11-dbg openjdk-17-jdk openjdk-17-dbg -y", 99 | "sudo update-java-alternatives -s /usr/lib/jvm/java-1.11.0-openjdk-${var.arch}", 100 | "sudo sed -i '/hl jexec.*/d' /usr/lib/jvm/.java-1.8.0-openjdk-${var.arch}.jinfo" 101 | ] 102 | } 103 | 104 | # install my extra nice tools, exa, bat, fd, ripgrep 105 | # wrapper for aprof to output results to a folder content shared by nginx 106 | # open to what port? 107 | 108 | # plop a file in with all the aliases I like 109 | provisioner "file" { 110 | source = "aliases.sh" 111 | destination = "aliases.sh" 112 | } 113 | 114 | provisioner "shell" { 115 | inline = [ 116 | "sudo mv aliases.sh /etc/profile.d/aliases.sh" 117 | ] 118 | } 119 | 120 | provisioner "shell" { 121 | inline = [ 122 | "wget https://training.ragozin.info/sjk.jar", 123 | "sudo mv sjk.jar /usr/local/lib", 124 | "" 125 | ] 126 | } 127 | } 128 | 129 | -------------------------------------------------------------------------------- /packer/base/install/install_async_profiler.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Get the architecture using uname 4 | cpu_arch=$(uname -m) 5 | 6 | # Set ARCH based on the CPU architecture 7 | if [[ "$cpu_arch" == "x86_64" ]]; then 8 | ARCH="x64" 9 | elif [[ "$cpu_arch" == "aarch64" ]]; then 10 | ARCH="arm64" 11 | else 12 | echo "Unsupported architecture: $cpu_arch" 13 | exit 1 14 | fi 15 | 16 | echo "ARCH is set to: $ARCH" 17 | 18 | RELEASE_VERSION="4.0" 19 | RELEASE="async-profiler-${RELEASE_VERSION}-linux-${ARCH}" 20 | ARCHIVE="${RELEASE}.tar.gz" 21 | 22 | sudo sysctl kernel.perf_event_paranoid=1 23 | sudo sysctl kernel.kptr_restrict=0 24 | wget "https://github.com/async-profiler/async-profiler/releases/download/v${RELEASE_VERSION}/${ARCHIVE}" 25 | tar zxvf $ARCHIVE 26 | sudo mv $RELEASE /usr/local/async-profiler -------------------------------------------------------------------------------- /packer/base/install/install_bcc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BCC_VERSION=0.31.0 4 | 5 | sudo apt update 6 | sudo apt purge bpfcc-tools libbpfcc python3-bpfcc 7 | sudo apt install -y zip bison build-essential cmake flex git libedit-dev \ 8 | libllvm14 llvm-14-dev libclang-14-dev python3 zlib1g-dev libelf-dev libfl-dev python3-setuptools \ 9 | liblzma-dev libdebuginfod-dev arping netperf iperf 10 | 11 | wget https://github.com/iovisor/bcc/releases/download/v${BCC_VERSION}/bcc-src-with-submodule.tar.gz 12 | tar xf bcc-src-with-submodule.tar.gz 13 | 14 | 15 | # from https://github.com/iovisor/bcc/blob/master/INSTALL.md 16 | 17 | cd bcc/ 18 | sudo ln -s /usr/bin/python3 /usr/bin/python 19 | mkdir build 20 | cd build/ 21 | cmake .. 22 | make 23 | sudo make install 24 | cmake -DPYTHON_CMD=/usr/bin/python3 .. # build python3 binding 25 | pushd src/python/ 26 | make 27 | sudo make install 28 | popd 29 | 30 | # cleanup 31 | cd 32 | sudo rm -rf bcc bcc-src-with-submodule.tar.gz 33 | 34 | -------------------------------------------------------------------------------- /packer/base/install/install_fio.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | VERSION="3.37" 6 | 7 | mkdir fio 8 | cd fio 9 | wget "https://github.com/axboe/fio/archive/refs/tags/fio-${VERSION}.zip" 10 | unzip fio-*.zip 11 | 12 | ( # subshell 13 | cd fio-fio* 14 | ./configure 15 | make 16 | sudo make install 17 | ) 18 | 19 | rm -rf fio -------------------------------------------------------------------------------- /packer/base/install/install_otel_collector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # OpenTelemetry Collector version 4 | OTEL_VERSION="0.123.0" 5 | 6 | # Get the architecture using uname 7 | cpu_arch=$(uname -m) 8 | 9 | # Set ARCH based on the CPU architecture 10 | if [[ "$cpu_arch" == "x86_64" ]]; then 11 | ARCH="amd64" 12 | elif [[ "$cpu_arch" == "aarch64" ]]; then 13 | ARCH="arm64" 14 | else 15 | echo "Unsupported architecture: $cpu_arch" 16 | exit 1 17 | fi 18 | 19 | echo "Installing OpenTelemetry Collector v${OTEL_VERSION} for ${ARCH} architecture" 20 | 21 | # Install OpenTelemetry Collector 22 | sudo apt-get update 23 | sudo apt-get -y install wget 24 | wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTEL_VERSION}/otelcol_${OTEL_VERSION}_linux_${ARCH}.deb 25 | sudo dpkg -i otelcol_${OTEL_VERSION}_linux_${ARCH}.deb -------------------------------------------------------------------------------- /packer/base/install/install_python.sh: -------------------------------------------------------------------------------- 1 | sudo apt update -y 2 | sudo apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev 3 | curl https://pyenv.run | bash 4 | # add to ~/.bash_profile for use on instance 5 | echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile 6 | echo 'eval "$(pyenv init --path)"' >> ~/.bash_profile 7 | echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile 8 | 9 | { 10 | # shellcheck disable=SC2016 11 | echo 'export PATH="$HOME/.pyenv/bin:$PATH"' 12 | # shellcheck disable=SC2016 13 | echo 'eval "$(pyenv init --path)"' 14 | # shellcheck disable=SC2016 15 | echo 'eval "$(pyenv virtualenv-init -)"' 16 | } >> ~/.bash_profile 17 | 18 | 19 | # now load it in for Packer build 20 | export PATH="$HOME/.pyenv/bin:$PATH" 21 | eval $(pyenv init --path) 22 | eval $(pyenv virtualenv-init -) 23 | # now install python 24 | pyenv install 2.7.18 25 | pyenv install 3.10.6 26 | 27 | # yeah.. this next part a little gross. 28 | # I assume we can keep the pip3 shim around 29 | /home/ubuntu/.pyenv/versions/3.10.6/bin/pip3 install iostat-tool 30 | # shellcheck disable=SC2046 31 | sudo ln -s $(find /home/ubuntu/.pyenv/ -name 'iostat-cli') /usr/local/bin/iostat-cli -------------------------------------------------------------------------------- /packer/base/install/prepare_instance.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # needed early on before we do anything with /mnt 4 | if mountpoint -q /mnt; then 5 | echo "/mnt is a mount point." 6 | sudo umount -l -f /mnt 7 | else 8 | echo "/mnt is not a mount point." 9 | fi 10 | 11 | sudo apt update 12 | sudo apt upgrade -y 13 | sudo apt update 14 | 15 | sudo apt install -y wget sysstat unzip ripgrep ant ant-optional tree zfsutils-linux nicstat 16 | 17 | cpu_arch=$(uname -m) 18 | # Set ARCH based on the CPU architecture 19 | if [[ "$cpu_arch" == "x86_64" ]]; then 20 | sudo apt install -y cpuid 21 | elif [[ "$cpu_arch" == "aarch64" ]]; then 22 | echo "No additional packages needed for ARM64" 23 | else 24 | echo "Unsupported architecture: $cpu_arch" 25 | exit 1 26 | fi 27 | 28 | 29 | -------------------------------------------------------------------------------- /packer/cassandra/axonops-sudoers: -------------------------------------------------------------------------------- 1 | axonops ALL=NOPASSWD: /sbin/service cassandra *, /usr/bin/systemctl * cassandra* -------------------------------------------------------------------------------- /packer/cassandra/bin/cass-yaml-search: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | YAML="${CASSANDRA_YAML:-/etc/cassandra.yaml}" 4 | SEARCH=$1 5 | 6 | 7 | rg -N -v '^\s*#[^:]*$' $CASSANDRA_YAML | \ 8 | rg -N -A5 -B5 -i --context-separator="====================================" -e "$SEARCH" | \ 9 | awk NF 10 | -------------------------------------------------------------------------------- /packer/cassandra/bin/cassandra-pid: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | systemctl show --property MainPID --value cassandra 4 | -------------------------------------------------------------------------------- /packer/cassandra/bin/flamegraph: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | date=$(date +"%Y-%m-%d_%H-%M-%S") 4 | 5 | OUTFILE=/mnt/cassandra/artifacts/flame-$(hostname)-${date}.html 6 | 7 | sudo /usr/local/async-profiler/bin/asprof -f $OUTFILE $@ $(cassandra-pid) 8 | 9 | echo $OUTFILE 10 | -------------------------------------------------------------------------------- /packer/cassandra/bin/iostat-plot: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # requires pip install iostat-tool to be installed 4 | 5 | DEFAULT_NAME="iostat-"$(date +%s) 6 | OUTPUT_BASE=/mnt/cassandra/artifacts/${1:-$DEFAULT_NAME} 7 | RAW=${OUTPUT_BASE}.output 8 | PLOT=${OUTPUT_BASE}.png 9 | ITERATIONS=60 10 | 11 | DISK=$(lsblk | grep '/mnt/cassandra' | awk '{print $1}') 12 | 13 | iostat -ymxt 1 $ITERATIONS | tee $RAW 14 | 15 | iostat-cli --data $RAW --disk $DISK --fig-output $PLOT plot -------------------------------------------------------------------------------- /packer/cassandra/bin/patch-config: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # patches the current cassandra.yaml using the original + modifications 4 | # accepts two arguments: the patch file and the version number, assumed to be current if omitted 5 | # we want the ability to be able to patch stuff before we apply it as the current version. 6 | 7 | # shellcheck disable=SC2034 8 | export PATCH=$1 9 | VERSION=${2:-current} 10 | export DEST=/usr/local/cassandra/$VERSION/conf/cassandra.yaml 11 | 12 | yq '. *= load(env(PATCH))' /usr/local/cassandra/$VERSION/conf.orig/cassandra.yaml > /tmp/cassandra.yaml 13 | sudo mv -f /tmp/cassandra.yaml $DEST 14 | sudo chown cassandra:cassandra $DEST -------------------------------------------------------------------------------- /packer/cassandra/bin/restart-cassandra-and-wait: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Maybe restarting Cassandra..." 4 | 5 | nodetool drain || true 6 | sudo systemctl restart cassandra 7 | 8 | # Get the start time in seconds 9 | start_time=$(date +%s) 10 | 11 | echo "Waiting for Cassandra to be available" 12 | # Loop until nodetool status exits with 0 or 60 seconds have gone by 13 | while true; do 14 | # Run the nodetool status command 15 | /usr/local/cassandra/current/bin/nodetool status > /dev/null 2>&1 16 | status=$? 17 | 18 | # If the command was successful, exit the loop 19 | if [ $status -eq 0 ]; then 20 | echo "nodetool status was successful." 21 | break 22 | fi 23 | 24 | # Calculate the elapsed time 25 | current_time=$(date +%s) 26 | elapsed=$((current_time - start_time)) 27 | 28 | # Check if 2 minutes seconds have passed 29 | if [ $elapsed -ge 240 ]; then 30 | echo "Timeout: nodetool status did not exit with 0 within 60 seconds." 31 | break 32 | fi 33 | 34 | # Optional: sleep for a bit before retrying to reduce load 35 | sleep 2 36 | done 37 | 38 | echo "Nodetool reporting node as up. Waiting on port 9042." 39 | 40 | while ! ss -tulwn | grep ':9042' ; do sleep 1; done 41 | echo "Port 9042 available" 42 | -------------------------------------------------------------------------------- /packer/cassandra/bin/set-java-version: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to set Java version for a Cassandra installation 4 | # Usage: set-java-version JAVA_VERSION [CASSANDRA_VERSION] 5 | # If CASSANDRA_VERSION is not provided, it will be derived from the current Cassandra symlink 6 | 7 | if [ $# -lt 1 ]; then 8 | echo "Usage: $0 JAVA_VERSION [CASSANDRA_VERSION]" 9 | echo "Example: $0 11 4.0" 10 | exit 1 11 | fi 12 | 13 | export JAVA_VERSION=$1 14 | export CASSANDRA_VERSION=$2 15 | 16 | # If CASSANDRA_VERSION is not provided, derive it from the current symlink 17 | if [ -z "$CASSANDRA_VERSION" ]; then 18 | if [ -L "/usr/local/cassandra/current" ]; then 19 | CASSANDRA_VERSION=$(basename $(readlink /usr/local/cassandra/current)) 20 | echo "Detected Cassandra version: $CASSANDRA_VERSION" 21 | else 22 | echo "Error: /usr/local/cassandra/current is not a symlink and no CASSANDRA_VERSION provided" 23 | exit 1 24 | fi 25 | fi 26 | 27 | # Update the Java version in cassandra_versions.yaml for the specified Cassandra version 28 | if yq -i e '.[] |= select(.version == env(CASSANDRA_VERSION)) |= .java = env(JAVA_VERSION)' /etc/cassandra_versions.yaml; then 29 | echo "Updated Java version to $JAVA_VERSION for Cassandra $CASSANDRA_VERSION in /etc/cassandra_versions.yaml" 30 | else 31 | echo "Failed to update Java version in /etc/cassandra_versions.yaml" 32 | exit 1 33 | fi 34 | 35 | echo "Java version successfully updated" -------------------------------------------------------------------------------- /packer/cassandra/bin/setup-axonops: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # validate inputs 4 | ORG=$1 5 | KEY=$2 6 | if [ -z "$ORG" ]; then 7 | echo "no org given" 8 | echo "setup-axonops " 9 | exit 1 10 | fi 11 | 12 | if [ -z "$KEY" ]; then 13 | echo "no api key key given" 14 | echo "setup-axonops " 15 | exit 1 16 | fi 17 | 18 | # setup axon-agent config 19 | axonconfig=/etc/axonops/axon-agent.yml 20 | sudo yq -i '.axon-server.hosts = "agents.axonops.cloud"' $axonconfig 21 | sudo yq -i ".axon-agent.org = \"$ORG\"" $axonconfig 22 | sudo yq -i ".axon-agent.key = \"$KEY\"" $axonconfig 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /packer/cassandra/bin/sjk-gc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | java -jar /usr/local/lib/sjk.jar gc -p $(cassandra-pid) $@ 4 | -------------------------------------------------------------------------------- /packer/cassandra/bin/sjk-mx: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo java -jar /usr/local/lib/sjk.jar mx -p $(cassandra-pid) $@ 2>/dev/null 4 | -------------------------------------------------------------------------------- /packer/cassandra/bin/use-cassandra: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CASSANDRA_VERSIONS="/etc/cassandra_versions.yaml" 4 | 5 | # set cassandra version to $1 6 | sudo ln -vfns /usr/local/cassandra/$1 /usr/local/cassandra/current 7 | sudo ln -vfns /usr/local/cassandra/$1/conf /etc/cassandra 8 | 9 | # hacky, not really what I want long term but whatever. 10 | sudo chown -R cassandra:cassandra /usr/local/cassandra 11 | 12 | # set the java version 13 | export VERSION=$1 14 | 15 | export JAVA_VERSION=$(yq e '.[] | select(.version == env(VERSION)).java' $CASSANDRA_VERSIONS) 16 | echo "Using java version from cassandra_versions.yaml: $JAVA_VERSION" 17 | 18 | export PY_VERSION=$(yq e '.[] | select(.version == env(VERSION)).python' $CASSANDRA_VERSIONS) 19 | 20 | if [ -z "$JAVA_VERSION" ]; then 21 | echo "No java version found for $VERSION" 22 | exit 1 23 | fi 24 | 25 | if [ -z "$PY_VERSION" ]; then 26 | echo "No python version found for $VERSION" 27 | exit 1 28 | fi 29 | 30 | # Get the architecture using uname 31 | cpu_arch=$(uname -m) 32 | 33 | # Set ARCH based on the CPU architecture 34 | if [[ "$cpu_arch" == "x86_64" ]]; then 35 | ARCH="amd64" 36 | elif [[ "$cpu_arch" == "aarch64" ]]; then 37 | ARCH="arm64" 38 | else 39 | echo "Unsupported architecture: $cpu_arch" 40 | exit 1 41 | fi 42 | 43 | # if java version = 8 then run update-java-alternatives -s java-1.8.0-openjdk-amd64 44 | if [ "$JAVA_VERSION" = "8" ]; then 45 | sudo update-java-alternatives -s java-1.8.0-openjdk-$ARCH 46 | # if java version = 11 then run update-java-alternatives -s java-1.11.0-openjdk-amd64 47 | elif [ "$JAVA_VERSION" = "11" ]; then 48 | sudo update-java-alternatives -s java-1.11.0-openjdk-$ARCH 49 | elif [ "$JAVA_VERSION" = "17" ]; then 50 | sudo update-java-alternatives -s java-1.17.0-openjdk-$ARCH 51 | else 52 | echo "Unknown java version $JAVA_VERSION" 53 | exit 1 54 | fi 55 | 56 | # set python version 57 | # NOTE: a bit hacky but we have to switch back to ubuntu user for this 58 | # since its expected that this script run as root by pyenv doesn't 59 | sudo -u ubuntu bash << EOF 60 | source ~/.bash_profile 61 | pyenv global $PY_VERSION 62 | EOF 63 | -------------------------------------------------------------------------------- /packer/cassandra/bin/wait-for-up-normal: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | CPID=$(cassandra-pid) 5 | 6 | if [[ $CPID -eq 0 ]]; then 7 | echo "CASSANDRA NOT RUNNING: FAIL" 8 | exit 1 9 | fi 10 | 11 | echo "Waiting for JMX" 12 | while ! ss -tulwn | grep ':7199' ; do sleep 1; done 13 | 14 | while true; do 15 | 16 | # first check if it's up 17 | 18 | CPID=$(cassandra-pid) 19 | 20 | if [[ $CPID -eq 0 ]]; then 21 | echo "CASSANDRA HAS SHUT DOWN: FAIL" 22 | exit 1 23 | fi 24 | 25 | sjk-mx -b org.apache.cassandra.db:type=StorageService -mg -f OperationMode | grep NORMAL 26 | status=$? 27 | 28 | if [[ $status -eq 0 ]]; then 29 | echo "OK" 30 | exit 0 31 | fi 32 | 33 | sleep 1 34 | 35 | done 36 | 37 | 38 | -------------------------------------------------------------------------------- /packer/cassandra/cassandra.in.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | ##### Begin easy-cass-lab customizations #### 4 | 5 | ### This is automatically appended to the end of every cassandra.in.sh 6 | 7 | # Extract Cassandra version from jar filename 8 | ECL_CASSANDRA_JAR=$(find /usr/local/cassandra/current/lib -name "apache-cassandra-[0-9]*.jar" | head -n 1) 9 | if [ -n "$ECL_CASSANDRA_JAR" ]; then 10 | # Extract X.Y.Z from filename and then get X.Y 11 | ECL_CASSANDRA_VERSION=$(basename "$ECL_CASSANDRA_JAR" | sed -E 's/apache-cassandra-([0-9]+\.[0-9]+)\.[0-9]+\.jar/\1/') 12 | export ECL_CASSANDRA_VERSION 13 | else 14 | echo "ERROR: Could not determine Cassandra version" >&2 15 | exit 1 16 | fi 17 | 18 | # Extract Java version 19 | ECL_JAVA_VERSION_OUTPUT=$(java -version 2>&1 | head -n 1) 20 | if [ -n "$ECL_JAVA_VERSION_OUTPUT" ]; then 21 | # Extract version like "17" from the output string 22 | ECL_JAVA_VERSION=$(echo "$ECL_JAVA_VERSION_OUTPUT" | sed -E 's/.*version "([0-9]+)\..*".*/\1/') 23 | export ECL_JAVA_VERSION 24 | else 25 | echo "ERROR: Could not determine Java version" >&2 26 | exit 1 27 | fi 28 | 29 | # Set AXONOPS_AGENT based on Cassandra and Java versions 30 | AXONOPS_AGENT="" 31 | case "$ECL_CASSANDRA_VERSION" in 32 | "3.0") 33 | AXONOPS_AGENT="3.0-agent" 34 | ;; 35 | "3.11") 36 | AXONOPS_AGENT="3.11-agent" 37 | ;; 38 | "4.0") 39 | if [ "$ECL_JAVA_VERSION" = "8" ] || [ "$ECL_JAVA_VERSION" = "1.8" ]; then 40 | AXONOPS_AGENT="4.0-agent-jdk8" 41 | else 42 | AXONOPS_AGENT="4.0-agent" 43 | fi 44 | ;; 45 | "4.1") 46 | if [ "$ECL_JAVA_VERSION" = "8" ] || [ "$ECL_JAVA_VERSION" = "1.8" ]; then 47 | AXONOPS_AGENT="4.1-agent-jdk8" 48 | else 49 | AXONOPS_AGENT="4.1-agent" 50 | fi 51 | ;; 52 | "5.0") 53 | if [ "$ECL_JAVA_VERSION" = "11" ]; then 54 | AXONOPS_AGENT="5.0-agent" 55 | elif [ "$ECL_JAVA_VERSION" = "17" ]; then 56 | ECL_AGENT_JAR="/usr/share/axonops/5.0-agent-jdk17/lib/axon-cassandra5.0-agent.jar" 57 | fi 58 | ;; 59 | "5.1"|"6.0") 60 | # No agent for these versions 61 | AXONOPS_AGENT="" 62 | ;; 63 | esac 64 | 65 | # Configure JVM_EXTRA_OPTS with agent if applicable 66 | if [ -n "$AXONOPS_AGENT" ]; then 67 | ECL_AGENT_JAR="/usr/share/axonops/${AXONOPS_AGENT}/lib/axon-cassandra${AXONOPS_AGENT}.jar" 68 | fi 69 | 70 | if [ -f "$ECL_AGENT_JAR" ]; then 71 | export JVM_EXTRA_OPTS="-javaagent:${ECL_AGENT_JAR}=/etc/axonops/axon-agent.yml" 72 | else 73 | echo "WARNING: AxonOps agent jar not found at $ECL_AGENT_JAR" >&2 74 | fi 75 | 76 | # Set log directory based on user 77 | if [ "$(whoami)" = "cassandra" ]; then 78 | CASSANDRA_LOG_DIR="/mnt/cassandra/logs" 79 | else 80 | CASSANDRA_LOG_DIR="$HOME/logs" 81 | fi 82 | 83 | mkdir -p "$CASSANDRA_LOG_DIR" 84 | 85 | # set logging depending on JVM version 86 | if [ "$ECL_JAVA_VERSION" = "17" ] || [ "$ECL_JAVA_VERSION" = "21" ]; then 87 | export JVM_OPTS="$JVM_OPTS -Xlog:gc=info:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level,tags:filecount=10,filesize=1M" 88 | fi 89 | -------------------------------------------------------------------------------- /packer/cassandra/cassandra.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | amazon = { 4 | version = ">= 1.2.8" 5 | source = "github.com/hashicorp/amazon" 6 | } 7 | } 8 | } 9 | 10 | variable "arch" { 11 | type = string 12 | default = "amd64" 13 | } 14 | 15 | variable "region" { 16 | type = string 17 | default = "us-west-2" 18 | } 19 | 20 | variable "release_version" { 21 | type = string 22 | default = "" 23 | } 24 | 25 | 26 | locals { 27 | timestamp = regex_replace(timestamp(), "[- TZ:]", "") 28 | base_version = var.release_version != "" ? var.release_version : "*" 29 | version = var.release_version != "" ? var.release_version : local.timestamp 30 | ami_groups = var.release_version != "" ? ["all"] : [] 31 | instance_type = var.arch == "amd64" ? "c3.xlarge" : "c8g.2xlarge" 32 | 33 | 34 | } 35 | 36 | source "amazon-ebs" "ubuntu" { 37 | ami_name = "rustyrazorblade/images/easy-cass-lab-cassandra-${var.arch}-${local.version}" 38 | ami_groups = local.ami_groups 39 | instance_type = local.instance_type 40 | region = "${var.region}" 41 | source_ami_filter { 42 | filters = { 43 | name = "rustyrazorblade/images/easy-cass-lab-base-${var.arch}-${local.base_version}" 44 | root-device-type = "ebs" 45 | virtualization-type = "hvm" 46 | } 47 | most_recent = true 48 | owners = ["self"] 49 | } 50 | ssh_username = "ubuntu" 51 | launch_block_device_mappings { 52 | device_name = "/dev/sda1" 53 | volume_size = 16 54 | volume_type = "gp2" 55 | delete_on_termination = true 56 | } 57 | } 58 | 59 | build { 60 | name = "easy-cass-lab" 61 | sources = [ 62 | "source.amazon-ebs.ubuntu" 63 | ] 64 | 65 | provisioner "shell" { 66 | script = "install/prepare_instance.sh" 67 | } 68 | 69 | # set up environment with PATH and aliases 70 | provisioner "file" { 71 | source = "environment" 72 | destination = "environment" 73 | } 74 | 75 | provisioner "file" { 76 | source = "config" 77 | destination = "config" 78 | } 79 | 80 | 81 | 82 | provisioner "shell" { 83 | inline = ["sudo mv environment /etc/environment"] 84 | } 85 | 86 | # catch all bin upload 87 | # just drop stuff you need in bin and the next 2 provisioners will take care of it 88 | provisioner "file" { 89 | source = "bin" 90 | destination = "/home/ubuntu/bin-cassandra" 91 | } 92 | provisioner "shell" { 93 | inline = [ 94 | "sudo mv -v bin-cassandra/* /usr/local/bin/", 95 | "sudo chmod +x /usr/local/bin/*", 96 | "ls /usr/local/bin", 97 | "rmdir bin-cassandra" 98 | ] 99 | } 100 | 101 | 102 | provisioner "shell" { 103 | script = "install/install_easy_cass_stress.sh" 104 | } 105 | 106 | # the cassandra_versions.yaml file is used to define all the version of cassandra we want 107 | # and it's matching java version. The use command will set the symlink of /usr/local/cassandra 108 | # to point to the version of cassandra we want to use, and set the java version using update-java-alternatives 109 | 110 | provisioner "file" { 111 | source = "cassandra_versions.yaml" 112 | destination = "cassandra_versions.yaml" 113 | } 114 | 115 | provisioner "shell" { 116 | script = "install/install_sidecar.sh" 117 | } 118 | 119 | provisioner "shell" { 120 | inline = ["sudo mv config/cassandra-sidecar.yaml /usr/local/cassandra-sidecar/conf/sidecar.yaml"] 121 | } 122 | 123 | provisioner "shell" { 124 | inline = [ 125 | "sudo mv cassandra_versions.yaml /etc/cassandra_versions.yaml" 126 | ] 127 | } 128 | 129 | # needs to be in place before install cassandra is run. At the end of the install, 130 | # the installer will append cassandra.in.sh to the end of the included cassandra.in.sh 131 | 132 | provisioner "file" { 133 | source = "cassandra.in.sh" 134 | destination = "/tmp/cassandra.in.sh" 135 | } 136 | 137 | provisioner "shell" { 138 | environment_vars = [ 139 | # we need this to be set because install_cassandra checks for it and exits if it's not there 140 | # this is so we can source the file and test the functions outside of packer 141 | "INSTALL_CASSANDRA=1", 142 | ] 143 | script = "install/install_cassandra.sh" 144 | } 145 | 146 | # instal axonops 147 | provisioner "shell" { 148 | script = "install/install_axon.sh" 149 | } 150 | 151 | provisioner "file" { 152 | source = "axonops-sudoers" 153 | destination = "axonops-sudoers" 154 | } 155 | 156 | provisioner "shell" { 157 | inline = [ 158 | "sudo chown root:root axonops-sudoers", 159 | "sudo mv axonops-sudoers /etc/sudoers.d/axonops", 160 | ] 161 | } 162 | 163 | provisioner "file" { 164 | source = "services" 165 | destination = "services" 166 | } 167 | 168 | provisioner "shell" { 169 | inline = [ 170 | "sudo mv services/* /etc/systemd/system/", 171 | "sudo systemctl enable cassandra.service", 172 | "sudo systemctl enable cassandra-sidecar.service" 173 | ] 174 | } 175 | 176 | provisioner "file" { 177 | source = "patch-jvm-options.py" 178 | destination = "patch-jvm-options.py" 179 | } 180 | 181 | provisioner "shell" { 182 | inline = [ 183 | "sudo mv patch-jvm-options.py /usr/local/bin/patch-jvm-options.py", 184 | "sudo chmod +x /usr/local/bin/patch-jvm-options.py" 185 | ] 186 | } 187 | 188 | provisioner "shell" { 189 | inline = [ 190 | "mkdir -p /home/ubuntu/.config/htop/" 191 | ] 192 | } 193 | 194 | provisioner "file" { 195 | source = "htoprc" 196 | destination = "/home/ubuntu/.config/htop/htoprc" 197 | } 198 | } 199 | 200 | 201 | -------------------------------------------------------------------------------- /packer/cassandra/cassandra_versions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | # the version uniquely identifes the cassandra version 4 | # and will be used as the name in /usr/local/cassandra- 5 | 6 | - version: "3.0" 7 | java: "8" 8 | python: "2.7.18" 9 | 10 | - version: "3.11" 11 | java: "8" 12 | python: "2.7.18" 13 | 14 | - version: "4.0" 15 | java: "11" 16 | python: "3.10.6" 17 | ant_flags: "-Duse.jdk11=true" 18 | 19 | - version: "4.1" 20 | java: "11" 21 | python: "3.10.6" 22 | ant_flags: "-Duse.jdk11=true" 23 | 24 | - version: "5.0" 25 | java: "11" 26 | python: "3.10.6" 27 | 28 | - version: "5.0-HEAD" 29 | url: https://github.com/apache/cassandra.git 30 | branch: cassandra-5.0 31 | java: "11" 32 | java_build: "11" 33 | python: "3.10.6" 34 | 35 | - version: "trunk" 36 | url: https://github.com/apache/cassandra.git 37 | branch: trunk 38 | java: "17" 39 | java_build: "11" 40 | python: "3.10.6" 41 | 42 | -------------------------------------------------------------------------------- /packer/cassandra/config/cassandra-sidecar.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # 20 | # Cassandra SideCar configuration file 21 | # 22 | cassandra_instances: 23 | - id: 1 24 | host: localhost 25 | port: 9042 26 | username: cassandra 27 | password: cassandra 28 | data_dirs: 29 | - /mnt/cassandra/data/ 30 | staging_dir: /mnt/cassandra/import 31 | jmx_host: 127.0.0.1 32 | jmx_port: 7199 33 | jmx_ssl_enabled: false 34 | # jmx_role: 35 | # jmx_role_password: 36 | 37 | sidecar: 38 | host: 0.0.0.0 39 | port: 9043 40 | request_idle_timeout_millis: 300000 # this field expects integer value 41 | request_timeout_millis: 300000 42 | tcp_keep_alive: false 43 | accept_backlog: 1024 44 | server_verticle_instances: 1 45 | throttle: 46 | stream_requests_per_sec: 5000 47 | delay_sec: 5 48 | timeout_sec: 10 49 | traffic_shaping: 50 | inbound_global_bandwidth_bps: 0 # 0 implies unthrottled, the inbound bandwidth in bytes per second 51 | outbound_global_bandwidth_bps: 0 # 0 implies unthrottled, the outbound bandwidth in bytes per second 52 | peak_outbound_global_bandwidth_bps: 419430400 # the peak outbound bandwidth in bytes per second. The default is 400 mebibytes per second 53 | max_delay_to_wait_millis: 15000 # 15 seconds 54 | check_interval_for_stats_millis: 1000 # 1 second 55 | inbound_global_file_bandwidth_bps: 0 # 0 implies unthrottled, the inbound bandwidth allocated for incoming files in bytes per second, upper-bounded by inbound_global_bandwidth_bps 56 | sstable_upload: 57 | concurrent_upload_limit: 80 58 | min_free_space_percent: 10 59 | # file_permissions: "rw-r--r--" # when not specified, the default file permissions are owner read & write, group & others read 60 | allowable_time_skew_in_minutes: 60 61 | sstable_import: 62 | poll_interval_millis: 100 63 | cache: 64 | expire_after_access_millis: 7200000 # 2 hours 65 | maximum_size: 10000 66 | worker_pools: 67 | service: 68 | name: "sidecar-worker-pool" 69 | size: 20 70 | max_execution_time_millis: 60000 # 60 seconds 71 | internal: 72 | name: "sidecar-internal-worker-pool" 73 | size: 20 74 | max_execution_time_millis: 900000 # 15 minutes 75 | jmx: 76 | max_retries: 3 77 | retry_delay_millis: 200 78 | schema: 79 | is_enabled: false 80 | keyspace: sidecar_internal 81 | replication_strategy: SimpleStrategy 82 | replication_factor: 1 83 | 84 | # 85 | # Enable SSL configuration (Disabled by default) 86 | # 87 | # ssl: 88 | # enabled: true 89 | # use_openssl: true 90 | # handshake_timeout_sec: 10 91 | # client_auth: NONE # valid options are NONE, REQUEST, REQUIRED 92 | # accepted_protocols: 93 | # - TLSv1.2 94 | # - TLSv1.3 95 | # cipher_suites: [] 96 | # keystore: 97 | # type: PKCS12 98 | # path: "path/to/keystore.p12" 99 | # password: password 100 | # check_interval_sec: 300 101 | # truststore: 102 | # path: "path/to/truststore.p12" 103 | # password: password 104 | 105 | driver_parameters: 106 | contact_points: 107 | - "127.0.0.1:9042" 108 | num_connections: 6 109 | # local_dc: datacenter1 110 | 111 | healthcheck: 112 | initial_delay_millis: 0 113 | poll_freq_millis: 30000 114 | 115 | metrics: 116 | registry_name: cassandra_sidecar 117 | vertx: 118 | enabled: true 119 | expose_via_jmx: false 120 | jmx_domain_name: sidecar.vertx.jmx_domain 121 | include: # empty include list means include all 122 | - type: "regex" # possible filter types are "regex" and "equals" 123 | value: "sidecar.*" 124 | - type: "regex" 125 | value: "vertx.*" 126 | exclude: # empty exclude list means exclude nothing 127 | # - type: "regex" # possible filter types are "regex" and "equals" 128 | # value: "vertx.eventbus.*" # exclude all metrics starts with vertx.eventbus 129 | 130 | cassandra_input_validation: 131 | forbidden_keyspaces: 132 | - system_schema 133 | - system_traces 134 | - system_distributed 135 | - system 136 | - system_auth 137 | - system_views 138 | - system_virtual_schema 139 | allowed_chars_for_directory: "[a-zA-Z][a-zA-Z0-9_]{0,47}" 140 | allowed_chars_for_quoted_name: "[a-zA-Z_0-9]{1,48}" 141 | allowed_chars_for_component_name: "[a-zA-Z0-9_-]+(.db|.cql|.json|.crc32|TOC.txt)" 142 | allowed_chars_for_restricted_component_name: "[a-zA-Z0-9_-]+(.db|TOC.txt)" 143 | 144 | blob_restore: 145 | job_discovery_active_loop_delay_millis: 5 146 | job_discovery_idle_loop_delay_millis: 10 147 | job_discovery_recency_days: 5 148 | slice_process_max_concurrency: 20 149 | restore_job_tables_ttl_seconds: 7776000 150 | 151 | s3_client: 152 | concurrency: 4 153 | thread_name_prefix: s3-client 154 | thread_keep_alive_seconds: 60 155 | # proxy_config: 156 | # uri: 157 | # username: 158 | # password: 159 | -------------------------------------------------------------------------------- /packer/cassandra/environment: -------------------------------------------------------------------------------- 1 | PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/cassandra/current/bin:/usr/local/async-profiler/bin/:/usr/local/cassandra-sidecar/bin" 2 | CASSANDRA_LOG_DIR=/mnt/cassandra/tmp 3 | EASY_CASS_STRESS_LOG_DIR=/mnt/cassandra/artifacts -------------------------------------------------------------------------------- /packer/cassandra/htoprc: -------------------------------------------------------------------------------- 1 | # Beware! This file is rewritten by htop when settings are changed in the interface. 2 | # The parser is also very primitive, and not human-friendly. 3 | fields=0 48 17 18 38 39 40 2 46 47 49 1 4 | sort_key=46 5 | sort_direction=-1 6 | tree_sort_key=0 7 | tree_sort_direction=1 8 | hide_kernel_threads=1 9 | hide_userland_threads=0 10 | shadow_other_users=0 11 | show_thread_names=1 12 | show_program_path=1 13 | highlight_base_name=0 14 | highlight_megabytes=1 15 | highlight_threads=1 16 | highlight_changes=0 17 | highlight_changes_delay_secs=5 18 | find_comm_in_cmdline=1 19 | strip_exe_from_cmdline=1 20 | show_merged_command=0 21 | tree_view=1 22 | tree_view_always_by_pid=0 23 | header_margin=1 24 | detailed_cpu_time=1 25 | cpu_count_from_one=0 26 | show_cpu_usage=1 27 | show_cpu_frequency=0 28 | show_cpu_temperature=0 29 | degree_fahrenheit=0 30 | update_process_names=0 31 | account_guest_in_cpu_meter=0 32 | color_scheme=0 33 | enable_mouse=1 34 | delay=15 35 | left_meters=AllCPUs Memory Swap 36 | left_meter_modes=1 1 1 37 | right_meters=LoadAverage Memory Uptime DiskIO NetworkIO Tasks 38 | right_meter_modes=2 1 2 2 2 2 39 | hide_function_bar=0 40 | -------------------------------------------------------------------------------- /packer/cassandra/install/install_axon.sh: -------------------------------------------------------------------------------- 1 | # setup the axon repo 2 | sudo apt-get update 3 | sudo apt-get install -y curl gnupg ca-certificates 4 | curl -L https://packages.axonops.com/apt/repo-signing-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/axonops.gpg 5 | echo "deb [arch=arm64,amd64 signed-by=/usr/share/keyrings/axonops.gpg] https://packages.axonops.com/apt axonops-apt main" | sudo tee /etc/apt/sources.list.d/axonops-apt.list 6 | sudo apt-get update 7 | 8 | sudo apt-get install -y axon-agent 9 | 10 | # need to add a new start script for C* 11 | 12 | # some versions have JVM specific agents 13 | declare -a agent_versions=("3.0-agent" "3.11-agent" "4.0-agent-jdk8" "4.0-agent" "4.1-agent-jdk8" "4.1-agent" "5.0-agent" "5.0-agent-jdk17") 14 | 15 | # Function to install an axon agent version 16 | install_axon_agent() { 17 | local agent_version=$1 18 | local package="axon-cassandra${agent_version}" 19 | 20 | echo -e "\e[32minstalling axonops-agent $package\e[0m" 21 | 22 | local tempdir=/tmp/axonops/ 23 | local targetdir=/usr/share/axonops/$agent_version/lib 24 | 25 | local jar="${package}.jar" 26 | echo "package=$package, tempdir=$tempdir, targetdir=$targetdir, jar=$jar" 27 | 28 | mkdir -p "$tempdir" 29 | 30 | # Try to install with architecture 'all' first 31 | echo "Trying to install ${package}:all..." 32 | if sudo apt install --download-only -y "${package}:all" 2>/dev/null; then 33 | local file=$(sudo find /var/cache/apt/archives/ -name "${package}*") 34 | if [ -n "$file" ]; then 35 | echo -e "\e[32mSuccessfully downloaded ${package}:all as $file\e[0m" 36 | else 37 | # If we couldn't find the file, fall back to amd64 38 | echo -e "\e[31mPackage downloaded but file not found, falling back to amd64...\e[0m" 39 | sudo apt install --download-only -y "${package}:amd64" 40 | file=$(sudo find /var/cache/apt/archives/ -name "${package}*") 41 | fi 42 | else 43 | # Fall back to amd64 if 'all' is not available 44 | echo "Package not available for 'all' architecture, falling back to amd64..." 45 | sudo apt install --download-only -y "${package}:amd64" 46 | file=$(sudo find /var/cache/apt/archives/ -name "${package}*") 47 | fi 48 | 49 | if [ -z "$file" ]; then 50 | echo -e "\e[31mERROR: Could not find package file for ${package}\e[0m" 51 | return 1 52 | fi 53 | 54 | echo "unpacking $file to $tempdir" 55 | if ! dpkg-deb -xv "$file" "$tempdir"; then 56 | echo -e "\e[31mERROR: Failed to unpack $file\e[0m" 57 | return 1 58 | fi 59 | 60 | sudo mkdir -p "$targetdir" 61 | if ! sudo cp "$tempdir"/usr/share/axonops/*.jar "$targetdir"; then 62 | echo -e "\e[31mERROR: Failed to copy jar files to $targetdir\e[0m" 63 | return 1 64 | fi 65 | 66 | # clean up after ourselves 67 | sudo rm -rf $file 68 | sudo rm -rf $tempdir 69 | } 70 | 71 | # download each version of the axon agent 72 | for agent_version in "${agent_versions[@]}" 73 | do 74 | install_axon_agent "$agent_version" 75 | done 76 | 77 | # install agent_service file included with every version since its expected 78 | # to be in a specific place 79 | #sudo cp /tmp/axonops/"$v"/var/lib/axonops/agent_service /var/lib/axonops 80 | #sudo chown -R cassandra:cassandra /var/lib/axonops 81 | 82 | # permissions setup 83 | sudo chown -R cassandra:cassandra /usr/share/axonops/ 84 | sudo chmod 0644 /etc/axonops/axon-agent.yml 85 | sudo usermod -aG cassandra axonops 86 | sudo usermod -aG axonops cassandra 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /packer/cassandra/install/install_cassandra.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #################################################################### 4 | ##### THE HEADER OF THIS FILE SHOULD BE SHELL FUNCTIONS ONLY ####### 5 | ### THE INTENT IS TO SAFELY SOURCE THE FILE WITHOUT SIDE EFFECTS ### 6 | #################################################################### 7 | 8 | ## Downloads the latest patch release of a cassandra version 9 | ## This saves us from having to update cassandra_versions.yaml 10 | ## every time a new patch release is made 11 | download_cassandra_version() { 12 | # Check if version prefix is provided 13 | if [ -z "$1" ]; then 14 | echo "Usage: download_cassandra_version " 15 | return 1 16 | fi 17 | 18 | # Assign the version prefix from the first argument 19 | version_prefix="$1" 20 | 21 | # Get the list of versions from the Cassandra download page 22 | versions=$(curl -s https://dlcdn.apache.org/cassandra/ | grep -o 'href="[0-9]\+\.[0-9]\+\.[0-9]\+/\"' | sed 's/href="//' | sed 's/\/"//') 23 | 24 | # Find the latest version that matches the given prefix 25 | full_version=$(echo "$versions" | grep "^$version_prefix" | sort -V | tail -n 1) 26 | 27 | # Check if a version was found 28 | if [ -z "$full_version" ]; then 29 | echo "No matching version found for prefix $version_prefix" 30 | return 1 31 | fi 32 | 33 | # Construct the download URL 34 | archive="apache-cassandra-$full_version-bin.tar.gz" 35 | download_url="https://dlcdn.apache.org/cassandra/$full_version/$archive" 36 | 37 | # Download the file 38 | echo "Downloading Cassandra version $full_version from $download_url..." 39 | curl -O "$download_url" 40 | 41 | # Verify if download was successful 42 | if [ $? -eq 0 ]; then 43 | echo "Download completed successfully." 44 | else 45 | echo "Failed to download Cassandra version $full_version." 46 | fi 47 | 48 | tar zxvf $archive 49 | mv apache-cassandra-$full_version $version_prefix 50 | } 51 | 52 | #################################################################### 53 | ###### DO NOT ADD ANYTHING ABOVE THIS LINE THAT MAKES CHANGES ###### 54 | ###### TO THE FILE SYSTEM OR DEPENDS ON EXTERNAL RESOURCES ######### 55 | ###### SHELL FUNCTIONS AND ALIASES ARE OK ########################## 56 | #################################################################### 57 | 58 | ## exit unless INSTALL_CASSANDRA=1 59 | if [ -z "$INSTALL_CASSANDRA" ]; then 60 | echo "INSTALL_CASSANDRA is not set, exiting." 61 | return 62 | exit 0 63 | fi 64 | 65 | set -x 66 | # creating cassandra user 67 | sudo useradd -m cassandra 68 | mkdir cassandra 69 | 70 | sudo mkdir -p /usr/local/cassandra 71 | sudo mkdir -p /mnt/cassandra/logs 72 | sudo chown -R cassandra:cassandra /mnt/cassandra 73 | 74 | # used to skip the expensive checkstyle checks 75 | 76 | sudo update-java-alternatives -s java-1.11.0-openjdk-amd64 77 | 78 | lsblk 79 | 80 | # shellcheck disable=SC2164 81 | ( 82 | cd cassandra 83 | 84 | YAML=/etc/cassandra_versions.yaml 85 | VERSIONS=$(yq '.[].version' $YAML) 86 | echo "Installing versions: $VERSIONS" 87 | 88 | for version in $VERSIONS; 89 | do 90 | echo "Configuring version: $version" 91 | export version 92 | 93 | URL=$(yq '.[] | select(.version == env(version)) | .url // ""' $YAML) 94 | echo $URL 95 | 96 | BRANCH=$(yq '.[] | select(.version == env(version)) | .branch // ""' $YAML) 97 | 98 | # if $version is set, $URL is blank, and $BRANCH is blank 99 | if [[ $version != "" && $URL == "" && $BRANCH == "" ]]; then 100 | download_cassandra_version $version 101 | # check if $version exists in the current directory 102 | if [ ! -d $version ]; then 103 | echo "Failed to download Cassandra version $version" 104 | exit 1 105 | fi 106 | sudo mv $version /usr/local/cassandra/$version 107 | 108 | # if a URL is set and ends in .tar.gz, download it 109 | elif [[ $URL == *.tar.gz ]]; then 110 | echo "Downloading $URL" 111 | wget $URL 112 | echo $(basename $URL) 113 | tar zxvf "$(basename $URL)" 114 | rm -f "$(basename $URL)" 115 | f=$(basename $URL -bin.tar.gz) 116 | sudo mv $f /usr/local/cassandra/$version 117 | else 118 | # Clone the git repos specified in the yaml file (ending in .git) 119 | # Use the directory name of the version field as the dir name 120 | # as the directory to clone into 121 | # checkout the branch specified in the yaml file 122 | # do a build and create the tar.gz 123 | ANT_FLAGS=$(yq '.[] | select(.version == env(version)) | .ant_flags // ""' $YAML) 124 | # all builds work with JDK 11 for now 125 | 126 | echo "Cloning repo" 127 | git clone --depth=1 --single-branch --branch $BRANCH $URL $version 128 | ( 129 | cd $version 130 | 131 | ant realclean && ant -Dno-checkstyle=true $ANT_FLAGS 132 | rm -rf .git 133 | ) 134 | 135 | sudo mv $version /usr/local/cassandra/$version 136 | fi 137 | # at this point the $version is in place, however it was installed 138 | # do any general customizations in the below subshell 139 | ( 140 | cd /usr/local/cassandra/$version 141 | rm -rf data 142 | cp -R conf conf.orig 143 | # create a pristine backup of the original conf 144 | sudo cp conf/cassandra.yaml conf/cassandra.orig.yaml 145 | cat /tmp/cassandra.in.sh >> bin/cassandra.in.sh 146 | ) 147 | rm -rf ~/.m2 148 | 149 | done 150 | ) 151 | 152 | #rm -rf cassandra 153 | sudo chown -R cassandra:cassandra /usr/local/cassandra 154 | 155 | -------------------------------------------------------------------------------- /packer/cassandra/install/install_easy_cass_stress.sh: -------------------------------------------------------------------------------- 1 | VERSION="9" 2 | 3 | wget https://github.com/rustyrazorblade/easy-cass-stress/releases/download/v${VERSION}/easy-cass-stress-${VERSION}.tar.gz 4 | tar zxvf easy-cass-stress-${VERSION}.tar.gz 5 | sudo mv easy-cass-stress-${VERSION} /usr/local/easy-cass-stress 6 | -------------------------------------------------------------------------------- /packer/cassandra/install/install_sidecar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # grab the repo 4 | git clone https://github.com/apache/cassandra-sidecar.git 5 | 6 | # build 7 | ( 8 | cd cassandra-sidecar 9 | ./gradlew copyJolokia installDist 10 | sudo cp -r build/install/apache-cassandra-sidecar /usr/local/cassandra-sidecar 11 | ) 12 | 13 | 14 | -------------------------------------------------------------------------------- /packer/cassandra/install/prepare_instance.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # needed early on before we do anything with /mnt 4 | if mountpoint -q /mnt; then 5 | echo "/mnt is a mount point." 6 | sudo umount -l -f /mnt 7 | else 8 | echo "/mnt is not a mount point." 9 | fi 10 | 11 | sudo apt update 12 | sudo apt upgrade -y 13 | sudo apt update 14 | 15 | sudo apt install -y wget sysstat unzip ripgrep ant ant-optional tree zfsutils-linux nicstat 16 | 17 | cpu_arch=$(uname -m) 18 | # Set ARCH based on the CPU architecture 19 | if [[ "$cpu_arch" == "x86_64" ]]; then 20 | sudo apt install -y cpuid 21 | elif [[ "$cpu_arch" == "aarch64" ]]; then 22 | echo "No additional packages needed for ARM64" 23 | else 24 | echo "Unsupported architecture: $cpu_arch" 25 | exit 1 26 | fi 27 | 28 | 29 | -------------------------------------------------------------------------------- /packer/cassandra/patch-jvm-options.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | ###### 5 | # This is a work in progress. I don't know if we're going to use it. 6 | # The goal is to be able to patch fragments of the JVM config, 7 | # but I honestly don't know if there's a point. 8 | # I started this with the assumption that it's similar to the C* config 9 | # However looking at it now I realize I'm going to need to edit the file rather than patch it. 10 | # I might come back to this later 11 | ###### 12 | 13 | # Examples: 14 | # python patch-jvm-options.py -v 3.0 -i jvm.patch.options 15 | 16 | # read the patch file line by line and index the lines 17 | 18 | # read the base file line by line 19 | 20 | # iterate over the base file. if we see an option that's patched, use the patch version 21 | # there's some extra logic we need to handle, like conflicting GC options 22 | # some things are also exclusive, like -Xmn and -XX:MaxNewSize 23 | 24 | # examples of valid JVM options 25 | 26 | # -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414 27 | # -XX:+FlightRecorder 28 | # -Xms4G 29 | # -Dcassandra.force_default_indexing_page_size=true 30 | 31 | import sys 32 | import re 33 | import argparse 34 | import yaml 35 | 36 | class JVMOption: 37 | key = None 38 | value = None 39 | sep = "" 40 | 41 | # regex to split on = 42 | 43 | def __init__(self, line): 44 | if "=" in line: 45 | self.sep = "=" 46 | self.key, self.value = line.split("=") 47 | elif line.startswith("-Xm"): 48 | tmp = re.match("(-Xm[a-z])(.*)", line) 49 | self.sep = "" 50 | self.key = tmp[1] 51 | self.value = tmp[2] 52 | else: 53 | self.key = line 54 | self.value = "" 55 | # print representation of key and value 56 | def __repr__(self): 57 | return self.key + self.sep + self.value 58 | 59 | 60 | ####################################################### 61 | 62 | # time to do the thing 63 | parser = argparse.ArgumentParser( 64 | description="Patch JVM options file with a patch file" 65 | ) 66 | 67 | group = parser.add_mutually_exclusive_group() 68 | 69 | # either specify the output file or the version to patch 70 | # when specifying a version, the output file is determined by cassandra_versions.yaml 71 | group.add_argument("-o", "--output", help="Output file.", required=False) 72 | group.add_argument("-v", "--version", help="Version of Cassandra to patch", required=False) 73 | 74 | # we don't need the base 75 | parser.add_argument("-b", "--base", help="Base config", required=False) 76 | parser.add_argument("-p", "--patch", help="Patch file", required=True) 77 | parser.add_argument("-c", help="cassandra_versions.yaml", default="/etc/cassandra_versions.yaml") 78 | 79 | args = parser.parse_args() 80 | 81 | ## extracting all the variables here 82 | patch = args.patch 83 | base = args.base 84 | 85 | ## if we have a version, we can figure out the base file and the output file 86 | if args.version: 87 | print("Version: " + args.version) 88 | with open(args.c, "r") as versions: 89 | data = yaml.safe_load(versions) 90 | if args.version in data: 91 | base = data[args.version]["jvm"] 92 | output = data[args.version]["jvm"] 93 | else: 94 | print("Version not found in cassandra_versions.yaml") 95 | sys.exit(1) 96 | 97 | print(args) 98 | sys.exit(1) 99 | 100 | 101 | 102 | # iterate over the patch file line by line and index the lines 103 | patch = {} 104 | with open(args.patch, "r") as patch_file: 105 | for line in patch_file: 106 | if line.startswith("#") or line.strip() == "": 107 | continue 108 | option = JVMOption(line.strip()) 109 | patch[option.key] = option 110 | 111 | # some arguments are mutually exclusive 112 | 113 | 114 | output = open(sys.argv[3], "w") 115 | 116 | with open(sys.argv[1], "r") as base_file: 117 | for line in base_file: 118 | if line.startswith("#") or line.strip() == "": 119 | output.write(line) 120 | continue 121 | option = JVMOption(line.strip()) 122 | if option.key in patch: 123 | print("Patching " + option.key + " with " + patch[option.key].value) 124 | print(patch[option.key]) 125 | output.write(str(patch[option.key]) + "\n") 126 | # remove the patch entry 127 | del patch[option.key] 128 | else: 129 | print("Keeping " + option.key + " with " + option.value) 130 | print(option) 131 | output.write(str(option) + "\n") 132 | 133 | 134 | 135 | 136 | # add any remaining patch entries 137 | for key in patch: 138 | print("Adding " + key + " with " + patch[key].value) 139 | print(patch[key]) 140 | output.write(str(patch[key]) + "\n") 141 | 142 | output.close() 143 | -------------------------------------------------------------------------------- /packer/cassandra/services/cassandra-sidecar.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Apache Cassandra Sidecar 3 | 4 | [Service] 5 | ExecStart=/usr/local/cassandra-sidecar/bin/cassandra-sidecar 6 | User=cassandra 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | 11 | -------------------------------------------------------------------------------- /packer/cassandra/services/cassandra.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Apache Cassandra 3 | 4 | [Service] 5 | ExecStart=/usr/local/cassandra/current/bin/cassandra -f 6 | ExecStop=/usr/local/cassandra/current/bin/nodetool drain 7 | User=cassandra 8 | Environment=CASSANDRA_LOG_DIR=/mnt/cassandra/logs 9 | Environment=CASSANDRA_HEAPDUMP_DIR=/mnt/cassandra/artifacts 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | 14 | -------------------------------------------------------------------------------- /packer/cassandra/test/jvm.patch.options: -------------------------------------------------------------------------------- 1 | -Dcassandra.disable_auth_caches_remote_configuration=true 2 | -XX:+HeapDumpOnOutOfMemoryError 3 | -Xss256k 4 | 5 | -XX:+UnlockExperimentalVMOptions 6 | -XX:+UseZGC -------------------------------------------------------------------------------- /packer/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ecl: 3 | stdin_open: true # docker run -i 4 | tty: true # docker run -t 5 | build: 6 | context: . 7 | dockerfile: Dockerfile 8 | volumes: 9 | - ./:/app 10 | working_dir: /app 11 | entrypoint: ["/bin/bash"] 12 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This settings file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * In a single project build this file can be empty or even removed. 6 | * 7 | * Detailed information about configuring a multi-project build in Gradle can be found 8 | * in the user guide at https://docs.gradle.org/3.4.1/userguide/multi_project_builds.html 9 | */ 10 | 11 | /* 12 | // To declare projects as part of a multi-project build use the 'include' method 13 | include 'shared' 14 | include 'api' 15 | include 'services:webservice' 16 | */ 17 | 18 | rootProject.name = 'easy-cass-lab' 19 | 20 | include "core" 21 | include "manual" -------------------------------------------------------------------------------- /src/integration-test/kotlin/com/rustyrazorblade/easycasslab/MainTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | import org.junit.jupiter.api.Test 4 | 5 | class MainTest { 6 | 7 | @Test 8 | fun basicTest() { 9 | com.rustyrazorblade.easycasslab.main( 10 | arrayOf( 11 | "init", 12 | "easy-cass-lab", 13 | "no ticket", 14 | "automated test suite", 15 | "-s", 16 | "1" 17 | ) 18 | ) 19 | com.rustyrazorblade.easycasslab.main(arrayOf("up", "--yes")) 20 | com.rustyrazorblade.easycasslab.main(arrayOf("use", "3.11.4")) 21 | com.rustyrazorblade.easycasslab.main(arrayOf("install")) 22 | com.rustyrazorblade.easycasslab.main(arrayOf("start")) 23 | com.rustyrazorblade.easycasslab.main(arrayOf("down", "--yes")) 24 | com.rustyrazorblade.easycasslab.main(arrayOf("clean")) 25 | } 26 | 27 | 28 | fun init() { 29 | 30 | } 31 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/Cassandra.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | import java.io.File 4 | 5 | /** 6 | * Manages a the Cassandra build process 7 | */ 8 | class Cassandra { 9 | 10 | // FIXME: un-hardcode 11 | val buildDir = File(System.getProperty("user.home"), "/.easy-cass-lab/builds") 12 | 13 | 14 | fun listBuilds() : List { 15 | return buildDir.listFiles().filter { it.isDirectory }.map { it.name } 16 | } 17 | 18 | 19 | 20 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/CommandLineParser.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | import com.beust.jcommander.JCommander 4 | import com.beust.jcommander.Parameter 5 | import com.fasterxml.jackson.annotation.JsonIgnore 6 | import com.rustyrazorblade.easycasslab.commands.* 7 | import org.apache.logging.log4j.kotlin.logger 8 | import java.util.regex.Pattern 9 | 10 | data class Command(val name: String, val command: ICommand, val aliases: List = listOf()) 11 | 12 | class MainArgs { 13 | @Parameter(names = ["--help", "-h"], description = "Shows this help.") 14 | var help = false 15 | } 16 | 17 | class CommandLineParser(val context: Context) { 18 | val commands: List 19 | 20 | @JsonIgnore 21 | private val logger = logger("com.rustyrazorblade.easycasslab.CommandLineParser") 22 | private val jc: JCommander 23 | private val regex = """("([^"\\]|\\.)*"|'([^'\\]|\\.)*'|[^\s"']+)+""".toRegex() 24 | 25 | 26 | init { 27 | 28 | val jcommander = JCommander.newBuilder().programName("easy-cass-lab") 29 | val args = MainArgs() 30 | jcommander.addObject(args) 31 | 32 | commands = listOf( 33 | Command("build-base", BuildBaseImage(context)), 34 | Command("build-cassandra", BuildCassandraImage(context)), 35 | Command("build-image", BuildImage(context)), 36 | Command("clean", Clean()), 37 | Command("down", Down(context)), 38 | Command("download-config", DownloadConfig(context), listOf("dc")), 39 | Command("hosts", Hosts(context)), 40 | Command("init", Init(context)), 41 | Command("list", ListVersions(context), listOf("ls")), 42 | Command("setup-instances", SetupInstance(context), listOf("si")), 43 | Command("start", Start(context)), 44 | Command("stop", Stop(context)), 45 | Command("restart", Restart(context)), 46 | Command("up", Up(context)), 47 | Command("update-config", UpdateConfig(context), listOf("uc")), 48 | Command("use", UseCassandra(context)), 49 | Command("write-config", WriteConfig(context), listOf("wc")), 50 | Command("configure-axonops", ConfigureAxonOps(context)), 51 | Command("upload-keys", UploadAuthorizedKeys(context)), 52 | Command("repl", Repl(context)), 53 | Command("version", Version(context)) 54 | ) 55 | 56 | for(c in commands) { 57 | logger.debug { "Adding command: ${c.name}" } 58 | jcommander.addCommand(c.name, c.command, *c.aliases.toTypedArray()) 59 | } 60 | 61 | jc = jcommander.build() 62 | } 63 | 64 | // For the repl 65 | fun eval(input: String) { 66 | val matches = regex.findAll(input) 67 | val result = mutableListOf() 68 | 69 | for (match in matches) { 70 | result.add(match.value.trim('"', '\'')) 71 | } 72 | 73 | return eval(result.toTypedArray()) 74 | } 75 | 76 | fun eval(input: Array) { 77 | jc.parse(*input) 78 | commands.filter { it.name == jc.parsedCommand }.firstOrNull()?.run { 79 | this.command.execute() 80 | } ?: run { 81 | jc.usage() 82 | } 83 | } 84 | 85 | 86 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/Containers.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | enum class Containers(val containerName: String, val tag: String) { 4 | TERRAFORM("ghcr.io/opentofu/opentofu", "1.7"), 5 | PACKER("hashicorp/packer", "full"); 6 | 7 | val imageWithTag : String 8 | get() = "$containerName:$tag" 9 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/EC2.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | import software.amazon.awssdk.auth.credentials.AwsBasicCredentials 4 | import software.amazon.awssdk.services.ec2.Ec2Client 5 | import software.amazon.awssdk.regions.Region 6 | 7 | 8 | class EC2(key: String, secret: String, region: Region) { 9 | val client : Ec2Client 10 | 11 | init { 12 | val creds = AwsBasicCredentials.create(key, secret) 13 | // TODO: Abstract the provider out 14 | // tlp cluster should have its own provider that uses the following order: 15 | // easy-cass-lab config, AWS config 16 | client = Ec2Client.builder().region(region) 17 | .credentialsProvider { creds } 18 | .build() 19 | } 20 | 21 | fun isInstanceStore(instanceType: String) : Boolean { 22 | 23 | return true 24 | } 25 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/Main.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | import com.github.ajalt.mordant.TermColors 4 | import java.io.File 5 | 6 | 7 | fun main(arguments: Array) { 8 | 9 | val easycasslabUserDirectory = File(System.getProperty("user.home"), "/.easy-cass-lab/") 10 | 11 | val context = Context(easycasslabUserDirectory) 12 | val parser = CommandLineParser(context) 13 | try { 14 | parser.eval(arguments) 15 | } catch (e: DockerException) { 16 | e.printStackTrace() 17 | 18 | with(TermColors()) { 19 | println(red("There was an error connecting to docker. Please check if it is running.")) 20 | } 21 | } catch (e: java.rmi.RemoteException) { 22 | with (TermColors()) { 23 | println(red("There was an error executing the remote command. Try rerunning it.")) 24 | } 25 | } 26 | catch (e: Exception) { 27 | with(TermColors()) { 28 | println(red("An unknown exception has occurred.")) 29 | } 30 | with(TermColors()) { 31 | println(red("Does this look like an error with easy-cass-lab? If so, please file a bug report at https://github.com/rustyrazorblade/easy-cass-lab/ with the following information:")) 32 | } 33 | println(e.message) 34 | println(e.stackTraceToString()) 35 | 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/ResourceFile.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | import org.apache.commons.io.FileUtils 4 | import org.apache.commons.io.FilenameUtils 5 | import org.apache.commons.io.IOUtils 6 | import org.apache.logging.log4j.kotlin.logger 7 | import java.io.File 8 | import java.io.InputStream 9 | 10 | /** 11 | * Creates a temporary file from a resource location 12 | */ 13 | class ResourceFile(val resource: InputStream) { 14 | 15 | val fp: File 16 | val log = logger() 17 | 18 | init { 19 | checkNotNull(resource) 20 | 21 | val tmpDir = File(System.getProperty("user.home"), ".easy-cass-lab/tmp") 22 | 23 | if(!tmpDir.exists()) { 24 | log.debug { "Creating temporary directory at $tmpDir" } 25 | tmpDir.mkdirs() 26 | } 27 | 28 | fp = File.createTempFile( 29 | FilenameUtils.getBaseName("resource"), 30 | FilenameUtils.getExtension("tmp"), 31 | tmpDir) 32 | 33 | IOUtils.copy(resource, FileUtils.openOutputStream(fp)) 34 | 35 | } 36 | 37 | val path : String 38 | get() = fp.absolutePath 39 | 40 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/Utils.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab 2 | 3 | import org.apache.commons.io.IOUtils 4 | import java.io.File 5 | import java.io.InputStream 6 | import java.io.FileOutputStream 7 | 8 | 9 | class Utils { 10 | companion object { 11 | fun inputstreamToTempFile(inputStream: InputStream, prefix: String, directory: String) : File { 12 | val tempFile = File.createTempFile(prefix, "", File(directory)) 13 | tempFile.deleteOnExit() 14 | 15 | val outputStream = FileOutputStream(tempFile) 16 | 17 | IOUtils.copy(inputStream, outputStream) 18 | outputStream.flush() 19 | outputStream.close() 20 | 21 | return tempFile 22 | } 23 | 24 | @Deprecated(message = "Please use ResourceFile") 25 | fun resourceToTempFile(resourcePath: String, directory: String) : File { 26 | val resourceName = File(resourcePath).name 27 | val resourceStream = this::class.java.getResourceAsStream(resourcePath) 28 | return Utils.inputstreamToTempFile(resourceStream, "${resourceName}_", directory) 29 | } 30 | 31 | fun prompt(question: String, default: String, secret : Boolean = false) : String { 32 | print("$question [$default]: ") 33 | 34 | var line : String = if(secret) { 35 | String(System.console()?.readPassword()!!) 36 | } 37 | else { 38 | (readLine() ?: default).trim() 39 | } 40 | 41 | if(line.equals("")) 42 | line = default 43 | 44 | return line 45 | } 46 | 47 | fun resolveSshKeyPath(keyPath: String) : String { 48 | val sshKeyPath: String by lazy { 49 | var path = keyPath 50 | 51 | if (path.startsWith("~/")) { 52 | path = path.replaceFirst("~/", "${System.getProperty("user.home")}/") 53 | } 54 | 55 | path 56 | } 57 | 58 | return File(sshKeyPath).absolutePath 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/BuildBaseImage.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameters 4 | import com.beust.jcommander.ParametersDelegate 5 | import com.rustyrazorblade.easycasslab.Context 6 | import com.rustyrazorblade.easycasslab.commands.delegates.BuildArgs 7 | import com.rustyrazorblade.easycasslab.containers.Packer 8 | 9 | @Parameters(commandDescription = "Build the base image.") 10 | class BuildBaseImage(val context: Context) : ICommand { 11 | @ParametersDelegate 12 | var buildArgs = BuildArgs(context) 13 | 14 | override fun execute() { 15 | val packer = Packer(context, "base") 16 | packer.build("base.pkr.hcl", buildArgs) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/BuildCassandraImage.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameters 4 | import com.beust.jcommander.ParametersDelegate 5 | import com.rustyrazorblade.easycasslab.Context 6 | import com.rustyrazorblade.easycasslab.commands.delegates.BuildArgs 7 | import com.rustyrazorblade.easycasslab.containers.Packer 8 | 9 | @Parameters(commandDescription = "Build the Cassandra image.") 10 | class BuildCassandraImage(val context: Context) : ICommand { 11 | @ParametersDelegate 12 | var buildArgs = BuildArgs(context) 13 | 14 | override fun execute() { 15 | val packer = Packer(context, "cassandra") 16 | packer.build("cassandra.pkr.hcl", buildArgs) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/BuildImage.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameters 4 | import com.beust.jcommander.ParametersDelegate 5 | import com.rustyrazorblade.easycasslab.Context 6 | import com.rustyrazorblade.easycasslab.commands.delegates.BuildArgs 7 | 8 | @Parameters(commandDescription = "Build both the base and Cassandra image.") 9 | class BuildImage(val context: Context) : ICommand { 10 | @ParametersDelegate 11 | var buildArgs = BuildArgs(context) 12 | 13 | override fun execute() { 14 | BuildBaseImage(context) 15 | .apply { 16 | this.buildArgs=this@BuildImage.buildArgs 17 | } 18 | .execute() 19 | BuildCassandraImage(context) 20 | .apply { 21 | this.buildArgs=this@BuildImage.buildArgs 22 | } 23 | .execute() 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Clean.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import java.io.File 4 | 5 | class Clean : ICommand { 6 | override fun execute() { 7 | val toDelete = listOf( 8 | "create_provisioning_resources.sh", 9 | "cassandra.patch.yaml", 10 | "jmx.options", 11 | "seeds.txt", 12 | "terraform.tfstate", 13 | "terraform.tfstate.backup", 14 | "stress_ips.txt", 15 | "hosts.txt", 16 | "terraform.tf.json", 17 | "terraform.tfvars", 18 | "sshConfig", 19 | "env.sh", 20 | "environment.sh", 21 | "setup_instance.sh", 22 | ".terraform.lock.hcl", 23 | "logs", 24 | "state.json", 25 | "axonops-dashboards.json", 26 | "cassandra_versions.yaml" 27 | ) 28 | 29 | for(f in toDelete) { 30 | File(f).deleteRecursively() 31 | } 32 | File(".terraform").deleteRecursively() 33 | File("provisioning").deleteRecursively() 34 | val artifacts = File("artifacts") 35 | 36 | if (artifacts.isDirectory) { 37 | if (artifacts.listFiles().isEmpty()) { 38 | artifacts.delete() 39 | } else { 40 | println("Not deleting artifacts directory, it contains artifacts.") 41 | } 42 | } 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/ConfigureAxonOps.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.rustyrazorblade.easycasslab.Context 7 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 8 | import com.rustyrazorblade.easycasslab.configuration.ServerType 9 | 10 | @Parameters(commandDescription = "setup / configure axon-agent for use with the Cassandra cluster") 11 | class ConfigureAxonOps(val context: Context) : ICommand { 12 | 13 | @Parameter(description = "AxonOps Organization Name", names = ["--org"]) 14 | var org = "" 15 | 16 | @Parameter(description = "AxonOps API Key", names = ["--key"]) 17 | var key = "" 18 | 19 | @ParametersDelegate 20 | var hosts = Hosts() 21 | 22 | 23 | override fun execute() { 24 | context.requireSshKey() 25 | 26 | val axonOrg = if (org.isNotBlank()) org else context.userConfig.axonOpsOrg 27 | val axonKey = if (key.isNotBlank()) key else context.userConfig.axonOpsKey 28 | if ((axonOrg.isBlank() || axonKey.isBlank())) { 29 | println("--org and --key are required") 30 | System.exit(1) 31 | } 32 | 33 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { 34 | println("Configure axonops on $it") 35 | 36 | context.executeRemotely(it, "/usr/local/bin/setup-axonops $axonOrg $axonKey", secret = true).text 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Down.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.rustyrazorblade.easycasslab.Context 6 | import com.rustyrazorblade.easycasslab.containers.Terraform 7 | 8 | @Parameters(commandDescription = "Shut down a cluster") 9 | class Down(val context: Context) : ICommand { 10 | @Parameter(description = "Auto approve changes", names = ["--auto-approve", "-a", "--yes"]) 11 | var autoApprove = false 12 | 13 | override fun execute() { 14 | println("Crushing dreams, terminating instances.") 15 | 16 | Terraform(context).down(autoApprove) 17 | } 18 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/DownloadConfig.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.github.ajalt.mordant.TermColors 7 | import com.rustyrazorblade.easycasslab.Context 8 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 9 | import com.rustyrazorblade.easycasslab.configuration.ServerType 10 | import org.apache.logging.log4j.kotlin.logger 11 | import java.io.File 12 | import java.nio.file.Path 13 | import kotlin.io.path.exists 14 | import kotlin.io.path.isDirectory 15 | 16 | /** 17 | * Downloads configuration. 18 | */ 19 | @Parameters(commandDescription = "Download JVM and YAML config files.") 20 | class DownloadConfig(val context: Context) : ICommand { 21 | @ParametersDelegate 22 | var hosts = Hosts() 23 | 24 | @Parameter(names = ["--version"], description = "Version to download, default is current") 25 | var version = "current" 26 | 27 | companion object { 28 | val logger = logger() 29 | } 30 | 31 | override fun execute() { 32 | val cassandraHosts = context.tfstate.getHosts(ServerType.Cassandra) 33 | 34 | // TODO: add host option 35 | val host = cassandraHosts.first() 36 | val resolvedVersion = context.getRemoteVersion(host, version) 37 | 38 | logger.info("Original version: $version. Resolved version: ${resolvedVersion.version}. ") 39 | val localDir = resolvedVersion.localDir.toFile() 40 | 41 | // Dont' overwrite. 42 | // TODO: Add a flag to customize this this later, and make it possible to have node specific settings 43 | if (!localDir.exists()) { 44 | localDir.mkdirs() 45 | 46 | context.downloadDirectory( 47 | host, 48 | remoteDir = "${resolvedVersion.path}/conf", 49 | localDir = localDir, 50 | excludeFilters = listOf( 51 | "cassandra*.yaml", 52 | "axonenv*", 53 | "cassandra-rackdc.properties" 54 | ) 55 | ) 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Hosts.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.rustyrazorblade.easycasslab.Context 5 | import com.rustyrazorblade.easycasslab.configuration.HostList 6 | import com.rustyrazorblade.easycasslab.configuration.ServerType 7 | import java.io.FileNotFoundException 8 | 9 | class Hosts(val context: Context) : ICommand { 10 | 11 | 12 | @Parameter(names = ["-c"], description = "Show Cassandra as a comma delimited list") 13 | var cassandra : Boolean = false 14 | 15 | data class HostOutput(val cassandra: HostList, val stress: HostList, val monitoring: HostList) 16 | 17 | override fun execute() { 18 | try { 19 | val output = with(context.tfstate) { 20 | HostOutput(getHosts(ServerType.Cassandra), 21 | getHosts(ServerType.Stress), 22 | getHosts(ServerType.Monitoring)) 23 | } 24 | 25 | if(cassandra) { 26 | val hosts = context.tfstate.getHosts(ServerType.Cassandra) 27 | val csv = hosts.map { it.public }.joinToString(",") 28 | println(csv) 29 | } else { 30 | context.yaml.writeValue(System.out, output) 31 | } 32 | } catch (e: FileNotFoundException) { 33 | println("terraform.tfstate does not exist yet, most likely easy-cass-lab up has not been run.") 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/ICommand.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | interface ICommand { 4 | 5 | fun execute() 6 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/ListVersions.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameters 4 | import com.rustyrazorblade.easycasslab.Context 5 | import com.rustyrazorblade.easycasslab.configuration.ServerType 6 | 7 | @Parameters(commandDescription = "List available versions", commandNames = ["list", "ls"]) 8 | class ListVersions(val context: Context) : ICommand { 9 | override fun execute() { 10 | context.tfstate.getHosts(ServerType.Cassandra).first().let { 11 | val response = context.executeRemotely(it, "ls /usr/local/cassandra", output = false) 12 | response.text.split("\n") 13 | .filter { !it.equals("current") } 14 | .forEach { println(it) } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Repl.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.rustyrazorblade.easycasslab.CommandLineParser 4 | import com.rustyrazorblade.easycasslab.Context 5 | 6 | import org.jline.reader.*; 7 | import org.jline.reader.impl.completer.StringsCompleter; 8 | import org.jline.terminal.Terminal; 9 | import org.jline.terminal.TerminalBuilder; 10 | import java.io.IOException 11 | 12 | class Repl(val context: Context) : ICommand { 13 | override fun execute() { 14 | try { 15 | // Set up the terminal 16 | val terminal: Terminal = TerminalBuilder.builder().build() 17 | 18 | // just to prove this out 19 | // todo: add cluster names, C* configuration options 20 | // also make it context aware 21 | var parser = CommandLineParser(context) 22 | val commands = parser.commands.map { it.name } 23 | 24 | // Set up the line reader 25 | val reader: LineReader = LineReaderBuilder.builder() 26 | .terminal(terminal) 27 | .completer(StringsCompleter(commands)) 28 | .build() 29 | 30 | var line: String? 31 | while (true) { 32 | // Read user input with prompt 33 | try { 34 | // TODO enhance this with a better status line about the cluster 35 | line = reader.readLine(getPrompt()) 36 | if (line.equals("exit", ignoreCase = true)) { 37 | break 38 | } 39 | 40 | // blank line we can just start the loop over 41 | if (line.isBlank()) { 42 | continue 43 | } 44 | 45 | // Handle the command input 46 | // we have to create a new parser every time due to a jcommander limitation 47 | // See https://github.com/cbeust/jcommander/issues/271 48 | parser = CommandLineParser(context) 49 | parser.eval(line) 50 | } catch (e: UserInterruptException) { 51 | // Handle CTRL+C 52 | break 53 | } catch (e: EndOfFileException) { 54 | // Handle CTRL+D 55 | break 56 | } 57 | } 58 | } catch (e: IOException) { 59 | e.printStackTrace() 60 | } 61 | } 62 | 63 | fun getPrompt() : String { 64 | return "> " 65 | } 66 | 67 | 68 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Restart.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.github.ajalt.mordant.TermColors 7 | import com.rustyrazorblade.easycasslab.Context 8 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 9 | import com.rustyrazorblade.easycasslab.configuration.ServerType 10 | 11 | @Parameters(commandDescription = "Restart cassandra", commandNames = ["restart"]) 12 | class Restart(val context: Context) : ICommand { 13 | @ParametersDelegate 14 | var hosts = Hosts() 15 | 16 | override fun execute() { 17 | // TODO wait for cassandra to become available 18 | println("Restarting cassandra service on all nodes.") 19 | with(TermColors()) { 20 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { 21 | println(green("Restarting $it")) 22 | context.executeRemotely(it, "/usr/local/bin/restart-cassandra-and-wait").text 23 | } 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/SetupInstance.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameters 4 | import com.beust.jcommander.ParametersDelegate 5 | import com.rustyrazorblade.easycasslab.Context 6 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 7 | import com.rustyrazorblade.easycasslab.configuration.Host 8 | import com.rustyrazorblade.easycasslab.configuration.ServerType 9 | import java.nio.file.Path 10 | 11 | @Parameters(commandDescription = "Runs setup_instance.sh on all Cassandra instances") 12 | class SetupInstance(val context: Context) : ICommand { 13 | @ParametersDelegate 14 | var hosts = Hosts() 15 | 16 | override fun execute() { 17 | fun setup(host: Host) { 18 | context.upload(host, Path.of("environment.sh"), "environment.sh") 19 | context.executeRemotely(host, "sudo mv environment.sh /etc/profile.d/stress.sh").text 20 | } 21 | 22 | context.tfstate.withHosts(ServerType.Stress, hosts) { 23 | setup(it) 24 | context.executeRemotely(it, "sudo hostnamectl set-hostname ${it.alias}").text 25 | context.upload(it, Path.of("setup_instance.sh"), "setup_instance.sh") 26 | context.executeRemotely(it, "sudo bash setup_instance.sh").text 27 | } 28 | context.tfstate.withHosts(ServerType.Cassandra, Hosts.all()) { 29 | setup(it) 30 | context.executeRemotely(it, "sudo hostnamectl set-hostname ${it.alias}").text 31 | context.upload(it, Path.of("setup_instance.sh"), "setup_instance.sh") 32 | context.executeRemotely(it, "sudo bash setup_instance.sh").text 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Start.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.github.ajalt.mordant.TermColors 7 | import com.rustyrazorblade.easycasslab.Context 8 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 9 | import com.rustyrazorblade.easycasslab.configuration.ServerType 10 | 11 | @Parameters(commandDescription = "Start cassandra on all nodes via service command") 12 | class Start(val context: Context) : ICommand { 13 | 14 | @Parameter(names = ["--sleep"], description = "Time to sleep between starts in seconds") 15 | var sleep : Long = 120 16 | 17 | @ParametersDelegate 18 | var hosts = Hosts() 19 | 20 | 21 | override fun execute() { 22 | context.requireSshKey() 23 | 24 | with(TermColors()) { 25 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { 26 | println(green("Starting $it")) 27 | context.executeRemotely(it, "sudo systemctl start cassandra").text 28 | println("Cassandra started, waiting for up/normal") 29 | context.executeRemotely(it, "sudo wait-for-up-normal").text 30 | context.executeRemotely(it, "sudo systemctl start cassandra-sidecar").text 31 | } 32 | } 33 | 34 | if (context.userConfig.axonOpsOrg.isNotBlank() && context.userConfig.axonOpsKey.isNotBlank()) { 35 | StartAxonOps(context).execute() 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/StartAxonOps.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameters 4 | import com.beust.jcommander.ParametersDelegate 5 | import com.rustyrazorblade.easycasslab.Context 6 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 7 | import com.rustyrazorblade.easycasslab.configuration.ServerType 8 | 9 | @Parameters(commandDescription = "Start axon-agent on all nodes via service command") 10 | class StartAxonOps(val context: Context) : ICommand { 11 | 12 | @ParametersDelegate 13 | var hosts = Hosts() 14 | 15 | override fun execute() { 16 | context.requireSshKey() 17 | context.tfstate.withHosts(ServerType.Cassandra, Hosts.all()) { 18 | context.executeRemotely(it, "sudo systemctl start axon-agent").text 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Stop.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.rustyrazorblade.easycasslab.Context 7 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 8 | import com.rustyrazorblade.easycasslab.configuration.ServerType 9 | 10 | @Parameters(commandDescription = "Stop cassandra on all nodes via service command") 11 | class Stop(val context: Context) : ICommand { 12 | 13 | @ParametersDelegate 14 | var hosts = Hosts() 15 | 16 | override fun execute() { 17 | context.requireSshKey() 18 | 19 | println("Stopping cassandra service on all nodes.") 20 | context.requireSshKey() 21 | 22 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { 23 | context.executeRemotely(it, "sudo systemctl stop cassandra").text 24 | } 25 | 26 | } 27 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Up.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.fasterxml.jackson.annotation.JsonIgnore 7 | import com.github.ajalt.mordant.TermColors 8 | import com.rustyrazorblade.easycasslab.Context 9 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 10 | import com.rustyrazorblade.easycasslab.configuration.Host 11 | import com.rustyrazorblade.easycasslab.configuration.ServerType 12 | import com.rustyrazorblade.easycasslab.containers.Terraform 13 | import org.apache.sshd.common.SshException 14 | import java.io.File 15 | import java.nio.file.Path 16 | 17 | @Parameters(commandDescription = "Starts instances") 18 | class Up(@JsonIgnore val context: Context) : ICommand { 19 | 20 | @Parameter(names = ["--no-setup", "-n"]) 21 | var noSetup = false 22 | 23 | @ParametersDelegate 24 | var hosts = Hosts() 25 | 26 | override fun execute() { 27 | // we have to list both the variable files explicitly here 28 | // even though we have a terraform.tvars 29 | // we need the local one to apply at the highest priority 30 | // specifying the user one makes it take priority over the local one 31 | // so we have to explicitly specify the local one to ensure it gets 32 | // priority over user 33 | val terraform = Terraform(context) 34 | 35 | with(TermColors()) { 36 | 37 | terraform.up().onFailure { 38 | println(it.message) 39 | println(it.printStackTrace()) 40 | println("${red("Some resources may have been unsuccessfully provisioned.")} Rerun ${green("easy-cass-lab up")} to provision the remaining resources.") 41 | }.onSuccess { 42 | 43 | println("""Instances have been provisioned. 44 | 45 | Use ${green("easy-cass-lab list")} to see all available versions 46 | 47 | Then use ${green("easy-cass-lab use ")} to use a specific version of Cassandra. 48 | 49 | """.trimMargin()) 50 | 51 | println("Writing ssh config file to sshConfig.") 52 | 53 | println("""The following alias will allow you to easily work with the cluster: 54 | | 55 | |${green("source env.sh")} 56 | | 57 | |""".trimMargin()) 58 | println("You can edit ${green("cassandra.patch.yaml")} with any changes you'd like to see merge in into the remote cassandra.yaml file.") 59 | } 60 | } 61 | 62 | val config = File("sshConfig").bufferedWriter() 63 | context.tfstate.writeSshConfig(config) 64 | 65 | val envFile = File("env.sh").bufferedWriter() 66 | context.tfstate.writeEnvironmentFile(envFile) 67 | 68 | // sets up any environment variables we need for the stress tool 69 | val stressEnvironmentVars = File("environment.sh").bufferedWriter() 70 | stressEnvironmentVars.write("#!/usr/bin/env bash") 71 | stressEnvironmentVars.newLine() 72 | 73 | val host = context.tfstate.getHosts(ServerType.Cassandra).first().private 74 | 75 | stressEnvironmentVars.write("export EASY_CASS_STRESS_CASSANDRA_HOST=$host") 76 | stressEnvironmentVars.newLine() 77 | stressEnvironmentVars.write("export EASY_CASS_STRESS_PROM_PORT=0") 78 | stressEnvironmentVars.newLine() 79 | 80 | stressEnvironmentVars.write("export EASY_CASS_STRESS_DEFAULT_DC=\$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | yq .region)") 81 | stressEnvironmentVars.newLine() 82 | stressEnvironmentVars.flush() 83 | stressEnvironmentVars.close() 84 | 85 | WriteConfig(context).execute() 86 | 87 | // once the instances are up we can connect and set up 88 | // the disks, axonops, system settings, etc 89 | // we can't set up the configs yet though, 90 | // because those are dependent on the C* version in use. 91 | println("Waiting for SSH to come up..") 92 | Thread.sleep(5000) 93 | 94 | // probably need to loop and wait 95 | // write to profile.d/stress.sh 96 | var done = false 97 | do { 98 | 99 | try { 100 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { 101 | context.executeRemotely(it, "echo 1").text 102 | // download /etc/cassandra_versions.yaml if we don't have it yet 103 | if (!File("cassandra_versions.yaml").exists()) { 104 | context.download(it, "/etc/cassandra_versions.yaml", Path.of("cassandra_versions.yaml")) 105 | } 106 | } 107 | done = true 108 | } catch (e: SshException) { 109 | println("SSH still not up yet, waiting..") 110 | Thread.sleep(1000) 111 | } 112 | 113 | } while (!done) 114 | 115 | if (noSetup) { 116 | with (TermColors()) { 117 | println("Skipping node setup. You will need to run ${green("easy-cass-lab setup-instance")} to complete setup") 118 | 119 | } 120 | } else { 121 | 122 | SetupInstance(context).execute() 123 | 124 | if (context.userConfig.axonOpsKey.isNotBlank() && context.userConfig.axonOpsOrg.isNotBlank()) { 125 | println("Setting up axonops for ${context.userConfig.axonOpsOrg}") 126 | 127 | ConfigureAxonOps(context).execute() 128 | } 129 | } 130 | } 131 | 132 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/UpdateConfig.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.fasterxml.jackson.databind.node.ObjectNode 7 | import com.rustyrazorblade.easycasslab.Context 8 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 9 | import com.rustyrazorblade.easycasslab.configuration.ServerType 10 | import java.nio.file.Files 11 | import java.nio.file.Path 12 | import kotlin.io.path.deleteExisting 13 | import kotlin.io.path.inputStream 14 | 15 | @Parameters(commandDescription = "Upload the cassandra.yaml fragment to all nodes and apply to cassandra.yaml. Done automatically after use-cassandra.") 16 | class UpdateConfig(val context: Context) : ICommand { 17 | @ParametersDelegate 18 | var hosts = Hosts() 19 | 20 | @Parameter(descriptionKey = "Patch file to upload") 21 | var file: String = "cassandra.patch.yaml" 22 | 23 | @Parameter(names = ["--version"], descriptionKey = "Version to upload, default is current") 24 | var version = "current" 25 | 26 | @Parameter(names = ["--restart", "-r"], descriptionKey = "Restart cassandra after patching") 27 | var restart = false 28 | 29 | override fun execute() { 30 | context.requireSshKey() 31 | // upload the patch file 32 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { 33 | println("Uploading $file to $it") 34 | 35 | val yaml = context.yaml.readTree(Path.of(file).inputStream()) 36 | (yaml as ObjectNode).put("listen_address", it.private) 37 | .put("rpc_address", it.private) 38 | 39 | println("Patching $it") 40 | val tmp = Files.createTempFile("easycasslab", "yaml") 41 | context.yaml.writeValue(tmp.toFile(), yaml) 42 | 43 | // call the patch command on the server 44 | context.upload(it, tmp, file) 45 | tmp.deleteExisting() 46 | val resolvedVersion = context.getRemoteVersion(it, version) 47 | context.executeRemotely(it, "/usr/local/bin/patch-config $file").text 48 | 49 | // Create a temporary directory on the remote filesystem using mktemp 50 | val tempDir = context.executeRemotely(it, "mktemp -d -t easycasslab.XXXXXX").text.trim() 51 | println("Created temporary directory $tempDir on $it") 52 | 53 | // Upload files to the temporary directory first 54 | println("Uploading configuration files to temporary directory $tempDir") 55 | context.uploadDirectory(it, resolvedVersion.file, tempDir) 56 | 57 | // Make sure the destination directory exists 58 | context.executeRemotely(it, "sudo mkdir -p ${resolvedVersion.conf}").text 59 | 60 | // Copy files from temp directory to the final location 61 | println("Copying files from temporary directory to ${resolvedVersion.conf}") 62 | context.executeRemotely(it, "sudo cp -R $tempDir/* ${resolvedVersion.conf}/").text 63 | 64 | // Change ownership of all files 65 | context.executeRemotely(it, "sudo chown -R cassandra:cassandra ${resolvedVersion.conf}").text 66 | 67 | // Clean up the temporary directory 68 | context.executeRemotely(it, "rm -rf $tempDir").text 69 | 70 | println("Configuration updated for $it") 71 | } 72 | 73 | if (restart) { 74 | val restart = Restart(context) 75 | restart.hosts = hosts 76 | restart.execute() 77 | } 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/UploadAuthorizedKeys.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.rustyrazorblade.easycasslab.Context 7 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 8 | import com.rustyrazorblade.easycasslab.configuration.Host 9 | import com.rustyrazorblade.easycasslab.configuration.ServerType 10 | import java.io.File 11 | import java.io.FileFilter 12 | import java.io.FileWriter 13 | import java.nio.file.Paths 14 | import kotlin.io.path.exists 15 | 16 | @Parameters(commandDescription = "Upload authorized (public) keys from the ./authorized_keys directory") 17 | class UploadAuthorizedKeys(val context: Context) : ICommand { 18 | 19 | @Parameter(descriptionKey = "Local directory of authorized keys") 20 | var localDir = "authorized_keys" 21 | 22 | @ParametersDelegate 23 | var hosts = Hosts() 24 | 25 | val authorizedKeysExtra = "~/.ssh/authorized_keys_extra" 26 | val authorizedKeys = "~/.ssh/authorized_keys" 27 | 28 | override fun execute() { 29 | val path = Paths.get(localDir) 30 | if (!path.exists()) { 31 | println("$localDir does not exist") 32 | System.exit(1) 33 | } 34 | 35 | var files = File(localDir).listFiles(FileFilter { it.name.endsWith(".pub") })!! 36 | println("Files: ${files.map { it.name }}") 37 | 38 | // collect all the keys into a single file then upload 39 | val keys = files.joinToString("\n") { it.readText().trim() } 40 | 41 | val authorizedKeysExtra = File("authorized_keys_extra") 42 | FileWriter(authorizedKeysExtra).use { 43 | it.write(keys) 44 | it.write("\n") 45 | } 46 | 47 | println("Uploading the following keys:") 48 | println(keys) 49 | 50 | val upload = doUpload(authorizedKeysExtra) 51 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { upload(it) } 52 | context.tfstate.withHosts(ServerType.Stress, Hosts.all()) { upload(it) } 53 | } 54 | 55 | private fun doUpload(authorizedKeys: File) = { it: Host -> 56 | // Upload the file using Context's upload method 57 | context.upload(it, authorizedKeys.toPath(), authorizedKeysExtra) 58 | 59 | // Append to authorized_keys 60 | context.executeRemotely(it, "cat $authorizedKeysExtra >> /home/ubuntu/.ssh/authorized_keys").text 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/UseCassandra.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.beust.jcommander.ParametersDelegate 6 | import com.fasterxml.jackson.annotation.JsonIgnore 7 | import com.github.ajalt.mordant.TermColors 8 | import com.rustyrazorblade.easycasslab.Context 9 | import com.rustyrazorblade.easycasslab.commands.delegates.Hosts 10 | import com.rustyrazorblade.easycasslab.configuration.* 11 | import org.apache.logging.log4j.kotlin.logger 12 | import java.io.FileNotFoundException 13 | import kotlin.system.exitProcess 14 | 15 | @Parameters(commandDescription = "Use a Cassandra version (3.0, 3.11, 4.0, 4.1)") 16 | class UseCassandra(@JsonIgnore val context: Context) : ICommand { 17 | @Parameter 18 | var version: String = "" 19 | 20 | @ParametersDelegate 21 | var hosts = Hosts() 22 | 23 | @JsonIgnore 24 | val log = logger() 25 | 26 | @Parameter(names = ["--java", "-j"], description = "Java Version Override, 8, 11 or 17 accepted") 27 | var javaVersion = "" 28 | 29 | // @Parameter(names = ["--bti"], description = "Enable BTI Storage") 30 | // var bti = false 31 | 32 | override fun execute() { 33 | check(version.isNotBlank()) 34 | val state = ClusterState.load() 35 | try { 36 | context.tfstate 37 | } catch (e: FileNotFoundException) { 38 | println("Error: terraform config file not found. Please run easy-cass-lab up first to establish IP addresses for seed listing.") 39 | exitProcess(1) 40 | } 41 | 42 | val cassandraHosts = context.tfstate.getHosts(ServerType.Cassandra) 43 | println("Using version ${version} on ${cassandraHosts.size} hosts, filter: $hosts") 44 | 45 | context.tfstate.withHosts(ServerType.Cassandra, hosts) { 46 | if (javaVersion.isNotBlank()) { 47 | context.executeRemotely(it, "set-java-version ${javaVersion} ${version}") 48 | } 49 | context.executeRemotely(it, "sudo use-cassandra ${version}").text 50 | state.versions?.put(it.alias, version) 51 | } 52 | 53 | state.save() 54 | 55 | DownloadConfig(context).execute() 56 | 57 | // make sure we only apply to the filtered hosts 58 | val uc = UpdateConfig(context) 59 | uc.hosts = hosts 60 | uc.execute() 61 | 62 | with (TermColors()) { 63 | println("You can update ${green("cassandra.patch.yaml")} and the JVM config files under ${green(version)}, " + 64 | "then run ${green("easy-cass-lab update-config")} to apply the changes.") 65 | } 66 | } 67 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/Version.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.rustyrazorblade.easycasslab.Context 4 | 5 | class Version(val context: Context) : ICommand { 6 | override fun execute() { 7 | println(context.version) 8 | } 9 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/WriteConfig.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.fasterxml.jackson.annotation.JsonIgnore 6 | import com.rustyrazorblade.easycasslab.Context 7 | import com.rustyrazorblade.easycasslab.configuration.ClusterState 8 | import com.rustyrazorblade.easycasslab.configuration.ServerType 9 | 10 | import java.io.File 11 | 12 | 13 | @Parameters(commandDescription = "Write a new cassandra configuration patch file") 14 | class WriteConfig(@JsonIgnore val context: Context) : ICommand { 15 | @Parameter(description = "Patch file name") 16 | var file: String = "cassandra.patch.yaml" 17 | 18 | @Parameter(names = ["-t", "--tokens"]) 19 | var tokens: Int = 4 20 | 21 | override fun execute() { 22 | println("Writing new configuration file to $file.") // create the cassandra.yaml patch file 23 | println("It can be applied to the lab via easy-cass-lab update-config (or automatically when calling use-cassandra)") 24 | 25 | val state = ClusterState.load() 26 | 27 | val data = object { 28 | val cluster_name = state.name 29 | val num_tokens = tokens 30 | val seed_provider = object { 31 | val class_name = "org.apache.cassandra.locator.SimpleSeedProvider" 32 | val parameters = object { 33 | val seeds = context.tfstate.getHosts(ServerType.Cassandra).map{ it.private }.take(1).joinToString(",") 34 | } 35 | } 36 | val hints_directory = "/mnt/cassandra/hints" 37 | val data_file_directories = listOf("/mnt/cassandra/data") 38 | val commitlog_directory = "/mnt/cassandra/commitlog" 39 | val concurrent_reads = 64 40 | val concurrent_writes = 64 41 | val trickle_fsync = true 42 | val endpoint_snitch = "Ec2Snitch" 43 | } 44 | 45 | context.yaml.writeValue(File("cassandra.patch.yaml"), data) 46 | } 47 | 48 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/converters/AZConverter.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands.converters 2 | 3 | import com.beust.jcommander.IStringConverter 4 | 5 | /** 6 | * The AZConverter is meant to be crazy simple and if a person is guessing, they'll just guess right 7 | * If somebody were to enter any of the following after --az, it should parse: 8 | * 9 | * abc 10 | * a,b,c 11 | * "a b , c" 12 | * 13 | * For now we don't care about regions - if a user enters this: 14 | * 15 | * us-west-2b 16 | * 17 | * It'll just fail. We might follow up to fix this is we see it's an issue. 18 | */ 19 | class AZConverter : IStringConverter> { 20 | override fun convert(value: String?): List { 21 | 22 | if(value == null) return listOf() 23 | 24 | return value.split("").filter { it.matches("[a-z]".toRegex()) } 25 | } 26 | 27 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/delegates/BuildArgs.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands.delegates 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.rustyrazorblade.easycasslab.Context 5 | 6 | enum class Arch(val type: String) { 7 | amd64("amd64"), 8 | arm64("arm64"), 9 | } 10 | 11 | class BuildArgs(val context: Context) { 12 | @Parameter(description = "Release flag", names = ["--release"]) 13 | var release: Boolean = false 14 | 15 | @Parameter(description = "AWS region to build the image in", names = ["--region", "-r"]) 16 | var region = context.userConfig.region 17 | 18 | @Parameter(description = "CPU architecture", names = ["--arch", "-a", "--cpu"]) 19 | var arch = Arch.amd64 20 | } 21 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/delegates/Hosts.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands.delegates 2 | 3 | import com.beust.jcommander.Parameter 4 | 5 | class Hosts { 6 | @Parameter(description = "Hosts to run this on, leave blank for all hosts.", names = ["--hosts"]) 7 | var hosts = "" 8 | companion object { 9 | fun all() = Hosts() 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/commands/formatters/HostOutput.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.commands.formatters 2 | 3 | class HostOutput { 4 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/CassandraVersion.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties 4 | import com.fasterxml.jackson.annotation.JsonInclude 5 | import com.fasterxml.jackson.databind.ObjectMapper 6 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 7 | import com.fasterxml.jackson.module.kotlin.registerKotlinModule 8 | import org.apache.logging.log4j.kotlin.logger 9 | import java.io.File 10 | import java.io.OutputStream 11 | import java.nio.file.Files 12 | import java.nio.file.Path 13 | import kotlin.io.path.exists 14 | import kotlin.io.path.isDirectory 15 | import kotlin.io.path.listDirectoryEntries 16 | 17 | @JsonInclude(JsonInclude.Include.NON_EMPTY) 18 | data class CassandraVersion( 19 | val version: String, 20 | val java: String, 21 | val python: String, 22 | 23 | @JsonIgnoreProperties(ignoreUnknown = true) 24 | val axonops: String? = null, 25 | @JsonIgnoreProperties(ignoreUnknown = true) 26 | val jvm_options: String?, 27 | @JsonIgnoreProperties(ignoreUnknown = true) 28 | val ant_flags: String? = null, 29 | @JsonIgnoreProperties(ignoreUnknown = true) 30 | val url: String? = null, 31 | @JsonIgnoreProperties(ignoreUnknown = true) 32 | val branch: String? = null, 33 | @JsonIgnoreProperties(ignoreUnknown = true) 34 | val java_build: String? = null, 35 | @JsonIgnoreProperties(ignoreUnknown = true) 36 | val jvm_config: String? = null 37 | ) { 38 | companion object { 39 | private val objectMapper = ObjectMapper(YAMLFactory()).registerKotlinModule() 40 | private var logger = logger() 41 | 42 | fun loadFromFile(filePath: Path): List { 43 | val fileContent = Files.readString(filePath) 44 | return objectMapper.readValue(fileContent, objectMapper.typeFactory.constructCollectionType(List::class.java, CassandraVersion::class.java)) 45 | } 46 | 47 | fun loadFromMainAndExtras(mainFilePath: Path, extrasDirectoryPath: Path): List { 48 | // Load main file 49 | val mainVersions = loadFromFile(mainFilePath).toMutableList() 50 | 51 | // Load each file in the extras directory 52 | if (extrasDirectoryPath.exists() && extrasDirectoryPath.isDirectory()) { 53 | val listDirectoryEntries = extrasDirectoryPath.listDirectoryEntries("*.yaml") 54 | logger.info("Loading from ${listDirectoryEntries.size} extra potential files" ) 55 | for (file in listDirectoryEntries) { 56 | logger.info("Loading additional cassandra_versions file: $file") 57 | val extraVersions = loadFromFile(file) 58 | logger.info("Adding ${extraVersions.size} versions") 59 | mainVersions.addAll(extraVersions) 60 | } 61 | } else { 62 | logger.info("Nothing to load") 63 | } 64 | 65 | // Remove duplicates based on the version field 66 | // TODO: improve the error message here 67 | val tmp = mainVersions.distinctBy { it.version } 68 | if (tmp.size != mainVersions.size) { 69 | throw RuntimeException("version conflict found") 70 | } 71 | return mainVersions 72 | } 73 | 74 | fun write(versions: List, outputStream: OutputStream) { 75 | objectMapper.writeValue(outputStream, versions) 76 | } 77 | fun write(versions: List, outputFile: File) { 78 | objectMapper.writeValue(outputFile, versions) 79 | } 80 | } 81 | 82 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/CassandraYaml.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import com.fasterxml.jackson.databind.JsonNode 4 | import com.fasterxml.jackson.databind.ObjectMapper 5 | import com.fasterxml.jackson.databind.node.ObjectNode 6 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 7 | import java.io.File 8 | import java.io.InputStream 9 | 10 | class CassandraYaml(val parser: JsonNode) { 11 | 12 | companion object { 13 | val mapper = ObjectMapper(YAMLFactory()) 14 | 15 | fun create(fp: File) : CassandraYaml { 16 | val tmp = mapper.readTree(fp) 17 | return CassandraYaml(tmp) 18 | } 19 | 20 | fun create(inputStream: InputStream) : CassandraYaml { 21 | val tmp = mapper.readTree(inputStream) 22 | return CassandraYaml(tmp) 23 | } 24 | } 25 | fun setProperty(name: String, value: String) { 26 | val tmp = parser as ObjectNode 27 | tmp.put(name, value) 28 | } 29 | 30 | 31 | fun setSeeds(seeds: List) { 32 | val seedNode = parser.get("seed_provider").first().get("parameters").first() 33 | val tmp = seedNode as ObjectNode 34 | val seedList = seeds.joinToString(",") 35 | tmp.put("seeds", seedList) 36 | } 37 | 38 | /** 39 | * Convenience method 40 | */ 41 | fun setSeeds(seeds: Seeds) { 42 | setSeeds(seeds.seeds) 43 | } 44 | 45 | fun write(path: String) { 46 | mapper.writeValue(File(path), parser) 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/ClusterState.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore 4 | import com.fasterxml.jackson.databind.ObjectMapper 5 | import com.fasterxml.jackson.module.kotlin.registerKotlinModule 6 | import java.io.File 7 | 8 | 9 | /** 10 | * Tracking state across multiple commands 11 | */ 12 | const val CLUSTER_STATE = "state.json" 13 | 14 | data class NodeState( 15 | var version: String = "", 16 | var javaVersion: String = "" 17 | ) 18 | 19 | data class ClusterState( 20 | 21 | var name: String, 22 | // if we fire up a new node and just tell it to go, it should use all the defaults 23 | var default: NodeState = NodeState(), 24 | // we also have a per-node mapping that lets us override, per node 25 | var nodes: MutableMap = mutableMapOf(), 26 | 27 | var versions: MutableMap? 28 | 29 | ) { 30 | 31 | companion object { 32 | @JsonIgnore 33 | private val mapper = ObjectMapper().registerKotlinModule() 34 | 35 | @JsonIgnore 36 | var fp = File(CLUSTER_STATE) 37 | 38 | fun load() = 39 | mapper.readValue(fp, ClusterState::class.java) 40 | } 41 | 42 | fun save() = 43 | mapper.writerWithDefaultPrettyPrinter().writeValue(fp, this) 44 | 45 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/Dashboards.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import org.apache.commons.io.FileUtils 4 | import org.reflections.Reflections 5 | import org.reflections.scanners.ResourcesScanner 6 | import java.io.File 7 | 8 | 9 | /** 10 | * simple class to manage copying dashboards to the right directory 11 | */ 12 | class Dashboards(private val dashboardLocation: File) { 13 | fun copyDashboards() { 14 | val reflections = Reflections("com.rustyrazorblade.dashboards", ResourcesScanner()) 15 | val resources = reflections.getResources(".*".toPattern()) 16 | for(f in resources) { 17 | val input = this.javaClass.getResourceAsStream("/" + f) 18 | val outputFile = f.replace("com/rustyrazorblade/dashboards", "") 19 | val output = File(dashboardLocation, outputFile) 20 | println("Writing ${output.absolutePath}") 21 | FileUtils.copyInputStreamToFile(input, output) 22 | } 23 | 24 | } 25 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/Host.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | 5 | typealias Alias = String 6 | 7 | data class Host(val public: String, 8 | val private: String, 9 | val alias: Alias, 10 | val availabilityZone: String) { 11 | 12 | companion object { 13 | val hostRegex = """aws_instance\.(\w+)(.(\d+))?""".toRegex() 14 | val log = logger() 15 | 16 | fun fromTerraformString(str: String, public: String, private: String, availabilityZone: String) : Host { 17 | val tmp = hostRegex.find(str)!!.groups 18 | 19 | val serverType = tmp[1]?.value.toString() 20 | val serverNum = (tmp[3]?.value ?: 0).toString() 21 | 22 | log.debug { "Regex find: $tmp" } 23 | return Host(public, private, serverType + serverNum, availabilityZone) 24 | 25 | } 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/HostInfo.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore 4 | import com.fasterxml.jackson.annotation.JsonProperty 5 | 6 | data class HostInfo(@JsonIgnore var address: String = "", 7 | @JsonProperty("instance") var name: String = "", 8 | var environment: String = "", 9 | var cluster: String = "", 10 | var datacenter: String = "", 11 | var rack: String = "") -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/Seeds.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import org.apache.commons.io.IOUtils 4 | import java.io.InputStream 5 | import java.io.StringWriter 6 | 7 | data class Seeds(val seeds: List) { 8 | companion object { 9 | fun open(stream: InputStream) : Seeds { 10 | val buf = StringWriter() 11 | IOUtils.copy(stream, buf) 12 | val seeds = buf.toString().split("\n") 13 | return Seeds(seeds) 14 | } 15 | } 16 | 17 | override fun toString(): String { 18 | return seeds.joinToString(",") 19 | } 20 | 21 | 22 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/ServerType.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | enum class ServerType(val serverType: String) { 4 | Cassandra("cassandra"), 5 | Stress("stress"), 6 | Monitoring("monitoring"), 7 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/configuration/User.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.configuration 2 | 3 | import com.rustyrazorblade.easycasslab.Context 4 | import com.rustyrazorblade.easycasslab.EC2 5 | import com.rustyrazorblade.easycasslab.Utils 6 | import org.apache.logging.log4j.kotlin.logger 7 | import software.amazon.awssdk.regions.Region 8 | import java.io.File 9 | import software.amazon.awssdk.services.ec2.model.CreateKeyPairRequest 10 | import java.nio.file.Files 11 | import java.util.* 12 | import java.nio.file.attribute.PosixFilePermission 13 | import java.util.HashSet 14 | 15 | 16 | data class User( 17 | var email : String, 18 | var region: String, 19 | var keyName: String, 20 | var sshKeyPath: String, 21 | 22 | // if true we'll load the profile from the AWS credentials rather than this file 23 | // can over 24 | var awsProfile: String, 25 | // fallback for people who haven't set up the aws cli 26 | 27 | var awsAccessKey: String, 28 | var awsSecret: String, 29 | 30 | var axonOpsOrg: String = "", 31 | var axonOpsKey: String = "" 32 | ) { 33 | companion object { 34 | 35 | val log = logger() 36 | 37 | /** 38 | * Asks a bunch of questions and generates the user file 39 | */ 40 | fun createInteractively(context: Context, location: File) { 41 | println("Welcome to the easy-cass-lab interactive setup.") 42 | println("We just need to know a few things before we get started.") 43 | 44 | val email = Utils.prompt("What's your email?", "") 45 | 46 | // we're not honoring it, so we'll take this out 47 | val regionAnswer = Utils.prompt("What AWS region do you use?", "us-west-2") 48 | val region = Region.of(regionAnswer) 49 | 50 | val awsAccessKey = Utils.prompt("Please enter your AWS Access Key:", "") 51 | val awsSecret = Utils.prompt("Please enter your AWS Secret Access Key:", "", secret = true) 52 | 53 | // create the key pair 54 | 55 | println("Attempting to validate credentials and generate easy-cass-lab login keys") 56 | val ec2 = EC2(awsAccessKey, awsSecret, region) 57 | val ec2Client = ec2.client 58 | 59 | val keyName = "easy-cass-lab-${UUID.randomUUID()}" 60 | val request = CreateKeyPairRequest.builder() 61 | .keyName(keyName).build() 62 | 63 | val response = ec2Client.createKeyPair(request) 64 | 65 | // write the private key into the ~/.easy-cass-lab/profiles// dir 66 | 67 | val secret = File(context.profileDir, "secret.pem") 68 | secret.writeText(response.keyMaterial()) 69 | 70 | fun getAxonOps(inputName : String) = 71 | Utils.prompt("AxonOps $inputName: ", "") 72 | 73 | val axonOpsChoice = Utils.prompt("Use AxonOps (https://axonops.com/) for monitoring. Requires an account. [y/N]", default = "N") 74 | val useAxonOps = axonOpsChoice.equals("y", true); 75 | val axonOpsOrg = if (useAxonOps) getAxonOps("Org") else "" 76 | val axonOpsKey = if (useAxonOps) getAxonOps("Key") else "" 77 | 78 | // set permissions 79 | val perms = HashSet() 80 | perms.add(PosixFilePermission.OWNER_READ) 81 | perms.add(PosixFilePermission.OWNER_WRITE) 82 | 83 | log.info { "Setting secret file permissions $perms"} 84 | Files.setPosixFilePermissions(secret.toPath(), perms) 85 | 86 | 87 | val user = User( 88 | email, 89 | region.toString(), 90 | keyName, 91 | secret.absolutePath, 92 | "", // future compatibility, when we start allowing people to use their existing AWS creds they've already set up. 93 | awsAccessKey, 94 | awsSecret, 95 | axonOpsOrg, 96 | axonOpsKey) 97 | 98 | context.yaml.writeValue(location, user) 99 | } 100 | } 101 | } 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/containers/Packer.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.containers 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 5 | import com.github.dockerjava.api.model.AccessMode 6 | import com.rustyrazorblade.easycasslab.Containers 7 | import com.rustyrazorblade.easycasslab.Context 8 | import com.rustyrazorblade.easycasslab.Docker 9 | import com.rustyrazorblade.easycasslab.VolumeMapping 10 | import com.rustyrazorblade.easycasslab.commands.delegates.BuildArgs 11 | import com.rustyrazorblade.easycasslab.configuration.CassandraVersion 12 | import org.apache.logging.log4j.kotlin.logger 13 | import java.io.File 14 | import kotlin.system.exitProcess 15 | import kotlin.io.path.createTempDirectory 16 | import org.apache.commons.io.FileUtils 17 | import java.nio.file.Path 18 | 19 | 20 | class Packer(val context: Context, var directory: String) { 21 | private val docker = Docker(context) 22 | 23 | private var containerWorkingDir = "/local" 24 | private var logger = logger() 25 | private var release = false 26 | 27 | // todo include the region defined in the profile 28 | fun build(name: String, buildArgs: BuildArgs) { 29 | val command = mutableListOf("build", 30 | "-var", "region=${buildArgs.region}", 31 | "-var", "arch=${buildArgs.arch.type}") 32 | 33 | if (buildArgs.release) { 34 | // When passing the release flag, 35 | // we use the release version as the image version. 36 | // We also make the AMI public. 37 | release = true 38 | command.addAll( 39 | arrayOf("-var", "release_version=${context.version}")) 40 | } 41 | 42 | command.add(name) 43 | 44 | // refactor to exit with status 1 if the Result is failure 45 | val result = execute(*command.toTypedArray()) 46 | when { 47 | result.isFailure -> { 48 | logger.error("Packer build failed: ${result.exceptionOrNull()}") 49 | exitProcess(1) 50 | } 51 | result.isSuccess -> { 52 | logger.info("Packer build succeeded") 53 | } 54 | } 55 | } 56 | 57 | private fun execute(vararg commands: String): Result { 58 | docker.pullImage(Containers.PACKER) 59 | 60 | val args = commands.toMutableList() 61 | 62 | var localPackerPath = context.packerHome + directory 63 | 64 | if (!File(localPackerPath).exists()) { 65 | println("packer directory not found: $localPackerPath") 66 | exitProcess(1) 67 | } 68 | 69 | val tempDir = createTempDirectory().toFile() 70 | FileUtils.copyDirectory(File(localPackerPath), tempDir) 71 | logger.info("Copied packer files from $localPackerPath to $tempDir") 72 | 73 | if (!release && directory == "cassandra") { 74 | // if we're doing a C* image, we 75 | val initial = Path.of(localPackerPath, "cassandra_versions.yaml") 76 | val extras = context.cassandraVersionsExtra.toPath() 77 | logger.info("Loading files in $extras") 78 | 79 | val versions = CassandraVersion.loadFromMainAndExtras(initial, extras) 80 | val outputFile = File(tempDir, "cassandra_versions.yaml") 81 | CassandraVersion.write(versions, outputFile) 82 | logger.info("Written updated versions to $outputFile") 83 | } 84 | 85 | logger.info("Mounting $tempDir to $containerWorkingDir, starting with $args") 86 | 87 | // mount credentials 88 | // get the main process and go up a directory 89 | val packerDir = VolumeMapping(tempDir.absolutePath, containerWorkingDir, AccessMode.ro) 90 | var creds = "/credentials" 91 | 92 | return docker 93 | .addVolume(packerDir) 94 | .addVolume(VolumeMapping(context.awsConfig.absolutePath, creds, AccessMode.ro)) 95 | .addEnv("AWS_SHARED_CREDENTIALS_FILE=$creds") 96 | .runContainer(Containers.PACKER, args, containerWorkingDir) 97 | } 98 | 99 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/containers/Terraform.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.containers 2 | 3 | import com.rustyrazorblade.easycasslab.Context 4 | import com.rustyrazorblade.easycasslab.Docker 5 | import com.github.dockerjava.api.model.AccessMode 6 | import com.rustyrazorblade.easycasslab.Containers 7 | import com.rustyrazorblade.easycasslab.VolumeMapping 8 | import org.apache.logging.log4j.kotlin.logger 9 | 10 | 11 | class Terraform(val context: Context) { 12 | 13 | private val docker = Docker(context) 14 | 15 | private var localDirectory = "/local" 16 | private var logger = logger() 17 | 18 | fun init() : Result { 19 | return execute("init") 20 | } 21 | 22 | 23 | fun up() : Result { 24 | val commands = mutableListOf("apply", "-auto-approve").toTypedArray() 25 | return execute(*commands) 26 | } 27 | 28 | 29 | fun down(autoApprove: Boolean) : Result { 30 | val commands = mutableListOf("destroy") 31 | if(autoApprove) { 32 | commands.add("-auto-approve") 33 | } 34 | return execute(*commands.toTypedArray()) 35 | } 36 | 37 | 38 | private fun execute(vararg command: String) : Result { 39 | val args = command.toMutableList() 40 | docker.pullImage(Containers.TERRAFORM) 41 | var mount = "/${context.awsCredentialsName}" 42 | logger.info("Mounting credentials at ${context.awsConfig.absolutePath}:$mount") 43 | 44 | return docker 45 | .addVolume(VolumeMapping(context.cwdPath, "/local", AccessMode.rw)) 46 | .addVolume(VolumeMapping(context.terraformCacheDir.absolutePath, "/tcache", AccessMode.rw)) 47 | .addVolume(VolumeMapping(context.awsConfig.absolutePath, mount, AccessMode.ro)) 48 | .addEnv("TF_PLUGIN_CACHE_DIR=/tcache") 49 | .runContainer(Containers.TERRAFORM, args, localDirectory) 50 | } 51 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/ssh/ConnectionManager.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.ssh 2 | 3 | import com.rustyrazorblade.easycasslab.configuration.Host 4 | import org.apache.logging.log4j.kotlin.logger 5 | import org.apache.sshd.client.SshClient 6 | import org.apache.sshd.common.keyprovider.KeyIdentityProvider 7 | import org.apache.sshd.common.util.security.SecurityUtils 8 | import kotlin.io.path.Path 9 | import java.security.KeyPair 10 | import java.time.Duration 11 | 12 | /** 13 | * Manages SSH connections to multiple hosts 14 | */ 15 | open class ConnectionManager(val keyPath: String) { 16 | private val log = logger() 17 | private val keyPairs: List 18 | private val sshClient: SshClient 19 | private val connections = mutableMapOf() 20 | 21 | init { 22 | // Load key pairs 23 | val loader = SecurityUtils.getKeyPairResourceParser() 24 | keyPairs = loader.loadKeyPairs(null, Path(keyPath), null).toList() 25 | 26 | // Set up SSH client 27 | sshClient = SshClient.setUpDefaultClient() 28 | sshClient.setKeyIdentityProvider(KeyIdentityProvider.wrapKeyPairs(keyPairs)) 29 | sshClient.start() 30 | } 31 | 32 | /** 33 | * Get a connection for the given host, creating one if it doesn't exist 34 | */ 35 | open fun getConnection(host: Host): ISSHClient { 36 | return connections.getOrPut(host) { 37 | val session = sshClient.connect("ubuntu", host.public, 22) 38 | .verify(Duration.ofSeconds(60)) 39 | .session 40 | session.addPublicKeyIdentity(keyPairs.first()) 41 | session.auth().verify() 42 | SSHClient(session) 43 | } 44 | } 45 | 46 | /** 47 | * Close all connections and stop the SSH client 48 | */ 49 | fun stop() { 50 | log.debug { "Stopping SSH client and closing all connections" } 51 | connections.values.forEach { it.close() } 52 | connections.clear() 53 | sshClient.stop() 54 | } 55 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/ssh/ISSHClient.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.ssh 2 | 3 | import com.rustyrazorblade.easycasslab.configuration.Host 4 | import org.apache.sshd.scp.client.CloseableScpClient 5 | import java.io.File 6 | import java.nio.file.Path 7 | 8 | /** 9 | * Interface for SSH operations 10 | * Allows for testing with mock implementations 11 | */ 12 | interface ISSHClient { 13 | /** 14 | * Execute a command on a remote host 15 | */ 16 | fun executeRemoteCommand(command: String, output: Boolean, secret: Boolean): Response 17 | 18 | /** 19 | * Upload a file to a remote host 20 | */ 21 | fun uploadFile(local: Path, remote: String) 22 | 23 | /** 24 | * Upload a directory to a remote host 25 | */ 26 | fun uploadDirectory(localDir: File, remoteDir: String) 27 | 28 | /** 29 | * Download a file from a remote host 30 | */ 31 | fun downloadFile(remote: String, local: Path) 32 | 33 | /** 34 | * Download a directory from a remote host 35 | * 36 | * @param remoteDir The remote directory to download 37 | * @param localDir The local directory where files will be downloaded 38 | * @param includeFilters Optional list of patterns to filter files for download 39 | * @param excludeFilters Optional list of patterns to exclude files from download 40 | */ 41 | fun downloadDirectory(remoteDir: String, localDir: File, includeFilters: List = listOf(), excludeFilters: List = listOf()) 42 | 43 | /** 44 | * Get an SCP client for this connection 45 | */ 46 | fun getScpClient(): CloseableScpClient 47 | 48 | /** 49 | * Close the connection 50 | */ 51 | fun close() {} 52 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/ssh/MockSSHClient.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.ssh 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | import org.apache.sshd.client.session.ClientSession 5 | import org.apache.sshd.scp.client.CloseableScpClient 6 | import org.apache.sshd.scp.client.ScpClient 7 | import org.apache.sshd.scp.common.helpers.ScpTimestampCommandDetails 8 | import java.io.File 9 | import java.io.InputStream 10 | import java.io.OutputStream 11 | import java.nio.file.Path 12 | import java.nio.file.attribute.PosixFilePermission 13 | 14 | /** 15 | * Mock implementation of ISSHClient for testing 16 | */ 17 | class MockSSHClient : ISSHClient { 18 | private val log = logger() 19 | val executedCommands = mutableListOf() 20 | val uploadedFiles = mutableListOf>() 21 | val uploadedDirectories = mutableListOf>() 22 | val downloadedFiles = mutableListOf>() 23 | val downloadedDirectories = mutableListOf>() 24 | 25 | /** 26 | * Mock command output can be customized per test 27 | */ 28 | var commandOutput = "" 29 | 30 | override fun executeRemoteCommand(command: String, output: Boolean, secret: Boolean): Response { 31 | log.debug { "MOCK: Executing command: $command" } 32 | executedCommands.add(command) 33 | return Response(commandOutput) 34 | } 35 | 36 | override fun uploadFile(local: Path, remote: String) { 37 | log.debug { "MOCK: Uploading file from ${local.toAbsolutePath()} to $remote" } 38 | uploadedFiles.add(Pair(local, remote)) 39 | } 40 | 41 | override fun uploadDirectory(localDir: File, remoteDir: String) { 42 | log.debug { "MOCK: Uploading directory from ${localDir.absolutePath} to $remoteDir" } 43 | uploadedDirectories.add(Pair(localDir, remoteDir)) 44 | } 45 | 46 | override fun downloadFile(remote: String, local: Path) { 47 | log.debug { "MOCK: Downloading file from $remote to ${local.toAbsolutePath()}" } 48 | downloadedFiles.add(Pair(remote, local)) 49 | } 50 | 51 | override fun downloadDirectory(remoteDir: String, localDir: File, includeFilters: List, excludeFilters: List) { 52 | log.debug { "MOCK: Downloading directory from $remoteDir to ${localDir.absolutePath} with includeFilters: $includeFilters, excludeFilters: $excludeFilters" } 53 | downloadedDirectories.add(Pair(remoteDir, localDir)) 54 | } 55 | 56 | override fun getScpClient(): CloseableScpClient { 57 | return object : CloseableScpClient { 58 | override fun upload(local: Path, remote: String, vararg options: ScpClient.Option) {} 59 | override fun upload(locals: Array, remote: String, options: Collection) {} 60 | override fun upload( 61 | p0: InputStream?, 62 | p1: String?, 63 | p2: Long, 64 | p3: MutableCollection?, 65 | p4: ScpTimestampCommandDetails? 66 | ) { 67 | TODO("Not yet implemented") 68 | } 69 | 70 | override fun upload(locals: Array, remote: String, options: Collection) {} 71 | override fun getClientSession(): ClientSession { 72 | TODO("Not yet implemented") 73 | } 74 | 75 | override fun download(p0: String?, p1: String?, p2: MutableCollection?) { 76 | TODO("Not yet implemented") 77 | } 78 | 79 | override fun download(remote: String, local: Path, vararg options: ScpClient.Option) {} 80 | override fun download(p0: String?, p1: Path?, p2: MutableCollection?) { 81 | TODO("Not yet implemented") 82 | } 83 | 84 | override fun download(p0: String?, p1: OutputStream?) { 85 | TODO("Not yet implemented") 86 | } 87 | 88 | override fun download(p0: Array?, p1: String?, p2: MutableCollection?) { 89 | TODO("Not yet implemented") 90 | } 91 | 92 | override fun download(remotes: Array, local: Path, options: Collection) {} 93 | override fun close() {} 94 | override fun isOpen(): Boolean { 95 | TODO("Not yet implemented") 96 | } 97 | } 98 | } 99 | 100 | override fun close() { 101 | log.debug { "MOCK: Stopping SSH client" } 102 | } 103 | 104 | /** 105 | * Reset all recorded calls (useful between tests) 106 | */ 107 | fun reset() { 108 | executedCommands.clear() 109 | uploadedFiles.clear() 110 | uploadedDirectories.clear() 111 | downloadedFiles.clear() 112 | downloadedDirectories.clear() 113 | commandOutput = "" 114 | } 115 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/ssh/Response.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.ssh 2 | 3 | /** 4 | * Represents a response from a remote command execution 5 | */ 6 | data class Response(val text: String) -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycasslab/ssh/SSHClient.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycasslab.ssh 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | import org.apache.sshd.client.session.ClientSession 5 | import org.apache.sshd.scp.client.CloseableScpClient 6 | import org.apache.sshd.scp.client.ScpClientCreator 7 | import java.io.File 8 | import java.nio.file.Path 9 | 10 | /** 11 | * Main class for SSH operations 12 | * Acts as a facade for all SSH-related functionality 13 | */ 14 | class SSHClient(private val session: ClientSession) : ISSHClient { 15 | private val log = logger() 16 | 17 | /** 18 | * Execute a command on a remote host 19 | * 20 | * @param command The command to execute 21 | * @param output Whether to print the command output 22 | * @param secret Whether the command contains sensitive information 23 | * @return The command output wrapped in a Response object 24 | */ 25 | override fun executeRemoteCommand(command: String, output: Boolean, secret: Boolean): Response { 26 | // Create connection for this host 27 | if (!secret) { 28 | println("Executing remote command: $command") 29 | } else { 30 | println("Executing remote command: [hidden]") 31 | } 32 | 33 | val result = session.executeRemoteCommand(command) 34 | 35 | if (output) { 36 | println(result) 37 | } 38 | 39 | return Response(result) 40 | } 41 | 42 | /** 43 | * Upload a file to a remote host 44 | */ 45 | override fun uploadFile(local: Path, remote: String) { 46 | println( "Uploading file ${local.toAbsolutePath()} to ${session}:$remote" ) 47 | getScpClient().upload(local, remote) 48 | } 49 | 50 | /** 51 | * Upload a directory to a remote host 52 | */ 53 | override fun uploadDirectory(localDir: File, remoteDir: String) { 54 | if (!localDir.exists() || !localDir.isDirectory) { 55 | log.error { "Local directory $localDir does not exist or is not a directory" } 56 | return 57 | } 58 | 59 | println("Uploading directory ${localDir.absolutePath} to ${session}:$remoteDir") 60 | 61 | executeRemoteCommand("mkdir -p $remoteDir", false, false) 62 | 63 | // Process each file in the directory 64 | localDir.listFiles()?.forEach { file -> 65 | val relativePath = file.toRelativeString(localDir) 66 | val remotePath = "$remoteDir/$relativePath" 67 | 68 | if (file.isDirectory) { 69 | // Recursively upload subdirectories 70 | uploadDirectory(file, remotePath) 71 | } else { 72 | // Upload individual file 73 | uploadFile(file.toPath(), remotePath) 74 | } 75 | } 76 | } 77 | 78 | /** 79 | * Download a file from a remote host 80 | */ 81 | override fun downloadFile(remote: String, local: Path) { 82 | log.debug { "Downloading file from ${session} ${remote} to ${local.toAbsolutePath()}" } 83 | getScpClient().download(remote, local) 84 | } 85 | 86 | /** 87 | * Download a directory from a remote host 88 | * 89 | * @param remoteDir The remote directory to download 90 | * @param localDir The local directory where files will be downloaded 91 | * @param includeFilters Optional list of patterns to filter files for download 92 | * @param excludeFilters Optional list of patterns to exclude files from download 93 | */ 94 | override fun downloadDirectory(remoteDir: String, localDir: File, includeFilters: List, excludeFilters: List) { 95 | if (!localDir.exists()) { 96 | localDir.mkdirs() 97 | } 98 | 99 | log.debug { "Downloading directory from ${session}:$remoteDir to ${localDir.absolutePath}" } 100 | 101 | val fileListOutput = executeRemoteCommand("find $remoteDir -type f", false, false) 102 | val remoteFiles = fileListOutput.text.split("\n").filter { it.isNotEmpty() } 103 | 104 | // Download each file 105 | for (remoteFile in remoteFiles) { 106 | val relativePath = remoteFile.removePrefix("$remoteDir/") 107 | val fileName = relativePath.substringAfterLast("/") 108 | 109 | // Skip if file matches exclude filter 110 | if (excludeFilters.isNotEmpty()) { 111 | val matchesExcludeFilter = excludeFilters.any { pattern -> 112 | fileName.matches(pattern.replace("*", ".*").toRegex()) 113 | } 114 | if (matchesExcludeFilter) continue 115 | } 116 | 117 | // Skip if include filters are specified and file doesn't match 118 | if (includeFilters.isNotEmpty()) { 119 | val matchesIncludeFilter = includeFilters.any { pattern -> 120 | fileName.matches(pattern.replace("*", ".*").toRegex()) 121 | } 122 | if (!matchesIncludeFilter) continue 123 | } 124 | 125 | val localFile = File(localDir, relativePath) 126 | 127 | // Ensure parent directory exists 128 | localFile.parentFile.mkdirs() 129 | 130 | downloadFile(remoteFile, localFile.toPath()) 131 | } 132 | } 133 | 134 | override fun getScpClient(): CloseableScpClient { 135 | val creator = ScpClientCreator.instance() 136 | val client = creator.createScpClient(session) 137 | val scpClient = CloseableScpClient.singleSessionInstance(client) 138 | return scpClient 139 | } 140 | /** 141 | * Stop the SSH client 142 | */ 143 | override fun close() { 144 | log.debug { "Stopping SSH client" } 145 | session.close() 146 | } 147 | 148 | } -------------------------------------------------------------------------------- /src/main/resources/com/rustyrazorblade/easycasslab/commands/setup_instance.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ###### CONFIGURATION ###### 4 | ## ANY VARIABLE NEEDED IN THIS SCRIPT 5 | ## SHOULD BE SET IN THIS BLOCK 6 | 7 | export READAHEAD=8 8 | 9 | DISK="" 10 | 11 | for VOL in nvme0n1 nvme1n1 xvdb; do 12 | export VOL 13 | echo "Checking $VOL" 14 | TMP=$(lsblk -o NAME,MOUNTPOINTS -J | yq '.blockdevices[] | select(.name == env(VOL)) | has("children")') 15 | echo $TMP 16 | 17 | if [[ "${TMP}" == "false" ]]; then 18 | DISK="/dev/$VOL" 19 | break 20 | fi 21 | done 22 | 23 | echo "Using disk: $DISK" 24 | 25 | ## END CONFIGURATION ### 26 | ########################### 27 | 28 | ###### SYSTEM SETTINGS // OS TUNINGS ##### 29 | 30 | sudo sysctl kernel.perf_event_paranoid=1 31 | sudo sysctl kernel.kptr_restrict=0 32 | 33 | echo 0 > /proc/sys/vm/zone_reclaim_mode 34 | 35 | cat <